From ab17c9eef04f8a0adc5080e9e55cb03ab572b2b0 Mon Sep 17 00:00:00 2001 From: simondalzell-aquaq Date: Fri, 25 Nov 2022 14:53:14 +0000 Subject: [PATCH 01/15] Included addproc.sh --- addproc.sh | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 addproc.sh diff --git a/addproc.sh b/addproc.sh new file mode 100644 index 000000000..f33182ee2 --- /dev/null +++ b/addproc.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +#before running the script ensure the stack is running +#this script takes an existing procname as an argument to determine what process to replicate +#procname is configured automatically based on the last process replica name +#all new processes are started automatically to avoid port number overlap upon running the script multiple times without starting the new procs first +#new processes are written to customprocess.csv which also contains the original processes from process.csv (this may have to be done manually) +#after process replication a summmary of all procs is provided +#stop any process replica using: +#bash {TORQHOME}/torq.sh stop -csv ${KDBAPPCONFIG}/customprocess.csv +#to automatize the process summary/start/stop using torq.sh for processes in both process.csv and customprocess.csv do: +# -copy all contents of process.csv into customprocess.csv before running the script +# -configure ${TORQPROCESSES} in setenv.sh to point to customprocess.csv + +#source the relevant setenv.sh +source setenv.sh + +#input procname from cmd line +inputprocname=$1 + +#get all procnames from process.csv +PROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/process.csv | awk -F',' '{print $4}')" + +#get all procnames from customprocess.csv +CUSTOMPROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/customprocess.csv | awk -F',' '{print $4}')" + +#check for correct argument number and content: +#check number of arguments is greater than 1 +#check if number of arguments is lower than one +#check if inputed procname matches a procname in process.csv +if [ $# -gt 1 ] ; then + echo -e '\nToo many arguments.' +elif [ $# -lt 1 ] ; then + echo -e '\nProcname needed. Use: \n ' $PROCNAMES +elif [ $# -eq 1 ] ; then + if [ "$(echo $PROCNAMES | grep -w $inputprocname)" == "" ] ; then + echo -e '\nUnavailable procname. Use: \n' $PROCNAMES + else + +#get the last replica of inputed procname + var=$(bash ${TORQHOME}/torq.sh procs -csv ${KDBAPPCONFIG}/customprocess.csv | grep $inputprocname | tail -n 1) + +#work out new procname based var + if [ `echo $var | grep -P -o '\d' | wc -l` -lt 2 ] ; then + next=2 + else + next=$((${var: -1} + 1)) + fi + newprocname="$inputprocname"."$next" + +#determine port number increment based on number of lines in the process.csv +#if that port is already in use, increment by 1 till succesful + portnum="$(wc -l < ${KDBAPPCONFIG}/process.csv)" + while [ "$(netstat -an | grep "$((${KDBBASEPORT}+$portnum))" | grep -i listen)" != "" ] + do + ((portnum++)) + done + +#use the inputed procname to determine what process to replicate: +#use configured port number and process name +#add new process to customprocess.csv +#starts the replicated process automatically using customprocess.csv in the -csv flag +#run summary of all processes in customprocess.csv + grep $inputprocname ${KDBAPPCONFIG}/process.csv | awk -F',' -vOFS=',' '{ $2 = "{KDBBASEPORT}+" '"$portnum"'; $4 ="'$newprocname'" }1' >> ${KDBAPPCONFIG}/customprocess.csv + echo "$inputprocname replicated as $newprocname" + bash torq.sh start $newprocname -csv ${KDBAPPCONFIG}/customprocess.csv + bash torq.sh summary -csv ${KDBAPPCONFIG}/customprocess.csv + fi +fi From 266b583cc00deca5e80bb378bc851769552a35fd Mon Sep 17 00:00:00 2001 From: jamielamont Date: Fri, 25 Nov 2022 15:51:13 +0000 Subject: [PATCH 02/15] basic template created for new orchestrator process --- code/processes/orchestrator.q | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 code/processes/orchestrator.q diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q new file mode 100644 index 000000000..b1b124c6a --- /dev/null +++ b/code/processes/orchestrator.q @@ -0,0 +1,20 @@ +/TorQ orchestrator process + +\d .orchestrator + +/default parameters + +/table for tracking scaling +/list of scalable processes +/current number of processes by their proctype + +/function to scale up a process +scaleup:{[procname] + system "bash addproc.sh ",string procname; + /update number of processes by category + /update table with record for scaling up + } + +/function to scale down a process +scaledown:{[] + } From 34c9652c016380837a5f720c4d911fd0b331928b Mon Sep 17 00:00:00 2001 From: jamielamont Date: Mon, 28 Nov 2022 16:18:55 +0000 Subject: [PATCH 03/15] Added function to obtain the number of instances of each scalable process --- code/processes/orchestrator.q | 22 ++++++++++++++++++---- config/processlimits.csv | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 config/processlimits.csv diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q index b1b124c6a..0e933157d 100644 --- a/code/processes/orchestrator.q +++ b/code/processes/orchestrator.q @@ -1,17 +1,31 @@ -/TorQ orchestrator process +/TorQ Orchestrator Process -\d .orchestrator +\d .orch /default parameters /table for tracking scaling -/list of scalable processes -/current number of processes by their proctype + +processlimitscsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file + +upperlimits:1!("SI";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each + +scaleprocslist:exec procname from upperlimits; /list of scalable processes + +/initialises connection to discovery process and creates keyed table containing the number of instances of each scalable process +getscaleprocsinstances:{[] + .servers.startup[]; + `.orch.procs set string@/:exec procname from .servers.procstab; + `.orch.scaleprocsinstances set ([procname:scaleprocslist] instances:{sum procs like x,"*"}@/:string@/:scaleprocslist); + } + +getscaleprocsinstances[]; /function to scale up a process scaleup:{[procname] system "bash addproc.sh ",string procname; /update number of processes by category + getscaleprocsinstances[]; /update table with record for scaling up } diff --git a/config/processlimits.csv b/config/processlimits.csv new file mode 100644 index 000000000..986c200aa --- /dev/null +++ b/config/processlimits.csv @@ -0,0 +1 @@ +procname,limit From a5d5dafbdae46406c8ae7c946b6466c39c79f209 Mon Sep 17 00:00:00 2001 From: jamielamont Date: Tue, 29 Nov 2022 11:13:50 +0000 Subject: [PATCH 04/15] Updated scaleup and scaledown funcs to upsert record of scaling to scalingdetails table plus added orchestrator to list of connections in gateway settings --- code/processes/orchestrator.q | 14 +++++++++----- config/settings/gateway.q | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q index 0e933157d..4227c266b 100644 --- a/code/processes/orchestrator.q +++ b/code/processes/orchestrator.q @@ -4,12 +4,10 @@ /default parameters -/table for tracking scaling +scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); totalnumofinstances:`int$(); limit:`int$()); /table for tracking scaling processlimitscsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file - upperlimits:1!("SI";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each - scaleprocslist:exec procname from upperlimits; /list of scalable processes /initialises connection to discovery process and creates keyed table containing the number of instances of each scalable process @@ -24,11 +22,17 @@ getscaleprocsinstances[]; /function to scale up a process scaleup:{[procname] system "bash addproc.sh ",string procname; - /update number of processes by category + /update number of process instances getscaleprocsinstances[]; /update table with record for scaling up + `.orch.scalingdetails upsert (.z.p;procname;`up;.orch.scaleprocsinstances[procname;`instances];.orch.upperlimits[procname;`limit]); } /function to scale down a process -scaledown:{[] +scaledown:{[procname] + system "bash removeproc.sh ",string procname; + /update number of process instances + getscaleprocsinstances[]; + /update table with record for scaling down + `.orch.scalingdetails upsert (.z.p;procname;`down;.orch.scaleprocsinstances[procname;`instances];.orch.upperlimits[procname;`limit]); } diff --git a/config/settings/gateway.q b/config/settings/gateway.q index 820c6c9bc..1362db942 100644 --- a/config/settings/gateway.q +++ b/config/settings/gateway.q @@ -17,7 +17,7 @@ loadprocesscode:1b // whether to load the process specific code def // Server connection details \d .servers -CONNECTIONS:`rdb`hdb // list of connections to make at start up +CONNECTIONS:`rdb`hdb`orchestrator // list of connections to make at start up RETRY:0D00:01 // period on which to retry dead connections. If 0, no reconnection attempts \d .aqrest From 07a2005bb7afa8f18d59301032e7b3a825214519 Mon Sep 17 00:00:00 2001 From: mmorelli Date: Tue, 29 Nov 2022 11:39:32 +0000 Subject: [PATCH 05/15] Script for scaling down individual processes --- removeproc.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 removeproc.sh diff --git a/removeproc.sh b/removeproc.sh new file mode 100644 index 000000000..cf5943471 --- /dev/null +++ b/removeproc.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +#source the relevant setenv.sh +source setenv.sh + +#get all procnames from process.csv +PROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/process.csv | awk -F',' '{print $4}')" + +#input procname from cmd line +inputprocname=$1 + +#get all procnames from customprocess.csv +CUSTOMPROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/customprocess.csv | awk -F',' '{print $4}')" +#echo '***** LOOK HERE' $CUSTOMPROCNAMES '****' +cp ${KDBAPPCONFIG}/customprocess.csv ${KDBAPPCONFIG}/customprocessCopy.csv + +#check for correct argument number and content: +#check number of arguments is greater than 1 +#check if number of arguments is lower than one +#check if inputed procname matches a procname in process.csv + +if [ $# -gt 1 ] ; then + echo -e '\nToo many arguments.' +elif [ $# -lt 1 ] ; then + echo -e '\nProcname needed. Use: \n ' $CUSTOMPROCNAMES +elif [ $# -eq 1 ] ; then + if [ "$(echo $CUSTOMPROCNAMES | grep -w $inputprocname)" ] ; then + echo -e 'Shutting down process now' + else + echo 'Process name does not match, try again' + fi + +fi + +#remove process from customprocess.csv +#echo $CUSTOMPROCNAMES | grep -w $inputprocname | sed -i $inputprocname ${KDBAPPCONFIG}/customprocess.csv +#sed -i '/$inputprocname/d' ${KDBAPPCONFIG}/customprocess.csv +grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv From e4a642c960edcccd9fc173c3acbc321ee82ffcad Mon Sep 17 00:00:00 2001 From: jamielamont Date: Wed, 30 Nov 2022 11:49:57 +0000 Subject: [PATCH 06/15] Added checks to scaleup and scaledown funcs so instances cannot go beyond or under the configured upper and lower limits --- code/processes/orchestrator.q | 33 ++++++++++++++++++++------------- config/processlimits.csv | 2 +- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q index 4227c266b..090393622 100644 --- a/code/processes/orchestrator.q +++ b/code/processes/orchestrator.q @@ -4,11 +4,11 @@ /default parameters -scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); totalnumofinstances:`int$(); limit:`int$()); /table for tracking scaling +scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); totalnumofinstances:`int$(); lowerlimit:`int$(); upperlimit:`int$()); /table for tracking scaling processlimitscsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file -upperlimits:1!("SI";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each -scaleprocslist:exec procname from upperlimits; /list of scalable processes +limits:1!("SII";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each +scaleprocslist:exec procname from limits; /list of scalable processes /initialises connection to discovery process and creates keyed table containing the number of instances of each scalable process getscaleprocsinstances:{[] @@ -21,18 +21,25 @@ getscaleprocsinstances[]; /function to scale up a process scaleup:{[procname] - system "bash addproc.sh ",string procname; - /update number of process instances - getscaleprocsinstances[]; - /update table with record for scaling up - `.orch.scalingdetails upsert (.z.p;procname;`up;.orch.scaleprocsinstances[procname;`instances];.orch.upperlimits[procname;`limit]); + $[.orch.scaleprocsinstances[procname;`instances]<.orch.limits[procname;`upper]; + [system "bash addproc.sh ",string procname; + /update number of process instances + getscaleprocsinstances[]; + /update table with record for scaling up + `.orch.scalingdetails upsert (.z.p;procname;`up;.orch.scaleprocsinstances[procname;`instances];.orch.limits[procname;`lower];.orch.limits[procname;`upper])]; + .lg.o[`scale;"upper limit hit for ",string procname] + ]; } /function to scale down a process scaledown:{[procname] - system "bash removeproc.sh ",string procname; - /update number of process instances - getscaleprocsinstances[]; - /update table with record for scaling down - `.orch.scalingdetails upsert (.z.p;procname;`down;.orch.scaleprocsinstances[procname;`instances];.orch.upperlimits[procname;`limit]); + $[.orch.scaleprocsinstances[procname;`instances]>.orch.limits[procname;`lower]; + [latestinstance:last .orch.procs where .orch.procs like string[procname],"*"; + system "bash removeproc.sh ",latestinstance; + /update number of process instances + getscaleprocsinstances[]; + /update table with record for scaling down + `.orch.scalingdetails upsert (.z.p;procname;`down;.orch.scaleprocsinstances[procname;`instances];.orch.limits[procname;`lower];.orch.limits[procname;`upper])]; + .lg.o[`scale;"lower limit hit for ",string procname] + ]; } diff --git a/config/processlimits.csv b/config/processlimits.csv index 986c200aa..9655479f3 100644 --- a/config/processlimits.csv +++ b/config/processlimits.csv @@ -1 +1 @@ -procname,limit +procname,lower,upper From abb461f304af9c735ce0b148362472bfd0e56b51 Mon Sep 17 00:00:00 2001 From: mmorelli Date: Wed, 30 Nov 2022 12:39:31 +0000 Subject: [PATCH 07/15] Adjusting order of things slightly, and add in line to ensure port is freed up after proc removal --- removeproc.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/removeproc.sh b/removeproc.sh index cf5943471..d8b6042aa 100644 --- a/removeproc.sh +++ b/removeproc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash #source the relevant setenv.sh source setenv.sh @@ -26,13 +26,17 @@ elif [ $# -lt 1 ] ; then elif [ $# -eq 1 ] ; then if [ "$(echo $CUSTOMPROCNAMES | grep -w $inputprocname)" ] ; then echo -e 'Shutting down process now' - else - echo 'Process name does not match, try again' - fi -fi + bash torq.sh stop $inputprocname -csv ${KDBAPPCONFIG}/customprocessCopy.csv + grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv + #bash ./torq.sh stop $inputprocname + else + echo 'Process name does not match, try again' + fi + +fi #remove process from customprocess.csv #echo $CUSTOMPROCNAMES | grep -w $inputprocname | sed -i $inputprocname ${KDBAPPCONFIG}/customprocess.csv #sed -i '/$inputprocname/d' ${KDBAPPCONFIG}/customprocess.csv -grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv +#grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv From d6bd8c6267be82f92433d3ac2ad2f07a49547235 Mon Sep 17 00:00:00 2001 From: jamielamont Date: Fri, 2 Dec 2022 15:15:45 +0000 Subject: [PATCH 08/15] Added function to scale processes to lower limit upon startup plus unit tests for scaleup and scaledown funcs --- addproc.sh | 7 ++----- code/processes/orchestrator.q | 27 +++++++++++++++++++-------- removeproc.sh | 5 +---- tests/orchestrator/customprocess.csv | 4 ++++ tests/orchestrator/process.csv | 4 ++++ tests/orchestrator/processlimits.csv | 2 ++ tests/orchestrator/run.sh | 22 ++++++++++++++++++++++ tests/orchestrator/test.csv | 13 +++++++++++++ 8 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 tests/orchestrator/customprocess.csv create mode 100644 tests/orchestrator/process.csv create mode 100644 tests/orchestrator/processlimits.csv create mode 100644 tests/orchestrator/run.sh create mode 100644 tests/orchestrator/test.csv diff --git a/addproc.sh b/addproc.sh index f33182ee2..c14142619 100644 --- a/addproc.sh +++ b/addproc.sh @@ -12,9 +12,6 @@ # -copy all contents of process.csv into customprocess.csv before running the script # -configure ${TORQPROCESSES} in setenv.sh to point to customprocess.csv -#source the relevant setenv.sh -source setenv.sh - #input procname from cmd line inputprocname=$1 @@ -63,7 +60,7 @@ elif [ $# -eq 1 ] ; then #run summary of all processes in customprocess.csv grep $inputprocname ${KDBAPPCONFIG}/process.csv | awk -F',' -vOFS=',' '{ $2 = "{KDBBASEPORT}+" '"$portnum"'; $4 ="'$newprocname'" }1' >> ${KDBAPPCONFIG}/customprocess.csv echo "$inputprocname replicated as $newprocname" - bash torq.sh start $newprocname -csv ${KDBAPPCONFIG}/customprocess.csv - bash torq.sh summary -csv ${KDBAPPCONFIG}/customprocess.csv + bash ${TORQHOME}/torq.sh start $newprocname -csv ${KDBAPPCONFIG}/customprocess.csv + bash ${TORQHOME}/torq.sh summary -csv ${KDBAPPCONFIG}/customprocess.csv fi fi diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q index 090393622..7b26fe522 100644 --- a/code/processes/orchestrator.q +++ b/code/processes/orchestrator.q @@ -4,7 +4,7 @@ /default parameters -scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); totalnumofinstances:`int$(); lowerlimit:`int$(); upperlimit:`int$()); /table for tracking scaling +scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); instancecreated:`$(); instanceremoved:`$(); totalnumofinstances:`int$(); lowerlimit:`int$(); upperlimit:`int$()); /table for tracking scaling processlimitscsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file limits:1!("SII";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each @@ -21,25 +21,36 @@ getscaleprocsinstances[]; /function to scale up a process scaleup:{[procname] - $[.orch.scaleprocsinstances[procname;`instances]<.orch.limits[procname;`upper]; - [system "bash addproc.sh ",string procname; + $[scaleprocsinstances[procname;`instances].orch.limits[procname;`lower]; - [latestinstance:last .orch.procs where .orch.procs like string[procname],"*"; - system "bash removeproc.sh ",latestinstance; + $[scaleprocsinstances[procname;`instances]>limits[procname;`lower]; + [latestinstance:last procs where procs like string[procname],"*"; + system "bash ${TORQHOME}/removeproc.sh ",latestinstance; /update number of process instances getscaleprocsinstances[]; /update table with record for scaling down - `.orch.scalingdetails upsert (.z.p;procname;`down;.orch.scaleprocsinstances[procname;`instances];.orch.limits[procname;`lower];.orch.limits[procname;`upper])]; + `.orch.scalingdetails upsert (.z.p;procname;`down;`;`$latestinstance;scaleprocsinstances[procname;`instances];limits[procname;`lower];limits[procname;`upper])]; .lg.o[`scale;"lower limit hit for ",string procname] ]; } + + +/function to ensure all processes have been scaled up to meet lower limit +initialscaling:{[procname] + if[scaleprocsinstances[procname;`instances]| ${KDBAPPCONFIG}/customprocess.csv #bash ./torq.sh stop $inputprocname else diff --git a/tests/orchestrator/customprocess.csv b/tests/orchestrator/customprocess.csv new file mode 100644 index 000000000..4ba6a21a0 --- /dev/null +++ b/tests/orchestrator/customprocess.csv @@ -0,0 +1,4 @@ +host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd +localhost,{KDBBASEPORT}+1,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1, +localhost,{KDBBASEPORT},segmentedtickerplant,stp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG}, +localhost,{KDBBASEPORT}+3,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,, diff --git a/tests/orchestrator/process.csv b/tests/orchestrator/process.csv new file mode 100644 index 000000000..4ba6a21a0 --- /dev/null +++ b/tests/orchestrator/process.csv @@ -0,0 +1,4 @@ +host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd +localhost,{KDBBASEPORT}+1,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1, +localhost,{KDBBASEPORT},segmentedtickerplant,stp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG}, +localhost,{KDBBASEPORT}+3,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,, diff --git a/tests/orchestrator/processlimits.csv b/tests/orchestrator/processlimits.csv new file mode 100644 index 000000000..5282fef3f --- /dev/null +++ b/tests/orchestrator/processlimits.csv @@ -0,0 +1,2 @@ +procname,lower,upper +hdb1,1,2 diff --git a/tests/orchestrator/run.sh b/tests/orchestrator/run.sh new file mode 100644 index 000000000..8a89b55c2 --- /dev/null +++ b/tests/orchestrator/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +#path to test directory +testpath=${KDBTESTS}/orchestrator + +OLDKDBAPPCONFIG=${KDBAPPCONFIG} +export KDBAPPCONFIG=${testpath} + +#start procs +${TORQHOME}/torq.sh start all -csv ${testpath}/customprocess.csv + +#Start test proc +/usr/bin/rlwrap $QCMD ${TORQHOME}/torq.q \ + -proctype orchestrator -procname orchestrator1 \ + -test ${testpath} \ + -load ${TORQHOME}/code/processes/orchestrator.q \ + -procfile ${testpath}/customprocess.csv -debug + +#Shut down procs +${TORQHOME}/torq.sh stop all -csv ${testpath}/customprocess.csv + +export KDBAPPCONFIG=${OLDKDBAPPCONFIG} diff --git a/tests/orchestrator/test.csv b/tests/orchestrator/test.csv new file mode 100644 index 000000000..6ffee4dad --- /dev/null +++ b/tests/orchestrator/test.csv @@ -0,0 +1,13 @@ +action,ms,bytes,lang,code,repeat,minver,comment + +/test for scaling up func +before,0,0,q,instancesbefore:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" +run,0,0,q,.orch.scaleup[`hdb1],1,,"scale up hdb1" +run,0,0,q,instancesafter:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" +true,0,0,q,instancesbeforeinstancesafter,1,,"check instances now smaller for hdb1" From bade3070e708f4db234dbef552bf768e22d15743 Mon Sep 17 00:00:00 2001 From: mmorelli Date: Fri, 2 Dec 2022 16:44:19 +0000 Subject: [PATCH 09/15] Working test.csv ready for review --- tests/orchestrator/test.csv | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/orchestrator/test.csv b/tests/orchestrator/test.csv index 6ffee4dad..160be7cc2 100644 --- a/tests/orchestrator/test.csv +++ b/tests/orchestrator/test.csv @@ -1,5 +1,7 @@ action,ms,bytes,lang,code,repeat,minver,comment +run,0,0,q,\c 300 300,1,,"" + /test for scaling up func before,0,0,q,instancesbefore:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" run,0,0,q,.orch.scaleup[`hdb1],1,,"scale up hdb1" @@ -11,3 +13,16 @@ run,0,0,q,instancesbefore:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get cu run,0,0,q,.orch.scaledown[`hdb1],1,,"scale down" run,0,0,q,instancesafter:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" true,0,0,q,instancesbefore>instancesafter,1,,"check instances now smaller for hdb1" + +/testing limits: upper then lower +run,0,0,q,if[.orch.scaleprocsinstances[`hdb1;`instances]<.orch.limits[`hdb1;`upper];.orch.scaleup[`hdb1];0],1,,"if no. instances is lower than limit, keep scaling up" +run,0,0,q,before:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current insatnces of hdb1" +run,0,0,q,.orch.scaleup[`hdb1],1,,"scaling up once more after limit is hit, to show no. instances doesn't chnage once limit hit" +run,0,0,q,after:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" +true,0,0,q,before=after,1,,"checking no. instances hasn't changed, so limit prevents further scaling up" + +run,0,0,q,if[.orch.scaleprocsinstances[`hdb1;`instances]>.orch.limits[`hdb1;`lower];.orch.scaledown[`hdb1];0],1,,"if no. instances is higher than limit, keep scaling down" +run,0,0,q,before:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current insatnces of hdb1" +run,0,0,q,.orch.scaledown[`hdb1],1,,"scale down once more to show no.instances stays same, indicating can't scale lower once limit hit" +run,0,0,q,after:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" +true,0,0,q,before=after,1,,"checking no. instances hasn't changed, so limit prevents further scaling down" From e4baca47eee908f6e08f0508e017d918b039bef8 Mon Sep 17 00:00:00 2001 From: jamielamont Date: Mon, 5 Dec 2022 09:41:51 +0000 Subject: [PATCH 10/15] Functions added to gateway to call orchestator to scale up/down --- code/processes/gateway.q | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/code/processes/gateway.q b/code/processes/gateway.q index e0fbbd566..c90aa8323 100644 --- a/code/processes/gateway.q +++ b/code/processes/gateway.q @@ -535,6 +535,18 @@ addserversfromconnectiontable[.servers.CONNECTIONS] // Join active .gw.servers to .servers.SERVERS table activeservers:{lj[select from .gw.servers where active;`handle xcol `w xkey .servers.SERVERS]} +/function to tell orchestrator to scale up +scaleup:{[procname] +handle:first exec w from .servers.SERVERS where proctype=`orchestrator; +neg[handle](`.orch.scaleup;procname); +} + +/function to tell orchestrator to scale down +scaledown:{[procname] +handle:first exec w from .servers.SERVERS where proctype=`orchestrator; +neg[handle](`.orch.scaledown;procname); +} + \d . // functions called by end-of-day processes From 512b9c5ad0bab84ea1e145aca0398b5c215ffd34 Mon Sep 17 00:00:00 2001 From: jamielamont Date: Tue, 6 Dec 2022 14:54:09 +0000 Subject: [PATCH 11/15] refactored code based on review --- code/processes/gateway.q | 18 +++---- code/processes/orchestrator.q | 55 +++++++++----------- scale.sh | 75 ++++++++++++++++++++++++++++ tests/orchestrator/customprocess.csv | 2 +- tests/orchestrator/processlimits.csv | 2 +- tests/orchestrator/test.csv | 15 +++--- 6 files changed, 115 insertions(+), 52 deletions(-) create mode 100644 scale.sh diff --git a/code/processes/gateway.q b/code/processes/gateway.q index c90aa8323..931941c28 100644 --- a/code/processes/gateway.q +++ b/code/processes/gateway.q @@ -396,7 +396,7 @@ asyncexecjpts:{[query;servertype;joinfunction;postback;timeout;sync] :()]; addquerytimeout[query;servertype;queryattributes;joinfunction;postback;timeout;sync]; - runnextquery[]; + /runnextquery[]; }; asyncexecjpt:asyncexecjpts[;;;;;0b] @@ -535,17 +535,11 @@ addserversfromconnectiontable[.servers.CONNECTIONS] // Join active .gw.servers to .servers.SERVERS table activeservers:{lj[select from .gw.servers where active;`handle xcol `w xkey .servers.SERVERS]} -/function to tell orchestrator to scale up -scaleup:{[procname] -handle:first exec w from .servers.SERVERS where proctype=`orchestrator; -neg[handle](`.orch.scaleup;procname); -} - -/function to tell orchestrator to scale down -scaledown:{[procname] -handle:first exec w from .servers.SERVERS where proctype=`orchestrator; -neg[handle](`.orch.scaledown;procname); -} +/function to tell orchestrator to scale up or down +scale:{[procname;dir] + handle:.servers.gethandlebytype[`orchestrator;`any]; + neg[handle](`.orch.scale;procname;dir); + } \d . diff --git a/code/processes/orchestrator.q b/code/processes/orchestrator.q index 7b26fe522..115abb412 100644 --- a/code/processes/orchestrator.q +++ b/code/processes/orchestrator.q @@ -6,51 +6,42 @@ scalingdetails:([] time:`timestamp$(); procname:`$(); dir:`$(); instancecreated:`$(); instanceremoved:`$(); totalnumofinstances:`int$(); lowerlimit:`int$(); upperlimit:`int$()); /table for tracking scaling -processlimitscsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file -limits:1!("SII";enlist ",")0:processlimitscsv; /table of scalable processes and the max number of instances allowed for each +inputcsv:hsym first .proc.getconfigfile"processlimits.csv"; /location of csv file +limits:@[{.lg.o[`scale;"Opening ", x];`procname xkey ("SII"; enlist ",") 0:`$x}; (string inputcsv); {.lg.e[`scale;"failed to open ", (x)," : ",y];'y}[string inputcsv]]; /table of scalable processes and the max number of instances allowed for each + scaleprocslist:exec procname from limits; /list of scalable processes /initialises connection to discovery process and creates keyed table containing the number of instances of each scalable process getscaleprocsinstances:{[] .servers.startup[]; - `.orch.procs set string@/:exec procname from .servers.procstab; - `.orch.scaleprocsinstances set ([procname:scaleprocslist] instances:{sum procs like x,"*"}@/:string@/:scaleprocslist); + `.orch.procs set exec procname from .servers.procstab where proctype=`hdb; + scaleprocsinstances:count each group first each ` vs/:procs; + `.orch.scaleprocsinstances set ([procname:key scaleprocsinstances] instances:value scaleprocsinstances); } getscaleprocsinstances[]; -/function to scale up a process -scaleup:{[procname] - $[scaleprocsinstances[procname;`instances]=y};limit:`upper;parentproc:procname]; + + if[dir=`down;op:"-d ";limitcheck:{x<=y};limit:`lower;parentproc:first ` vs procname]; -/function to scale down a process -scaledown:{[procname] - $[scaleprocsinstances[procname;`instances]>limits[procname;`lower]; - [latestinstance:last procs where procs like string[procname],"*"; - system "bash ${TORQHOME}/removeproc.sh ",latestinstance; - /update number of process instances - getscaleprocsinstances[]; - /update table with record for scaling down - `.orch.scalingdetails upsert (.z.p;procname;`down;`;`$latestinstance;scaleprocsinstances[procname;`instances];limits[procname;`lower];limits[procname;`upper])]; - .lg.o[`scale;"lower limit hit for ",string procname] - ]; - } + if[limitcheck[scaleprocsinstances[parentproc;`instances];limits[parentproc;limit]]; .lg.o[`scale;string[limit]," limit hit for ",string parentproc]; :()]; + system "bash ${TORQHOME}/scale.sh ",op,string procname; + /update number of process instances + getscaleprocsinstances[]; + /update table with record for scaling + `.orch.scalingdetails upsert (.z.p;parentproc;dir;$[dir=`up;` sv parentproc,`$string .orch.scaleprocsinstances[parentproc;`instances];`];$[dir=`down;procname;`];scaleprocsinstances[parentproc;`instances];limits[parentproc;`lower];limits[parentproc;`upper]); + } /function to ensure all processes have been scaled up to meet lower limit initialscaling:{[procname] - if[scaleprocsinstances[procname;`instances]> ${KDBAPPCONFIG}/customprocess.csv + echo "$inputprocname replicated as $newprocname" + bash ${TORQHOME}/torq.sh start $newprocname -csv ${KDBAPPCONFIG}/customprocess.csv + bash ${TORQHOME}/torq.sh summary -csv ${KDBAPPCONFIG}/customprocess.csv + fi + ;; + d) + if [ "$(echo $CUSTOMPROCNAMES | grep -w $inputprocname)" ] ; then + echo -e 'Shutting down process now' + bash ${TORQHOME}/torq.sh stop $inputprocname -csv ${KDBAPPCONFIG}/customprocess.csv + cp ${KDBAPPCONFIG}/customprocess.csv ${KDBAPPCONFIG}/customprocessCopy.csv + grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv + else + echo 'Process name does not match, try again' + fi + ;; + *) + usage + ;; + esac + done +else + usage +fi diff --git a/tests/orchestrator/customprocess.csv b/tests/orchestrator/customprocess.csv index 4ba6a21a0..74ea50f33 100644 --- a/tests/orchestrator/customprocess.csv +++ b/tests/orchestrator/customprocess.csv @@ -1,4 +1,4 @@ host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd localhost,{KDBBASEPORT}+1,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1, localhost,{KDBBASEPORT},segmentedtickerplant,stp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG}, -localhost,{KDBBASEPORT}+3,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,, +localhost,{KDBBASEPORT}+2,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,, diff --git a/tests/orchestrator/processlimits.csv b/tests/orchestrator/processlimits.csv index 5282fef3f..be16bea67 100644 --- a/tests/orchestrator/processlimits.csv +++ b/tests/orchestrator/processlimits.csv @@ -1,2 +1,2 @@ procname,lower,upper -hdb1,1,2 +hdb1,2,3 diff --git a/tests/orchestrator/test.csv b/tests/orchestrator/test.csv index 160be7cc2..cdc270c5f 100644 --- a/tests/orchestrator/test.csv +++ b/tests/orchestrator/test.csv @@ -2,27 +2,30 @@ action,ms,bytes,lang,code,repeat,minver,comment run,0,0,q,\c 300 300,1,,"" +/test initial scaling +true,0,0,q,.orch.scaleprocsinstances[`hdb1;`instances]=.orch.limits[`hdb1;`lower],1,,"check hdb1 scaled to lower limit upon startup" + /test for scaling up func before,0,0,q,instancesbefore:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" -run,0,0,q,.orch.scaleup[`hdb1],1,,"scale up hdb1" +run,0,0,q,.orch.scale[`hdb1;`up],1,,"scale up hdb1" run,0,0,q,instancesafter:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" true,0,0,q,instancesbeforeinstancesafter,1,,"check instances now smaller for hdb1" /testing limits: upper then lower -run,0,0,q,if[.orch.scaleprocsinstances[`hdb1;`instances]<.orch.limits[`hdb1;`upper];.orch.scaleup[`hdb1];0],1,,"if no. instances is lower than limit, keep scaling up" +run,0,0,q,.orch.scale[`hdb1;`up],1,,"scale up" run,0,0,q,before:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current insatnces of hdb1" -run,0,0,q,.orch.scaleup[`hdb1],1,,"scaling up once more after limit is hit, to show no. instances doesn't chnage once limit hit" +run,0,0,q,.orch.scale[`hdb1;`up],1,,"scaling up once more after limit is hit, to show no. instances doesn't chnage once limit hit" run,0,0,q,after:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" true,0,0,q,before=after,1,,"checking no. instances hasn't changed, so limit prevents further scaling up" -run,0,0,q,if[.orch.scaleprocsinstances[`hdb1;`instances]>.orch.limits[`hdb1;`lower];.orch.scaledown[`hdb1];0],1,,"if no. instances is higher than limit, keep scaling down" +run,0,0,q,.orch.scale[`hdb1.3;`down],1,,"scale down" run,0,0,q,before:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current insatnces of hdb1" -run,0,0,q,.orch.scaledown[`hdb1],1,,"scale down once more to show no.instances stays same, indicating can't scale lower once limit hit" +run,0,0,q,.orch.scale[`hdb1.2;`down],1,,"scale down once more to show no.instances stays same, indicating can't scale lower once limit hit" run,0,0,q,after:.orch.scaleprocsinstances[`hdb1;`instances],1,,"get current instances of hdb1" true,0,0,q,before=after,1,,"checking no. instances hasn't changed, so limit prevents further scaling down" From 3978e97d6f5565c282f1c7f368ebe3ed563205f4 Mon Sep 17 00:00:00 2001 From: jamielamont Date: Tue, 6 Dec 2022 14:56:23 +0000 Subject: [PATCH 12/15] remove comments from debugging --- code/processes/gateway.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/processes/gateway.q b/code/processes/gateway.q index 931941c28..bc1a1ef83 100644 --- a/code/processes/gateway.q +++ b/code/processes/gateway.q @@ -396,7 +396,7 @@ asyncexecjpts:{[query;servertype;joinfunction;postback;timeout;sync] :()]; addquerytimeout[query;servertype;queryattributes;joinfunction;postback;timeout;sync]; - /runnextquery[]; + runnextquery[]; }; asyncexecjpt:asyncexecjpts[;;;;;0b] From 5512451cfc41ce6e6a9f7bee0418fd39fa0ace5a Mon Sep 17 00:00:00 2001 From: jamielamont <110402933+jamielamont@users.noreply.github.com> Date: Tue, 6 Dec 2022 14:57:37 +0000 Subject: [PATCH 13/15] Delete addproc.sh --- addproc.sh | 66 ------------------------------------------------------ 1 file changed, 66 deletions(-) delete mode 100644 addproc.sh diff --git a/addproc.sh b/addproc.sh deleted file mode 100644 index c14142619..000000000 --- a/addproc.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -#before running the script ensure the stack is running -#this script takes an existing procname as an argument to determine what process to replicate -#procname is configured automatically based on the last process replica name -#all new processes are started automatically to avoid port number overlap upon running the script multiple times without starting the new procs first -#new processes are written to customprocess.csv which also contains the original processes from process.csv (this may have to be done manually) -#after process replication a summmary of all procs is provided -#stop any process replica using: -#bash {TORQHOME}/torq.sh stop -csv ${KDBAPPCONFIG}/customprocess.csv -#to automatize the process summary/start/stop using torq.sh for processes in both process.csv and customprocess.csv do: -# -copy all contents of process.csv into customprocess.csv before running the script -# -configure ${TORQPROCESSES} in setenv.sh to point to customprocess.csv - -#input procname from cmd line -inputprocname=$1 - -#get all procnames from process.csv -PROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/process.csv | awk -F',' '{print $4}')" - -#get all procnames from customprocess.csv -CUSTOMPROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/customprocess.csv | awk -F',' '{print $4}')" - -#check for correct argument number and content: -#check number of arguments is greater than 1 -#check if number of arguments is lower than one -#check if inputed procname matches a procname in process.csv -if [ $# -gt 1 ] ; then - echo -e '\nToo many arguments.' -elif [ $# -lt 1 ] ; then - echo -e '\nProcname needed. Use: \n ' $PROCNAMES -elif [ $# -eq 1 ] ; then - if [ "$(echo $PROCNAMES | grep -w $inputprocname)" == "" ] ; then - echo -e '\nUnavailable procname. Use: \n' $PROCNAMES - else - -#get the last replica of inputed procname - var=$(bash ${TORQHOME}/torq.sh procs -csv ${KDBAPPCONFIG}/customprocess.csv | grep $inputprocname | tail -n 1) - -#work out new procname based var - if [ `echo $var | grep -P -o '\d' | wc -l` -lt 2 ] ; then - next=2 - else - next=$((${var: -1} + 1)) - fi - newprocname="$inputprocname"."$next" - -#determine port number increment based on number of lines in the process.csv -#if that port is already in use, increment by 1 till succesful - portnum="$(wc -l < ${KDBAPPCONFIG}/process.csv)" - while [ "$(netstat -an | grep "$((${KDBBASEPORT}+$portnum))" | grep -i listen)" != "" ] - do - ((portnum++)) - done - -#use the inputed procname to determine what process to replicate: -#use configured port number and process name -#add new process to customprocess.csv -#starts the replicated process automatically using customprocess.csv in the -csv flag -#run summary of all processes in customprocess.csv - grep $inputprocname ${KDBAPPCONFIG}/process.csv | awk -F',' -vOFS=',' '{ $2 = "{KDBBASEPORT}+" '"$portnum"'; $4 ="'$newprocname'" }1' >> ${KDBAPPCONFIG}/customprocess.csv - echo "$inputprocname replicated as $newprocname" - bash ${TORQHOME}/torq.sh start $newprocname -csv ${KDBAPPCONFIG}/customprocess.csv - bash ${TORQHOME}/torq.sh summary -csv ${KDBAPPCONFIG}/customprocess.csv - fi -fi From 59848f4d47edaf920120703302f8665c4402b9ba Mon Sep 17 00:00:00 2001 From: jamielamont <110402933+jamielamont@users.noreply.github.com> Date: Tue, 6 Dec 2022 14:57:57 +0000 Subject: [PATCH 14/15] Delete removeproc.sh --- removeproc.sh | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 removeproc.sh diff --git a/removeproc.sh b/removeproc.sh deleted file mode 100644 index a4d9ec6e2..000000000 --- a/removeproc.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -#get all procnames from process.csv -PROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/process.csv | awk -F',' '{print $4}')" - -#input procname from cmd line -inputprocname=$1 - -#get all procnames from customprocess.csv -CUSTOMPROCNAMES="$(sed '1d' ${KDBAPPCONFIG}/customprocess.csv | awk -F',' '{print $4}')" -#echo '***** LOOK HERE' $CUSTOMPROCNAMES '****' -cp ${KDBAPPCONFIG}/customprocess.csv ${KDBAPPCONFIG}/customprocessCopy.csv - -#check for correct argument number and content: -#check number of arguments is greater than 1 -#check if number of arguments is lower than one -#check if inputed procname matches a procname in process.csv - -if [ $# -gt 1 ] ; then - echo -e '\nToo many arguments.' -elif [ $# -lt 1 ] ; then - echo -e '\nProcname needed. Use: \n ' $CUSTOMPROCNAMES -elif [ $# -eq 1 ] ; then - if [ "$(echo $CUSTOMPROCNAMES | grep -w $inputprocname)" ] ; then - echo -e 'Shutting down process now' - - bash ${TORQHOME}/torq.sh stop $inputprocname -csv ${KDBAPPCONFIG}/customprocessCopy.csv - grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv - #bash ./torq.sh stop $inputprocname - else - echo 'Process name does not match, try again' - fi - -fi - -#remove process from customprocess.csv -#echo $CUSTOMPROCNAMES | grep -w $inputprocname | sed -i $inputprocname ${KDBAPPCONFIG}/customprocess.csv -#sed -i '/$inputprocname/d' ${KDBAPPCONFIG}/customprocess.csv -#grep -v "$inputprocname" ${KDBAPPCONFIG}/customprocessCopy.csv >| ${KDBAPPCONFIG}/customprocess.csv From bf7ce3f883be339346a12700eae18b5d2fd3eae1 Mon Sep 17 00:00:00 2001 From: jamielamont <110402933+jamielamont@users.noreply.github.com> Date: Tue, 6 Dec 2022 14:58:56 +0000 Subject: [PATCH 15/15] Update process.csv --- tests/orchestrator/process.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/orchestrator/process.csv b/tests/orchestrator/process.csv index 4ba6a21a0..74ea50f33 100644 --- a/tests/orchestrator/process.csv +++ b/tests/orchestrator/process.csv @@ -1,4 +1,4 @@ host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd localhost,{KDBBASEPORT}+1,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1, localhost,{KDBBASEPORT},segmentedtickerplant,stp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG}, -localhost,{KDBBASEPORT}+3,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,, +localhost,{KDBBASEPORT}+2,hdb,hdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,,