Skip to content

Commit d844917

Browse files
ConorSwainDIConorSwainDI
authored
Fix partbyenum to enumerate against placeholding symbol (#691)
Fix partbyenum logic to enumerate against TORQNULLSYMBOL symbol in the event of a null partition name
1 parent 5ede681 commit d844917

File tree

8 files changed

+105
-4
lines changed

8 files changed

+105
-4
lines changed

‎code/processes/idb.q‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,5 +107,5 @@ reload:.idb.intradayreload;
107107

108108
/- helper function to support queries against the sym column
109109
maptoint:{[symbol]
110-
sym?symbol
110+
sym?`TORQNULLSYMBOL^symbol
111111
};

‎code/processes/wdb.q‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,16 +114,20 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]}
114114
tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes;
115115
(sortedlist union tables[`.]) except ignorelist}
116116

117+
/- function that ensures a list of syms is returned no matter what is passed to it
118+
ensuresymlist:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)}
119+
117120
/- function to upsert to specified directory
118121
upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode]
119122
/- enumerate current extra partition against the hdb sym file
120-
if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)?first expt;];
123+
/- if extra partition is null, send to a partition enumerated against `TORQNULLSYMBOL symbol
124+
if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ ensuresymlist[expt]]];
121125
/- create directory location for selected partiton
122126
/- replace non-alphanumeric characters in symbols with _
123-
/- convert to symbols and replace any null values with `NONE
127+
/- convert to symbols and replace any null values with `TORQNULLSYMBOL
124128
directory:$[writedownmode~`partbyenum;
125129
` sv .Q.par[dir;pt;`$string i],tablename,`;
126-
` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`];
130+
` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`];
127131
.lg.o[`save;"saving ",(string tablename)," data to partition ",string directory];
128132
/- selecting rows of table with matching partition
129133
r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()];

‎tests/wdb/nullpartbyenum/config/sort.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
tabname,att,column,sort
2+
quote,p,sym,1
3+
trade,p,sym,1
4+
default,p,sym,1

‎tests/wdb/nullpartbyenum/database.q‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
quote:([]time:`timestamp$(); sym:`g#`symbol$(); bid:`float$(); ask:`float$(); bsize:`long$(); asize:`long$(); mode:`char$(); ex:`char$(); src:`symbol$())
2+
trade:([]time:`timestamp$(); sym:`g#`symbol$(); price:`float$(); size:`int$(); stop:`boolean$(); cond:`char$(); ex:`char$();side:`symbol$())

‎tests/wdb/nullpartbyenum/nullpartbyenum.csv‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
action,ms,bytes,lang,code,repeat,minver,comment
2+
beforeany,0,0,q,system "sleep 5",1,,"Wait for processes to start"
3+
before,0,0,q,.servers.startup[],1,,"Start connection management"
4+
before,0,0,q,system "sleep 2",1,,"Wait for connections"
5+
before,0,0,q,stpHandle:gethandle[`stp1],1,,"Open handle to STP"
6+
before,0,0,q,wdbEnumHandle:gethandle[`wdbenum],1,,"Open handle to WDB enum"
7+
before,0,0,q,idbHandle:gethandle[`idbenum],1,,"Open handle to IDB"
8+
before,0,0,q,hdbEnumHandle:gethandle[`hdbenum],1,,"Open handle to partbyenum HDB"
9+
before,0,0,q,"wdbEnumSym:` sv (wdbEnumHandle(`.wdb.hdbsettings;`hdbdir)),`sym",1,,"Get location of partbyenum hdb sym file"
10+
before,0,0,q,wdbEnumHandle(set;`.wdb.numtab;`quote`trade!12 15),1,,"Set low treshold numbers for merging on partbyenum WDB"
11+
12+
before,0,0,q,stpHandle(`.stplg.init;testlogdb),1,,"Create test stplog test directory"
13+
before,0,0,q,logdir:1_string stpHandle(`.stplg.dldir),1,,"Get location of log directory"
14+
before,0,0,q,.proc.sys "sleep 3",1,,"Wait for connections to get set up"
15+
before,0,0,q,.os.md temphdbpartbyenumdir,1,,"Creating HDB directory for partbyenum DB"
16+
17+
run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP"
18+
run,0,0,q,.proc.sys "sleep 2",1,,"Wait for updates to publish"
19+
true,0,0,q,count[first testtrade]~wdbEnumHandle "count trade",1,,"Check trade update was correctly published to partbyenum WDB"
20+
true,0,0,q,count[first testquote]~wdbEnumHandle "count quote",1,,"Check quote update was correctly published to partbyenum WDB"
21+
22+
run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP to force a writedown"
23+
run,0,0,q,.proc.sys "sleep 3",1,,"Wait for updates to publish and writedown to happen"
24+
true,0,0,q,"all (distinct `TORQNULLSYMBOL^(testtrade[0]),(testquote[0])) in get wdbEnumSym",1,,"Check if all symbols are enumerated against partbyenum hdb sym, fill null sym in test data with TORQNULLSYMBOL to confirm this has been enumerated against"
25+
26+
true,0,0,q,all {any x like/: y}[;folder_patterns] exec ptdir from wdbEnumHandle `.merge.partsizes,1,,"Check if all partitions are placed in the correct enumerated tables in the partbyenum WDB dir"
27+
28+
true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from quote where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on quote"
29+
true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from trade where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on trade"
30+
31+
run,0,0,q,wdbEnumHandle(`.u.end;`.wdb.currentpartition),1,,"Trigger EOD on partbyenum WDB"
32+
run,0,0,q,.proc.sys "sleep 5",1,,"Wait for merge to HDB to happen"
33+
true,0,0,q,(2*count[first testtrade])~hdbEnumHandle "count select from trade",1,,"Check trade in hdb"
34+
true,0,0,q,(2*count[first testquote])~hdbEnumHandle "count select from quote",1,,"Check quote in hdb"
35+
36+
after,0,0,q,hclose each (wdbEnumHandle;stpHandle;idbHandle;hdbEnumHandle);,1,,"Close handles"
37+
after,0,0,q,kill9proc each ("stp1";"wdbenum";"hdbenum";"idbenum";"sort1"),1,,"Stop all procs"
38+
after,0,0,q,.os.deldir logdir,1,,"Delete test segmented tickerplant logs"
39+
after,0,0,q,.os.deldir 1_string temphdbpartbyenumdir,1,,"Delete temp WDB"
40+
after,0,0,q,.os.deldir 1_string wdbpartbyenumdir,1,,"Delete temp WDB"

‎tests/wdb/nullpartbyenum/process.csv‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd
2+
localhost,{KDBBASEPORT}+100,discovery,discovery1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q
3+
localhost,{KDBBASEPORT}+101,wdb,wdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q -.wdb.writedownmode partbyenum -.wdb.mode saveandsort -.wdb.sortcsv ${KDBTESTS}/wdb/nullpartbyenum/config/sort.csv -.wdb.savedir ${KDBTESTS}/wdb/nullpartbyenum/tempwdbpartbyenum/ -.wdb.hdbdir ${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/ -.wdb.settimer 0D00:00:02,1,,q
4+
localhost,{KDBBASEPORT}+103,hdb,hdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/,1,,q
5+
localhost,{KDBBASEPORT}+105,segmentedtickerplant,stp1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQHOME}/database.q,q
6+
localhost,{KDBBASEPORT}+106,idb,idbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/idb.q,1,,q
7+
localhost,{KDBBASEPORT}+108,sort,sort1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q ${KDBTESTS}/merge/wdb/config/wdbsettings.q,1,-s -2,

‎tests/wdb/nullpartbyenum/run.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
# Handle command-line arguments
4+
source $KDBTESTS/flagparse.sh
5+
6+
# Path to test directory
7+
testpath=${KDBTESTS}/wdb/nullpartbyenum
8+
9+
# Start procs
10+
${TORQHOME}/torq.sh start discovery1 stp1 wdbenum hdbenum idbenum sort1 -csv ${testpath}/process.csv
11+
12+
# Start test proc
13+
/usr/bin/rlwrap q ${TORQHOME}/torq.q \
14+
-proctype test -procname test1 \
15+
-test ${testpath} \
16+
-load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q \
17+
-testresults ${KDBTESTS}/stp/results/ \
18+
-runtime $run \
19+
-procfile ${testpath}/process.csv \
20+
$debug $stop $write $quiet
21+
22+
# Shut down procs
23+
${TORQHOME}/torq.sh stop discovery1 -csv ${testpath}/process.csv

‎tests/wdb/nullpartbyenum/settings.q‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// IPC connection parameters
2+
.servers.CONNECTIONS:`wdb`segmentedtickerplant`tickerplant`hdb`idb`sort;
3+
.servers.USERPASS:`admin:admin;
4+
5+
// Paths to process CSV and test STP log directory
6+
processcsv:getenv[`KDBTESTS],"/wdb/nullpartbyenum/process.csv";
7+
wdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/tempwdbpartbyenum/";
8+
temphdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/temphdbpartbyenum/";
9+
testlogdb:"testlog";
10+
11+
// Test updates
12+
testtrade:((3#`GOOG),``,5?`4;10?100.0;10?100i;10#0b;10?.Q.A;10?.Q.A;10#`buy);
13+
testquote:((8?`4),``;(5?50.0),50+5?50.0;10?100.0;10?100i;10?100i;10?.Q.A;10?.Q.A;10#`3);
14+
15+
// expected WDB folder structure
16+
folder_patterns:{"*",x,"*"}each 1_/:string ` sv/: cross[hsym each `$string til count distinct testtrade[0],testquote[0];`trade`quote];
17+
18+
19+
// Function projections (using functions from helperfunctions.q)
20+
startproc:startorstopproc["start";;processcsv];
21+
stopproc:startorstopproc["stop";;processcsv];

0 commit comments

Comments
 (0)