Add the virtual-table capture pack

.gitignore

@@ -0,0 +1,7 @@
+# runtime state - the database, logs and tickerplant logs are all regenerated
+var/
+
+# editor / tooling
+.vscode/
+.claude/
+*.swp

appconfig/compressionconfig.csv

@@ -0,0 +1,2 @@
+table,minage,column,calgo,cblocksize,clevel
+default,7,default,2,16,9

appconfig/passwords/accesslist.txt

@@ -0,0 +1,7 @@
+compression:pass
+discovery:pass
+feed:pass
+idb:pass
+segmentedtickerplant:pass
+wdb:pass
+admin:admin

appconfig/passwords/feed.txt

@@ -0,0 +1 @@
+feed:pass

appconfig/process.csv

@@ -0,0 +1,7 @@
+host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd
+localhost,{KDBBASEPORT}+1,discovery,discovery1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q
+localhost,{KDBBASEPORT},segmentedtickerplant,stp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG},q
+localhost,{KDBBASEPORT}+5,wdb,wdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q,1,,q
+localhost,{KDBBASEPORT}+14,feed,feed1,,1,0,,,${KDBAPPCODE}/tick/feed.q,1,,q
+localhost,{KDBBASEPORT}+30,idb,idb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBAPPCODE}/processes/vtidb.q,1,-s 4,q
+localhost,{KDBBASEPORT}+40,compression,cmp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBAPPCODE}/processes/vtcompress.q,0,,q

appconfig/settings/compression.q

@@ -0,0 +1,19 @@
+// Virtual-table capture pack : compression config
+// see docs/virtual-table-capture-pack.md §4.4 and §7
+
+\d .cmp
+hdbpath:hsym`$getenv`KDBHDB              // one database root - the writer writes where the
+                                         // readers read, so this is the capture tree itself
+maxage:365                               // oldest partition to consider. The lower bound is
+                                         // minage in compressionconfig.csv, which must stay
+                                         // >0 so the live partition is untouched
+
+minfilesize:4096                         // skip column files this size or smaller: a file
+                                         // already inside one filesystem block frees nothing
+                                         // and only adds work to every read (doc §7.3).
+                                         // 0 compresses everything, as stock TorQ does
+
+// The hdbstructure override lives in code/processes/vtcompress.q, not here: settings files
+// load BEFORE code/common/compress.q, so defining it here is silently overwritten and the
+// job compresses nothing.
+\d .

appconfig/settings/default.q

@@ -0,0 +1,12 @@
+system"c 23 2000"
+
+// End-of-day roll time, applied to every process. The business day is GMT
+// (appconfig/settings/segmentedtickerplant.q), so an offset of zero rolls at midnight UTC.
+// Set an offset to end the day elsewhere: 0D09:00 rolls at 09:00 UTC.
+//
+// NOTE an offset moves the business-day BOUNDARY, not just the event, so .z.D and the live
+// partition disagree for those hours. The reader takes the live partition from disk rather
+// than .z.D for exactly this reason (doc 5.8).
+\d .eodtime
+rolltimeoffset:0D00:00:00.000
+\d .

appconfig/settings/feed.q

@@ -0,0 +1,8 @@
+// Bespoke Feed config : Finance Starter Pack
+
+\d .servers	
+enabled:1b						
+CONNECTIONS:enlist `segmentedtickerplant		// Feedhandler connects to the tickerplant
+HOPENTIMEOUT:30000
+
+\d .

appconfig/settings/idb.q

@@ -0,0 +1,35 @@
+// Virtual-table capture pack : IDB config
+// see docs/virtual-table-capture-pack.md §5
+
+\d .vtidb
+roots:enlist hsym`$getenv`KDBDB          // database roots to scan. a list rather than an atom
+                                         // so one reader can serve several capture stacks (§8.3)
+tabs:`                                   // ` = discover the table list from disk. Scans the
+                                         // live partition only once the catalogue is warm, so
+                                         // the cost does not grow with retention. A table
+                                         // added to an already-rolled date needs dropcache[]
+                                         // (6.1). Set explicitly to restrict, e.g. `trade
+historydays:0W                           // how many days back to attach. 0W = everything
+sweep:0D00:00:30                         // backstop rescan. The primary path is the wdb's
+                                         // notification (§4.1); this only bounds a dropped
+                                         // message, so it is deliberately slack
+symsweep:0D00:00:01                      // how often to check the enumeration domain. A new
+                                         // symbol VALUE in an existing partition creates no
+                                         // directory, so the writer never announces it and it
+                                         // reads as null until reloaded (§5.4). One hcount per
+                                         // root, so this can be fast
+
+partitioncol:`sym                       // name the partition column is exposed under. Not
+                                         // stored on disk, so it cannot be derived: it must
+                                         // match the schema or client queries will not port
+wdbtypes:`wdb
+wdbcheckcycles:3                         // wait this many cycles for the wdb, then start
+wdbconnsleepintv:5                       // anyway - without a writer the sweep keeps the
+                                         // reader current, just slower
+
+\d .servers
+CONNECTIONS:`wdb`discovery               // wdb: to register for new-partition notifications
+STARTUP:1b
+
+\d .proc
+loadprocesscode:0b                       // process code comes from -load, not $KDBAPPCODE/idb/

appconfig/settings/segmentedtickerplant.q

@@ -0,0 +1,21 @@
+\d .
+
+createlogs:1b;                  // create a logs
+
+\d .stplg
+
+multilog:`tabperiod;            // [tabperiod|singular|periodic|tabular|custom]
+multilogperiod:0D01;
+errmode:1b;
+batchmode:`defaultbatch;        // [memorybatch|defaultbatch|immediate]
+customcsv:hsym first .proc.getconfigfile["stpcustom.csv"];
+replayperiod:`day               // [period|day|prior]
+
+\d .proc
+
+loadprocesscode:1b;
+
+\d .eodtime
+
+datatimezone:`$"GMT";
+rolltimezone:`$"GMT";

appconfig/settings/wdb.q

@@ -0,0 +1,46 @@
+// Virtual-table capture pack : WDB config
+// see docs/virtual-table-capture-pack.md §3.3
+
+\d .wdb
+savedir:hdbdir:hsym`$getenv`KDBWDB       // one directory; sym file lives at its root
+writedownmode:`partbyattr                // date + instrument directories.
+                                         // NB necessary but NOT sufficient - on its own it
+                                         // also writes the partition column into the files,
+                                         // which defeats the purpose. see code/wdb/vtwrite.q
+mode:`saveandsort                        // the sort phase is overridden to a no-op (4.2)
+immediate:1b                             // flush on every timer tick, ignore maxrows
+settimer:0D00:00:01                      // ...every second
+gc:0b                                    // at 1s cadence do not gc on every flush
+rdbtypes:hdbtypes:gatewaytypes:()        // none of these processes exist
+sorttypes:sortworkertypes:()
+idbtypes:`idb
+permitreload:0b                          // nothing to reload
+sortcsv:hsym`$getenv[`KDBAPPCONFIG],"/sort.csv"
+// Seed the partition from the BUSINESS date, not the calendar date.
+//
+// TorQ seeds .wdb.currentpartition from .proc.cd[], and clearwdbdata[] deletes THAT
+// partition before replaying the tickerplant log. Under a roll offset the two disagree, so
+// the delete misses, fixpartition corrects the date afterwards, and the replay writes the
+// day on top of data that was never removed. Measured: 442 duplicate rows after one restart.
+// .eodtime.getday is what the tickerplant dates its own logs with, so seeding from it makes
+// the two agree at any offset, including none.
+//
+// NOTE .eodtime is not loaded when this file runs, so the lookup sits inside the function
+// body rather than at the top level.
+startpartition:{[]
+  d:@[{[x] .eodtime.getday .z.p};(::);{[e] .proc.cd[]}];
+  (`date^@[value;`.wdb.partitiontype;`date])$d
+  };
+
+getpartition:{[] @[value;`.wdb.currentpartition;{[e] .wdb.startpartition[]}]};
+
+symdomain:`sym                           // this stack's enumeration domain file. One reader
+                                         // serving several stacks needs a distinct name per
+                                         // stack (`syma, `symb...) - two roots both calling
+                                         // it `sym cannot be read together (8.3.1)
+
+\d .servers
+CONNECTIONS:`segmentedtickerplant`idb`discovery
+
+\d .proc
+loadprocesscode:1b                       // loads $KDBAPPCODE/wdb/vtwrite.q

appconfig/sort.csv

@@ -0,0 +1,4 @@
+tabname,att,column,sort
+default,p,sym,1
+quote,p,sym,1
+trade,p,sym,1

code/processes/vtcompress.q

@@ -0,0 +1,121 @@
+// Virtual-table capture pack : compression process
+// see docs/virtual-table-capture-pack.md §4.4 and §7
+//
+// Replaces code/processes/compression.q: same job - walk the tree, compress anything older
+// than minage, exit - but first corrects the directory classifier, which cannot see a
+// date+instrument layout.
+//
+//   ./compress.sh              compress
+//   ./compress.sh --dry-run    report what would be compressed, change nothing
+
+\d .cmp
+
+inputcsv:@[value;`inputcsv;.proc.getconfigfile["compressionconfig.csv"]];
+hdbpath:@[value;`hdbpath;`:hdb];
+maxage:@[value;`maxage;365];
+dryrun:`dryrun in key .proc.params;
+
+// the age tier lives in the csv as minage. VTCMP_CONFIG points at a different csv, which is
+// how ./compress.sh --test exercises the job against a database only a couple of days old
+if[count e:getenv`VTCMP_CONFIG; inputcsv:e];
+
+// The size gate. A column file already inside one filesystem block frees nothing when
+// compressed and only adds decompression work to every read; at this layout a third of all
+// files are in that state and free 0% of disk (§7.2). 0 disables the gate, as stock does.
+minfilesize:@[value;`minfilesize;4096];
+if[count e:getenv`VTCMP_MINFILESIZE; minfilesize:"J"$e];
+
+// Stock hdbstructure classifies a path by depth and knows only partition/table/column and
+// table/column. A partbyattr column file sits one deeper, so it matches neither, `table` stays
+// null, and showcomp drops every row - the job succeeds and compresses nothing. Adding the
+// extra depth folds the instrument level away, so per-column rules keep working unchanged.
+//
+// Must be applied here, not in appconfig/settings/compression.q: settings load before
+// code/common/compress.q, so an override there is overwritten by the stock definition.
+hdbstructure:{
+  t:([]fullpath:(raze/)traverse x);
+  base:count "/" vs string x;
+  t:update splitcount:count each split from update split:"/" vs' string fullpath,column:`,table:`,partition:(count t)#enlist"" from t;
+  / date partitioned : partition/table/column
+  t:update partition:split[;base],table:`$split[;base+1],column:`$split[;base+2] from t where splitcount=base+3;
+  / partbyattr : partition/table/instrument/column
+  t:update partition:split[;base],table:`$split[;base+1],column:`$split[;base+3] from t where splitcount=base+4;
+  / splayed : table/column
+  t:update table:`$split[;base],column:`$split[;base+1] from t where splitcount=base+2;
+  t:update partition:{$[not all null r:"D"$'x;r;not all null r:"M"$'x;r;"I"$'x]}[partition] from t;
+  $[14h=type t`partition; t:update age:.z.D - partition from t;
+    13h=type t`partition; t:update age:(`month$.z.D) - partition from t;
+    t:update age:{$[all x within 1000 3000; x - `year$.z.D;(count x)#0Ni]}[partition] from t];
+  delete splitcount,split from t
+  };
+
+// The --dry-run report. The interesting number is not how many files are in scope but how big
+// they are: a column file already inside one filesystem block cannot get smaller on disk however
+// well its bytes compress.
+dryrunreport:{[t]
+  fsblock:4096;                                     // ext4 default: the floor on any file's disk usage
+  alloc:{[b;x] b*ceiling x%b}[fsblock];             // bytes actually allocated for a file of x bytes
+  s:asc t`currentsize;
+  pct:{[s;p] s `long$(count[s]-1)*p%100}[s];
+  mb:{.Q.f[2;x%2 xexp 20]};
+  / hcount reports a compressed file's UNCOMPRESSED length, so currentsize alone cannot tell
+  / whether the job has already run - ask each file's header instead
+  done:sum {0<count -21!x} each t`fullpath;
+  -1 "";
+  -1 "  database   ",string .cmp.hdbpath;
+  -1 "  config     ",.cmp.inputcsv;
+  -1 "  age tier   partitions older than ",string[exec first compressage from t]," days";
+  -1 "  size gate  files of ",string[.cmp.minfilesize]," bytes or less are skipped: ",
+     string[.cmp.toosmall]," excluded";
+  -1 "";
+  -1 "  in scope   ",string[count t]," files, ",string[count distinct t`partition]," partitions, ",
+     string[count distinct t`table]," tables";
+  -1 "  to do      ",string[count[t]-done]," files (",string[done]," already compressed, which the job skips)";
+  -1 "";
+  -1 "  the sizes below are logical - what the columns hold - so they do not change once the";
+  -1 "  job has run. they describe what compression has to work with, not what is on disk now.";
+  -1 "";
+  -1 "  file size  min ",string[first s],"  p50 ",string[pct 50],"  p90 ",string[pct 90],"  max ",string[last s]," bytes";
+  -1 "  under 4 kB ",string[sum s<fsblock]," of ",string[count s]," files (",string[`long$100*avg s<fsblock],"%)";
+  -1 "";
+  -1 "  logical    ",mb[sum s]," MB     <- bytes the columns hold";
+  -1 "  allocated  ",mb[sum alloc s]," MB     <- what those bytes cost on disk uncompressed";
+  -1 "  floor      ",mb[fsblock*count s]," MB     <- one block per file, unavoidable";
+  -1 "  headroom   ",mb[(sum alloc s)-fsblock*count s]," MB     <- the most compression can ever free";
+  -1 "";
+  -1 "  by table";
+  show 0!select files:count i, logicalmb:"F"$mb sum currentsize, underblock:sum currentsize<fsblock by table from t;
+  -1 "";
+  };
+
+\d .
+
+if[not count key hsym .cmp.hdbpath;
+  .lg.e[`compression;err:"invalid database path ",string .cmp.hdbpath]; 'err];
+
+.cmp.scope:.cmp.showcomp[hsym .cmp.hdbpath;.cmp.inputcsv;.cmp.maxage];
+
+// apply the size gate. done here rather than in the csv because compressionconfig.csv rules
+// are per table and per column, and this depends on how much data landed in one directory
+.cmp.toosmall:0;
+if[.cmp.minfilesize>0;
+  .cmp.toosmall:exec count i from .cmp.scope where currentsize<=.cmp.minfilesize;
+  .cmp.scope:select from .cmp.scope where currentsize>.cmp.minfilesize;
+  if[.cmp.toosmall; .lg.o[`compression;"size gate: skipping ",string[.cmp.toosmall],
+    " files of ",string[.cmp.minfilesize]," bytes or less - they cannot free a block"]]];
+
+if[not count .cmp.scope;
+  .lg.o[`compression;"nothing in scope - either the tree is empty, or every partition is younger than minage"];
+  exit 0];
+
+.lg.o[`compression;"in scope: ",string[count .cmp.scope]," files across ",
+  string[count distinct .cmp.scope`partition]," partitions"];
+
+if[.cmp.dryrun;
+  .cmp.dryrunreport .cmp.scope;
+  exit 0];
+
+.cmp.compressfromtable[.cmp.scope];
+.cmp.summarystats[];
+.lg.o[`compression;"finished compression"];
+exit 0