Anycache

code/common/anymaputils.q

@@ -0,0 +1,5 @@
+.anymap.writetoanymap:{[filepath;data] (hsym filepath) 1: data; :(::)};
+.anymap.deriveanymapfiles:{[filepath] `$string[filepath],/:("";"#";"##")};
+.anymap.util.copyanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.cpy[;destfilepath] each fromfilepaths};
+.anymap.util.removeanymap:{[filepath] filepaths:.anymap.deriveanymapfiles[filepath]; .os.del each filepaths};
+.anymap.util.moveanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.ren[;destfilepath] each fromfilepaths};

code/common/getter.q

@@ -0,0 +1,28 @@
+// Bootstrap script that will enable an existing process to load in cache getter functionalities.
+
+\d .anycache.getter
+
+// Get cache from disk
+getcachefromdisk:{[filepath] get hsym filepath};
+
+// Get location of cache config and load it in.
+cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"];
+cacheconfig:.j.k raze read0 hsym first cacheconfiglocation;
+cachename: cacheconfig`cachename;
+asyncprocessname: cacheconfig`asyncprocessname;
+
+loadcaches:{
+    caches:key cacheconfig`componentcaches;
+    cachefilepaths: ` sv' ((hsym `$cacheconfig`cacherootdir),2#`$cachename),/:`$(string caches),\:"/data";
+    cachesdata:getcachefromdisk each cachefilepaths;
+    cachevarnames:` sv' `.anycache.cache,/:caches;
+    cachevarnames set' cachesdata;
+ };
+
+// Example of args: `cache1`cache2!(`a`b`c! 1 2 3;`d`e`f!4 5 6)
+requestnewcache:{[args]
+    maincache:` sv (hsym `$cacheconfig`cacherootdir),(`$cachename),`$asyncprocessname, "_", string .z.P;
+    (` sv maincache,`args) set args
+ };
+
+\d .

code/processes/anycachereadme.md

@@ -0,0 +1 @@
+AnyCache README

code/processes/gateway.q

@@ -592,6 +592,9 @@ if[@[value;`.timer.enabled;0b];
  .timer.repeat[.proc.cp[];0Wp;0D00:00:05;(`.gw.checktimeout;`);"Timeout queries which have been waiting too long"];
  .timer.repeat[.proc.cp[];0Wp;0D00:05;(`.gw.removeinactive;.gw.clearinactivetime);"Remove data for inactive handles"]];
 
+//AnyCache sample analytic
+.anycache.sampleanalytic:{ .gw.syncexec["select by sym from trade";`rdb] };
+
 // add in some api details 
 .api.add[`.gw.asyncexecjpt;1b;"Execute a function asynchronously.  The result is posted back to the client either directly down the socket (in which case the client must block and wait for the result - deferred synchronous) or wrapped in the postback function";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against; lambda: the function used to join the resulting data; symbol or lambda: postback;timespan: query timeout]";"The result of the query either directly or through the postback function"]
 .api.add[`.gw.asyncexec;1b;"Execute a function asynchronously.  The result is posted back to the client directly down the socket. The client must block and wait for the result - deferred synchronous.  Equivalent to .gw.asyncexecjpt with join function of raze, no postback and no timeout";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against]";"The result of the query"]

code/processes/setter.q

@@ -0,0 +1,90 @@
+// Setter process to set cache to disk
+
+\d .anycache.setter
+
+// Save cache down to disk
+savecachedowntodisk:{[data;filepath] (hsym filepath) set data };
+
+// Get location of cache config and load it in.
+cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"];
+cacheconfig:.j.k raze read0 hsym first cacheconfiglocation;
+
+// Mode of cache
+isrequest: "request" ~ cacheconfig.setter.mode;
+
+detectandwritecache:{ 
+    cacheinfo:detectcachetobuild[];
+    if[not count cacheinfo;
+        :(::)
+    ];
+    writetoken[cachepath:cacheinfo`cachepath;`start]; 
+    //Included some basic error trapping here 
+    success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; 
+    if[not success; 
+        cleanupcache cachepath; 
+        :(::) 
+    ];
+    writetoken[cachepath;`end]; 
+    //Can eject now if there are still caches to be built in the main cache 
+    if[not count remainingcaches:cacheinfo`maincachepath; 
+        :(::) 
+    ];
+    completecache cacheinfo`maincachepath 
+ };
+
+ writetoken:{[dir;stage] 
+    //accepts start or end and saves the current time as a timestamp to a flat file 
+    //in dir as `:startTime or `:endTime
+    (` sv (dir;stage)) set .z.P
+ };
+
+detectcachetobuild:{
+    cachename: cacheconfig`cachename;
+    maincachepath:` sv (hsym `$cacheconfig.cacheRootDir),`$cachename; 
+    caches:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like cachename,"_*";
+    if[not 0 = count caches; cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches];
+
+    latestcache:{
+    if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)];
+    if[(not isRequest) and ("N"$cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)];
+    :`cachename`newcache!(cachewithmaxstarttime;0b)}[];
+
+    if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]];
+    if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`cachename]];
+
+    componentcaches:` sv' cachewithmaxstarttime,/:key cacheconfig.componentCaches;
+    incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches;
+    writetoken[;`start] each incompletecomponentcaches;
+    writetoken[;`setter1] each incompletecomponentcaches;
+    args:enlist`;
+    if[isRequest;
+        argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like "AsyncCache*";
+        maxstarttime:string first max "P"$-1#' "_" vs' string argpaths;
+        argwithmaxstarttime:first argpaths where argpaths like "*",maxstarttime;
+        args: get ` sv argwithmaxstarttime,`args
+        ];
+    `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;args)
+ };
+
+generateandwritecache:{[cachepath; args] 
+    cachename:last ` vs cachepath;
+    connectiondetails: cacheconfig.componentCaches[cachename].dataSource;
+    cache: connectiondetails".anycache.sampleanalytic[(::)]";
+    .anymap.writetoanymap[cachepath;cache]
+ };
+
+cleanupcache:{[cachepath] 
+    //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure
+    hdel cachepath
+ };
+
+completecache:{[maincachepath]
+    cachename: string last ` vs maincachepath;
+    latestcache:first system"ls -lt ",(1_string maincachepath), " | grep ", cachename, " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'";
+    latestcachefilepath: ` sv maincachepath,`$latestcache;
+    writetoken[latestcachefilepath;`end];
+    if[not isRequest; system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename];
+    if[not isRequest; hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache)]
+ };
+
+\d .

config/cacheconfig.json

@@ -0,0 +1,23 @@
+{ 
+    "cacherootdir": "opt/anycache",
+    "cachename": "myfirstcache",
+    "asyncprocessname": "asynccache",
+    "componentcaches":{ 
+        "cache1": { 
+            "analytic": "analytic1", 
+            "dataSource": "connection1" 
+        }, 
+        "cache2": { 
+            "analytic": "analytic2", 
+            "dataSource": "connection2" 
+        } 
+    }, 
+   "getter":{ 
+    "interval": "0D00:05"
+   },
+   "setter":{ 
+    "mode": "timer", 
+    "interval": "0D00:30", 
+    "compression": "16 3 0"
+   }
+}