diff --git a/code/common/anymaputils.q b/code/common/anymaputils.q new file mode 100644 index 000000000..ee7e88189 --- /dev/null +++ b/code/common/anymaputils.q @@ -0,0 +1,5 @@ +.anymap.writetoanymap:{[filepath;data] (hsym filepath) 1: data; :(::)}; +.anymap.deriveanymapfiles:{[filepath] `$string[filepath],/:("";"#";"##")}; +.anymap.util.copyanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.cpy[;destfilepath] each fromfilepaths}; +.anymap.util.removeanymap:{[filepath] filepaths:.anymap.deriveanymapfiles[filepath]; .os.del each filepaths}; +.anymap.util.moveanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.ren[;destfilepath] each fromfilepaths}; \ No newline at end of file diff --git a/code/common/getter.q b/code/common/getter.q new file mode 100644 index 000000000..152b516e8 --- /dev/null +++ b/code/common/getter.q @@ -0,0 +1,28 @@ +// Bootstrap script that will enable an existing process to load in cache getter functionalities. + +\d .anycache.getter + +// Get cache from disk +getcachefromdisk:{[filepath] get hsym filepath}; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; +cachename: cacheconfig`cachename; +asyncprocessname: cacheconfig`asyncprocessname; + +loadcaches:{ + caches:key cacheconfig`componentcaches; + cachefilepaths: ` sv' ((hsym `$cacheconfig`cacherootdir),2#`$cachename),/:`$(string caches),\:"/data"; + cachesdata:getcachefromdisk each cachefilepaths; + cachevarnames:` sv' `.anycache.cache,/:caches; + cachevarnames set' cachesdata; + }; + +// Example of args: `cache1`cache2!(`a`b`c! 1 2 3;`d`e`f!4 5 6) +requestnewcache:{[args] + maincache:` sv (hsym `$cacheconfig`cacherootdir),(`$cachename),`$asyncprocessname, "_", string .z.P; + (` sv maincache,`args) set args + }; + +\d . \ No newline at end of file diff --git a/code/processes/anycachereadme.md b/code/processes/anycachereadme.md new file mode 100644 index 000000000..7700bbf32 --- /dev/null +++ b/code/processes/anycachereadme.md @@ -0,0 +1 @@ +AnyCache README \ No newline at end of file diff --git a/code/processes/gateway.q b/code/processes/gateway.q index 32d076bc7..6dbb5c27f 100644 --- a/code/processes/gateway.q +++ b/code/processes/gateway.q @@ -592,6 +592,9 @@ if[@[value;`.timer.enabled;0b]; .timer.repeat[.proc.cp[];0Wp;0D00:00:05;(`.gw.checktimeout;`);"Timeout queries which have been waiting too long"]; .timer.repeat[.proc.cp[];0Wp;0D00:05;(`.gw.removeinactive;.gw.clearinactivetime);"Remove data for inactive handles"]]; +//AnyCache sample analytic +.anycache.sampleanalytic:{ .gw.syncexec["select by sym from trade";`rdb] }; + // add in some api details .api.add[`.gw.asyncexecjpt;1b;"Execute a function asynchronously. The result is posted back to the client either directly down the socket (in which case the client must block and wait for the result - deferred synchronous) or wrapped in the postback function";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against; lambda: the function used to join the resulting data; symbol or lambda: postback;timespan: query timeout]";"The result of the query either directly or through the postback function"] .api.add[`.gw.asyncexec;1b;"Execute a function asynchronously. The result is posted back to the client directly down the socket. The client must block and wait for the result - deferred synchronous. Equivalent to .gw.asyncexecjpt with join function of raze, no postback and no timeout";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against]";"The result of the query"] diff --git a/code/processes/setter.q b/code/processes/setter.q new file mode 100644 index 000000000..f203a8e90 --- /dev/null +++ b/code/processes/setter.q @@ -0,0 +1,90 @@ +// Setter process to set cache to disk + +\d .anycache.setter + +// Save cache down to disk +savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; + +// Mode of cache +isrequest: "request" ~ cacheconfig.setter.mode; + +detectandwritecache:{ + cacheinfo:detectcachetobuild[]; + if[not count cacheinfo; + :(::) + ]; + writetoken[cachepath:cacheinfo`cachepath;`start]; + //Included some basic error trapping here + success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; + if[not success; + cleanupcache cachepath; + :(::) + ]; + writetoken[cachepath;`end]; + //Can eject now if there are still caches to be built in the main cache + if[not count remainingcaches:cacheinfo`maincachepath; + :(::) + ]; + completecache cacheinfo`maincachepath + }; + + writetoken:{[dir;stage] + //accepts start or end and saves the current time as a timestamp to a flat file + //in dir as `:startTime or `:endTime + (` sv (dir;stage)) set .z.P + }; + +detectcachetobuild:{ + cachename: cacheconfig`cachename; + maincachepath:` sv (hsym `$cacheconfig.cacheRootDir),`$cachename; + caches:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like cachename,"_*"; + if[not 0 = count caches; cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches]; + + latestcache:{ + if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; + if[(not isRequest) and ("N"$cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + :`cachename`newcache!(cachewithmaxstarttime;0b)}[]; + + if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]]; + if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`cachename]]; + + componentcaches:` sv' cachewithmaxstarttime,/:key cacheconfig.componentCaches; + incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches; + writetoken[;`start] each incompletecomponentcaches; + writetoken[;`setter1] each incompletecomponentcaches; + args:enlist`; + if[isRequest; + argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like "AsyncCache*"; + maxstarttime:string first max "P"$-1#' "_" vs' string argpaths; + argwithmaxstarttime:first argpaths where argpaths like "*",maxstarttime; + args: get ` sv argwithmaxstarttime,`args + ]; + `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;args) + }; + +generateandwritecache:{[cachepath; args] + cachename:last ` vs cachepath; + connectiondetails: cacheconfig.componentCaches[cachename].dataSource; + cache: connectiondetails".anycache.sampleanalytic[(::)]"; + .anymap.writetoanymap[cachepath;cache] + }; + +cleanupcache:{[cachepath] + //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure + hdel cachepath + }; + +completecache:{[maincachepath] + cachename: string last ` vs maincachepath; + latestcache:first system"ls -lt ",(1_string maincachepath), " | grep ", cachename, " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; + latestcachefilepath: ` sv maincachepath,`$latestcache; + writetoken[latestcachefilepath;`end]; + if[not isRequest; system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename]; + if[not isRequest; hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache)] + }; + +\d . \ No newline at end of file diff --git a/config/cacheconfig.json b/config/cacheconfig.json new file mode 100644 index 000000000..efb7f1565 --- /dev/null +++ b/config/cacheconfig.json @@ -0,0 +1,23 @@ +{ + "cacherootdir": "opt/anycache", + "cachename": "myfirstcache", + "asyncprocessname": "asynccache", + "componentcaches":{ + "cache1": { + "analytic": "analytic1", + "dataSource": "connection1" + }, + "cache2": { + "analytic": "analytic2", + "dataSource": "connection2" + } + }, + "getter":{ + "interval": "0D00:05" + }, + "setter":{ + "mode": "timer", + "interval": "0D00:30", + "compression": "16 3 0" + } +} \ No newline at end of file