diff --git a/CHANGELOG.md b/CHANGELOG.md
index 213665c64..2dfb42fd6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- #1117: Add `sync` command for incremental loading of changed files in dev-mode modules. Detects modified files since last sync using SHA-1 hash and recompiles only what is stale. Supports `-delete` for processing removed files and `-test` for running changed test-phase unit tests.
- #106: A module can now specify `` to prevent installation in non-%SYS namespaces.
+- #536: Improve filesystem repository cache through performance improvements, smart auto-cache rebuilding on install, and new `-rebuild-cache` flag for `repo` command to manually rebuild the entire cache.
### Changed
- Minimum supported Python version is now 3.9
+- #536: An explicit `-depth` on a filesystem repository is no longer overwritten during the initial scan, so a module added at a level within the configured depth is still discovered.
### Fixed
- Performance: Studio project creation on package load in dev mode is now 80% faster.
diff --git a/src/cls/IPM/General/Sync/Pipeline.cls b/src/cls/IPM/General/Sync/Pipeline.cls
index b99ce2088..47134de42 100644
--- a/src/cls/IPM/General/Sync/Pipeline.cls
+++ b/src/cls/IPM/General/Sync/Pipeline.cls
@@ -36,7 +36,7 @@ ClassMethod Run(
}
// Step 1: Check if module.xml changed; reload manifest if so
- set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml")
+ set moduleXmlRelPath = ##class(%IPM.Utils.File).NormalizePath("module.xml")
set moduleXmlPath = root _ "module.xml"
set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath)
@@ -305,7 +305,7 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource
set relPath = resource.Processor.OnItemRelativePath(childName)
}
if relPath '= "" {
- set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath)
+ set normalizedRelPath = ##class(%IPM.Utils.File).NormalizePath(relPath)
set reverseIndex(normalizedRelPath) = resource.Name
set reverseIndex(normalizedRelPath, "Processor") = resource.Processor
set reverseIndex(normalizedRelPath, "Resource") = resource
@@ -320,7 +320,7 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource
// directory. Prefix-scan allFiles to map every file under that dir to this resource.
set syncDir = resource.Processor.GetSyncDirectory()
if syncDir '= "" {
- set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/")
+ set prefix = ##class(%IPM.Utils.File).NormalizePath(syncDir _ "/")
set prefixLen = $length(prefix)
set dirRelPath = prefix
for {
diff --git a/src/cls/IPM/General/TempLocalRepoManager.cls b/src/cls/IPM/General/TempLocalRepoManager.cls
index 2619ec216..6fe73fbcc 100644
--- a/src/cls/IPM/General/TempLocalRepoManager.cls
+++ b/src/cls/IPM/General/TempLocalRepoManager.cls
@@ -27,7 +27,7 @@ Method Create(useFirst As %Boolean) [ Internal, Private ]
// Make sure this is the first/last repo to be found by SQL query in %IPM.Repo.Manager:SearchRepositoriesForModule
set ..Repo.OverriddenSortOrder = $select(useFirst:-1000 ,1:1000)
- $$$ThrowOnError(..Repo.BuildCache(1,1,1))
+ $$$ThrowOnError(..Repo.BuildCache(1,1))
}
ClassMethod SkipCreate(location As %String) As %Boolean [ Internal ]
diff --git a/src/cls/IPM/Lifecycle/Module.cls b/src/cls/IPM/Lifecycle/Module.cls
index f5304f94e..b1c2c7992 100644
--- a/src/cls/IPM/Lifecycle/Module.cls
+++ b/src/cls/IPM/Lifecycle/Module.cls
@@ -41,6 +41,8 @@ Method %Clean(ByRef pParams) As %Status
kill ^IPM.Repo.DefinitionD
kill ^IPM.Repo.DefinitionI
kill ^IPM.Repo.Filesystem.CacheD
+ kill ^IPM.Repo.Filesystem.CacheI
+ kill ^IPM.Repo.Filesystem.CacheS
kill ^IPM.General.SettingsD
kill ^IPM.UpdateStep.AnyMemberD
kill ^IPM.UpdateStep.PrimaryOnlyD
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 13afceb39..5488405f8 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -2193,6 +2193,29 @@ ClassMethod Repository(ByRef pCommandInfo) [ Internal ]
set tType = "%IPM.Repo.Remote.Definition"
$$$ThrowOnError($classmethod(tType,"Configure",1,.tModifiers,.tData))
do ..Shell("repo -list")
+ } elseif $$$HasModifier(pCommandInfo,"rebuild-cache") {
+ // Validate repository exists and is filesystem type
+ set repoName = $$$GetModifier(pCommandInfo,"name")
+ if (repoName = "") {
+ $$$ThrowStatus($$$ERROR($$$GeneralError,"-rebuild-cache requires -name "))
+ }
+ set serverDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(repoName,,.sc)
+ $$$ThrowOnError(sc)
+
+ if 'serverDef.%IsA("%IPM.Repo.Filesystem.Definition") {
+ $$$ThrowStatus($$$ERROR($$$GeneralError,"Cache rebuild only supported for filesystem repositories"))
+ }
+
+ // Rebuild cache with purge
+ write !,"Rebuilding cache for repository: ",repoName
+ // $ztimestamp rather than $zhorolog: the latter resets at midnight, making the delta negative.
+ set start = $ztimestamp
+ set sc = serverDef.BuildCache(1, 1) // purge=1, verbose=1
+ $$$ThrowOnError(sc)
+
+ set end = $ztimestamp
+ set elapsed = (($piece(end,",",1) - $piece(start,",",1)) * 86400) + ($piece(end,",",2) - $piece(start,",",2))
+ write !,"Cache rebuilt successfully in ", $fnumber(elapsed,"",2), " seconds."
} else {
set tName = $$$GetModifier(pCommandInfo,"name")
set tType = $listget(serverClassList)
@@ -2294,6 +2317,16 @@ ClassMethod ShowModulesForRepository(
set list("width") = width
write !
do ..DisplayModules(.list)
+
+ // pRepoName reaches here straight from -name, so it need not be a repository that exists.
+ if '##class(%IPM.Repo.Definition).ServerDefinitionKeyExists(pRepoName) {
+ quit
+ }
+ set server = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(pRepoName,,.tSC)
+ $$$ThrowOnError(tSC)
+ if server.%IsA("%IPM.Repo.Filesystem.Definition") && (server.CacheLastRebuilt '= "") {
+ write !,"Last cache rebuild for '", pRepoName, "' was on ", server.CacheLastRebuilt, " (UTC). If needed, run 'repo -n "_pRepoName_" -rebuild-cache' to refresh the cache.",!
+ }
}
Query SourceControlClasses() As %SQLQuery(ROWSPEC = "ID:%String,Name:%String") [ SqlProc ]
@@ -2510,6 +2543,29 @@ ClassMethod CheckModuleNamespace() As %Status
quit $$$OK
}
+/// Rebuild one filesystem repository's cache ahead of a dependency search.
+/// Best effort: a failure warns rather than failing the install, since the existing cache is still
+/// usable, it just might not list modules added since the last rebuild.
+/// Announcement is written once, only when there is a repository to rebuild.
+ClassMethod RebuildFilesystemRepoCache(
+ repoDef As %IPM.Repo.Filesystem.Definition,
+ ByRef announced As %Boolean = 0) [ Internal, Private ]
+{
+ // A root that is not currently mounted is skipped rather than reported: BuildCache refuses it,
+ // and an install has no business failing or warning over an unrelated repository.
+ if 'repoDef.GetPackageService().IsAvailable() {
+ quit
+ }
+ if 'announced {
+ write !,"Rebuilding cache(s) for filesystem repo(s)..."
+ set announced = 1
+ }
+ set sc = repoDef.BuildCache(0, 0) // purge=0, verbose=0
+ if $$$ISERR(sc) {
+ do ##class(%IPM.General.LogManager).Warning("Could not rebuild cache for filesystem repository '"_repoDef.Name_"': "_$system.Status.GetOneErrorText(sc),1)
+ }
+}
+
ClassMethod Install(
ByRef pCommandInfo,
pLog As %IPM.General.AbstractHistory = "") [ Internal ]
@@ -2543,6 +2599,29 @@ ClassMethod Install(
$$$ThrowStatus($$$ERROR($$$GeneralError, "No repositories are configured and enabled in this namespace."))
}
+ // Rebuild the filesystem repository caches before searching, so modules added since the last
+ // rebuild are discoverable.
+ set announced = 0
+ if (tRegistry '= "") {
+ set tRepoDef = ##class(%IPM.Repo.Definition).ServerDefinitionKeyOpen(tRegistry,,.tSC)
+ if $$$ISERR(tSC) {
+ $$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("No repository with name '%1' exists in this namespace.",tRegistry)))
+ }
+ if tRepoDef.%IsA("%IPM.Repo.Filesystem.Definition") {
+ do ..RebuildFilesystemRepoCache(tRepoDef,.announced)
+ }
+ } else {
+ set tFsRepoResult = ##class(%SQL.Statement).%ExecDirect(,"SELECT ID FROM %IPM_Repo_Filesystem.Definition WHERE Enabled = 1")
+ $$$ThrowSQLIfError(tFsRepoResult.%SQLCODE,tFsRepoResult.%Message)
+ while tFsRepoResult.%Next(.tFetchSC) {
+ $$$ThrowOnError(tFetchSC)
+ set tFsRepo = ##class(%IPM.Repo.Filesystem.Definition).%OpenId(tFsRepoResult.%Get("ID"),,.tOpenSC)
+ $$$ThrowOnError(tOpenSC)
+ do ..RebuildFilesystemRepoCache(tFsRepo,.announced)
+ }
+ $$$ThrowOnError(tFetchSC)
+ }
+
set tVersion = $get(pCommandInfo("parameters","version"))
set tKeywords = $$$GetModifier(pCommandInfo,"keywords")
set tForce = $$$HasModifier(pCommandInfo,"force")
diff --git a/src/cls/IPM/Repo/Filesystem/Cache.cls b/src/cls/IPM/Repo/Filesystem/Cache.cls
index 03edd9f2f..d3c196793 100644
--- a/src/cls/IPM/Repo/Filesystem/Cache.cls
+++ b/src/cls/IPM/Repo/Filesystem/Cache.cls
@@ -5,16 +5,33 @@ Class %IPM.Repo.Filesystem.Cache Extends (%Persistent, %IPM.General.ModuleInfo)
Parameter DEFAULTGLOBAL = "^IPM.Repo.Filesystem.Cache";
-Index CacheItemIndex On (Root, SubDirectory) [ Data = LastModified, Unique ];
+/// Uniqueness constraint only: one cache entry per directory under a given root.
+Index CacheItemIndex On (Root, SubDirectoryHash) [ Unique ];
Property Root As %String(MAXLEN = 260) [ Required ];
ForeignKey RootFK(Root) References %IPM.Repo.Filesystem.Definition(RootIndex) [ OnDelete = cascade ];
-Property SubDirectory As %String(MAXLEN = 260);
+Property SubDirectory As %String(MAXLEN = "");
+
+/// SHA-1 of SubDirectory as uppercase hex, giving the unique index a fixed 40-character subscript.
+/// SubDirectory itself cannot be one: a path can exceed the combined subscript budget, and %String's
+/// SQLUPPER collation reports "mods/Foo" and "mods/foo" as the same key when they are two directories.
+Property SubDirectoryHash As %String(MAXLEN = 40) [ Required, SqlComputeCode = {set {*} = ##class(%IPM.Repo.Filesystem.Cache).PathHash({SubDirectory})}, SqlComputed, SqlComputeOnChange = (%%INSERT, %%UPDATE, SubDirectory) ];
Property LastModified As %TimeStamp [ Required ];
+/// SHA-1 content hash (lowercase hex) of the module.xml file this entry was built from.
+/// Content, not modification time: mtime is unreliable across container bind mounts, where it can
+/// be coarse, stale, or never propagate at all.
+Property ContentHash As %String(MAXLEN = 40);
+
+/// Indicates if full manifest has been populated (deferred loading optimization).
+/// When 0: Only metadata (Name, Version) is cached; Manifest stream is empty.
+/// When 1: Full XML has been loaded into Manifest property.
+/// Use LoadManifestForCacheEntry() to populate on-demand when needed.
+Property ManifestLoaded As %Boolean [ InitialExpression = 0 ];
+
/// Full module manifest
Property Manifest As %Stream.GlobalCharacter;
@@ -25,8 +42,22 @@ Property SemVer As %String(MAXLEN = 512) [ SqlComputeCode = {set {*} = ##class(%
Index RootNameVersion On (Root, Name, VersionString) [ Unique ];
+ClassMethod PathHash(subDirectory As %String) As %String [ CodeMode = expression ]
+{
+##class(%xsd.hexBinary).LogicalToXSD($system.Encryption.SHA1Hash(subDirectory))
+}
+
ClassMethod %OnBeforeBuildIndices(ByRef indexlist As %String(MAXLEN="") = "") As %Status [ Private, ServerOnly = 1 ]
{
+ if (indexlist [ "CacheItemIndex") || (indexlist = "") {
+ // CacheItemIndex is unique on SubDirectoryHash, so rows carried over from a release that
+ // predates that property would all index under the same empty value. A cache row is derived
+ // data. Drop them and let the next build re-parse the manifests.
+ set tRes = ##class(%SQL.Statement).%ExecDirect(,"delete from %IPM_Repo_Filesystem.Cache where SubDirectoryHash is null or SubDirectoryHash = ''")
+ if tRes.%SQLCODE < 0 {
+ quit $$$ERROR($$$SQLCode,tRes.%SQLCODE,tRes.%Message)
+ }
+ }
if (indexlist [ "SemVer") || (indexlist = "") {
// Force recomputation of SemVer property if index needs to be rebuilt (i.e., because structure has changed)
set tRes = ##class(%SQL.Statement).%ExecDirect(,"update %IPM_Repo_Filesystem.Cache set SemVer = ''")
@@ -113,6 +144,68 @@ Method HandleSaveError(pSC As %Status) As %Status
quit tSC
}
+/// Open a cache entry and validate it against the filesystem
+/// If the cached entry is stale (ContentHash differs from the hash of the module.xml on disk),
+/// re-parse the module.xml and update the cache entry
+/// Returns the cache object (possibly refreshed) or throws error if not found
+ClassMethod RootNameVersionOpenValidated(
+ root As %String,
+ name As %String,
+ versionString As %String,
+ concurrency As %Integer = -1,
+ Output status As %Status) As %IPM.Repo.Filesystem.Cache
+{
+ set status = $$$OK
+ set cacheObj = ""
+
+ try {
+ // Open the cache entry
+ set cacheObj = ..RootNameVersionOpen(root, name, versionString, concurrency, .status)
+ if $$$ISERR(status) {
+ quit
+ }
+
+ set dirPath = ##class(%File).NormalizeFilename(cacheObj.SubDirectory, cacheObj.Root)
+ set filePath = ##class(%File).NormalizeFilename("module.xml", dirPath)
+ if '##class(%File).Exists(filePath) {
+ // Report it but leave the row alone. The absence may be transient, such as a source
+ // control sync in flight or an unmounted share. Pruning belongs to CleanupStaleEntries,
+ // which only runs after a full walk has established the file is really gone.
+ set cacheObj = ""
+ set status = $$$ERROR($$$GeneralError, "module.xml not found in " _ dirPath)
+ quit
+ }
+
+ set currentHash = $$$lcase(##class(%File).SHA1Hash(filePath, 1))
+ if (cacheObj.ContentHash '= currentHash) {
+ // An unsaved row still holds the old Name and Version, so the index re-check below would
+ // pass on stale data.
+ if '##class(%IPM.Repo.Filesystem.Definition).RefreshCacheEntry(cacheObj, filePath, currentHash) {
+ set message = filePath_" now describes a module already cached under "_root
+ set cacheObj = ""
+ set status = $$$ERROR($$$GeneralError, message)
+ quit
+ }
+
+ // The re-parse can change Name or Version, so the entry may no longer be the module that
+ // was asked for. Returning it would install something other than what the caller resolved.
+ // Re-check through the index rather than comparing properties, so the same key semantics
+ // apply on both sides of the comparison as on the original lookup.
+ if '..RootNameVersionExists(root, name, versionString, .stillId) || (stillId '= cacheObj.%Id()) {
+ set message = filePath_" now describes "_cacheObj.Name_" "_cacheObj.VersionString_", not "_name_" "_versionString
+ set cacheObj = ""
+ set status = $$$ERROR($$$GeneralError, message)
+ quit
+ }
+ }
+ } catch ex {
+ set status = ex.AsStatus()
+ set cacheObj = ""
+ }
+
+ quit cacheObj
+}
+
Storage Default
{
@@ -164,6 +257,15 @@ Storage Default
DisplayName
+
+ContentHash
+
+
+ManifestLoaded
+
+
+SubDirectoryHash
+
^IPM.Repo.Filesystem.CacheD
CacheDefaultData
diff --git a/src/cls/IPM/Repo/Filesystem/Definition.cls b/src/cls/IPM/Repo/Filesystem/Definition.cls
index 582b32405..2612f0765 100644
--- a/src/cls/IPM/Repo/Filesystem/Definition.cls
+++ b/src/cls/IPM/Repo/Filesystem/Definition.cls
@@ -17,6 +17,9 @@ Property Root As %String(MAXLEN = 260) [ Required ];
/// How many levels of depth to search for module.xml files; 0 indicates unlimited.
Property Depth As %Integer [ InitialExpression = 0, Required ];
+/// Timestamp of last cache rebuild for this repository
+Property CacheLastRebuilt As %TimeStamp;
+
/// Prompt to use for Root in interactive configuration of this repository type
Parameter RootPromptString = {$$$Text("Root File Path:","ZPM")};
@@ -29,6 +32,7 @@ XData Commands
+
repo -name LocalFiles -snapshots 1 -fs -depth 2 -path C:\MyWorkspace\RootModuleDir\
@@ -81,8 +85,14 @@ ClassMethod OnConfigure(
set pInstance.Depth = tDepth
}
- // This also saves it.
- $$$ThrowOnError(pInstance.BuildCache(1,1,1))
+ if ##class(%File).DirectoryExists(pInstance.Root) {
+ $$$ThrowOnError(pInstance.BuildCache(1,1))
+ } else {
+ // Still configure the repository, since the root may be a share that mounts later, but
+ // warn that nothing resolves from it until the cache is built. Configure() does the save.
+ set tMessage = "Root '"_pInstance.Root_"' does not exist. No modules will resolve from this repository until it does and the cache is rebuilt."
+ do ##class(%IPM.General.LogManager).Warning(tMessage,1)
+ }
} catch e {
set tSC = e.AsStatus()
}
@@ -114,13 +124,15 @@ Trigger RootChanged [ Event = UPDATE, Foreach = row/object ]
}
}
+/// Build the cache of modules found in the filesystem repository.
+/// pPurge - if 1, re-parses every manifest instead of skipping the ones whose content is unchanged
Method BuildCache(
- pPurge As %Boolean = 1,
- pVerbose As %Integer = 0,
- pAutoDetectDepth As %Boolean = 0) As %Status
+ pPurge As %Boolean = 0,
+ pVerbose As %Integer = 0) As %Status
{
set tSC = $$$OK
set tInitTLevel = $tlevel
+
try {
set tLogManager = ##class(%IPM.General.LogManager).%Get(.tSC)
$$$ThrowOnError(tSC)
@@ -129,15 +141,32 @@ Method BuildCache(
set tXSLTProvider = ##class(%IPM.Repo.XSLTProvider).%Get(.tSC)
$$$ThrowOnError(tSC)
+ // Refuse to rebuild against a root we cannot see. The walk would report no files, every
+ // existing entry would look deleted, and the repository would be silently emptied. An
+ // unmounted share or an interrupted source control sync looks exactly like this.
+ if '##class(%File).DirectoryExists(..Root) {
+ set tSC = $$$ERROR($$$GeneralError,"Filesystem repository root does not exist: "_..Root)
+ quit
+ }
+
tstart
- if (pPurge) && (..%Id() '= "") {
+
+ // Serialize against other rebuilds of this repository. Two concurrent rebuilds each compute
+ // their own visited set, so whichever finished last would delete the other's new entries.
+ // A repository with no ID yet is not visible to any other process.
+ if (..%Id() '= "") {
set tLockManager = ##class(%IPM.Utils.LockManager).%New()
$$$ThrowOnError(tLockManager.LockClassId($classname(),..%Id()))
- #dim tResult As %SQL.StatementResult
- set tResult = ##class(%SQL.Statement).%ExecDirect(,"delete from %IPM_Repo_Filesystem.Cache where Root = ?",..Root)
- if (tResult.%SQLCODE < 0) {
- set tSC = $$$ERROR($$$SQLCode,tResult.%SQLCODE,tResult.%Message)
- quit
+
+ if (pPurge) {
+ // Blank the hashes rather than deleting the rows: every entry then mismatches and is
+ // re-parsed in place, so a concurrent reader never sees an emptied repository.
+ #dim tResult As %SQL.StatementResult
+ set tResult = ##class(%SQL.Statement).%ExecDirect(,"update %IPM_Repo_Filesystem.Cache set ContentHash = '' where Root = ?",..Root)
+ if (tResult.%SQLCODE < 0) {
+ set tSC = $$$ERROR($$$SQLCode,tResult.%SQLCODE,tResult.%Message)
+ quit
+ }
}
}
@@ -147,106 +176,326 @@ Method BuildCache(
quit
}
- // Scan root directory recursively, up to ..Depth levels down, for module.xml files.
- set tSC = ..ScanDirectory(..Root,.tFilenameList,,..Depth,$select(pVerbose>1:1,1:0),.tMaxDepth)
+ // Read the rows already cached for this root up front, so an unchanged module.xml costs one
+ // hash comparison instead of an object instantiation.
+ $$$ThrowOnError(..LoadExistingEntries(.tExisting))
+
+ // Scan root directory recursively, up to ..Depth levels down, for module.xml files,
+ // hashing each one so unchanged manifests can be skipped without re-parsing.
+ set tDirList(1) = ..Root
+ set tSC = ##class(%IPM.Utils.File).WalkDirectories(.tDirList,..Root,..Depth,"module.xml",1,.tFilenameList,.tHashList)
if $$$ISERR(tSC) {
quit
}
- if (pAutoDetectDepth) && (tMaxDepth > 0) {
- set ..Depth = tMaxDepth
- set tSC = ..%Save()
- if $$$ISERR(tSC) {
- quit
- }
- }
+ // Track visited cache entries (for cleanup of stale entries)
+ kill tVisitedIds
// Ensure that we're looking at valid MODULE documents (as opposed to classes named Module, which the package manager has!)
set tAggSC = $$$OK
- set tKey = ""
+ set tRelPath = ""
for {
- set tKey = $order(tFilenameList(tKey),1,tFile)
- quit:(tKey="")
+ set tRelPath = $order(tFilenameList(tRelPath),1,tFile)
+ quit:(tRelPath="")
+
+ // Keys are relative to the root and always end in "module.xml"; dropping that and the
+ // separator before it leaves the module's subdirectory ("" for a module.xml in the root).
+ set tSubDirectory = $extract(tRelPath,1,*-$length("module.xml")-1)
+ set tAggSC = $$$ADDSC(tAggSC,..AddCacheItem(tFile,tSubDirectory,$get(tHashList(tRelPath)),.tName,.tVersionString,.tCacheId,.tExisting))
+
+ // Track this cache entry as visited (still exists on filesystem)
+ if tCacheId '= "" {
+ set tVisitedIds(tCacheId) = 1
+ }
- set tSubDirectory = tFilenameList(tKey,"sub")
- set tAggSC = $$$ADDSC(tAggSC,..AddCacheItem(tFile,tSubDirectory,.tName,.tVersionString))
write:pVerbose !,tName," ",tVersionString," @ ",##class(%File).NormalizeDirectory(..Root_tSubDirectory)
}
$$$ThrowOnError(tAggSC)
+
+ // Cleanup stale entries (files deleted from filesystem)
+ set tSC = ..CleanupStaleEntries(.tVisitedIds, pVerbose)
+ $$$ThrowOnError(tSC)
+
+ // set cache rebuild timestamp using UTC
+ set ..CacheLastRebuilt = $zdatetime($ztimestamp,3)
+ $$$ThrowOnError(..%Save())
tcommit
} catch e {
set tSC = e.AsStatus()
}
while ($tlevel > tInitTLevel) { trollback 1 }
+
quit tSC
}
+/// Read the cache rows already stored for this root, keyed by SubDirectoryHash so a long subdirectory
+/// cannot overflow a local array subscript.
+/// existing(hash) = $listbuild(id, contentHash, name, versionString)
+Method LoadExistingEntries(Output existing) As %Status [ Private ]
+{
+ set sc = $$$OK
+ kill existing
+ try {
+ #dim result As %SQL.StatementResult
+ set result = ##class(%SQL.Statement).%ExecDirect(,"SELECT ID, SubDirectory, SubDirectoryHash, ContentHash, Name, VersionString FROM %IPM_Repo_Filesystem.Cache WHERE Root = ?",..Root)
+ if (result.%SQLCODE < 0) {
+ $$$ThrowStatus($$$ERROR($$$SQLCode,result.%SQLCODE,result.%Message))
+ }
+ while result.%Next(.fetchSC) {
+ $$$ThrowOnError(fetchSC)
+ set hash = result.%Get("SubDirectoryHash")
+ // Rows carried over from a release without SubDirectoryHash have an empty stored value.
+ // Compute it from SubDirectory so the preload finds them correctly.
+ if hash = "" {
+ set hash = ##class(%IPM.Repo.Filesystem.Cache).PathHash(result.%Get("SubDirectory"))
+ }
+ set existing(hash) = $listbuild(result.%Get("ID"),result.%Get("ContentHash"),result.%Get("Name"),result.%Get("VersionString"))
+ }
+ $$$ThrowOnError(fetchSC)
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+/// Parse a module.xml file and return parsed metadata
+ClassMethod ParseModuleFile(
+ filePath As %String,
+ Output name As %String,
+ Output versionString As %String)
+{
+ set name = ""
+ set versionString = ""
+
+ // Validate file is a valid Studio document export
+ $$$ThrowOnError($system.OBJ.Load(filePath, "-d", , .loadedList, 1))
+
+ if ($length(loadedList, ",") > 1) {
+ $$$ThrowStatus($$$ERROR($$$GeneralError, "File contains multiple documents"))
+ }
+
+ set ext = $zconvert($piece($get(loadedList), ".", *), "U")
+ if (ext '= "ZPM") {
+ $$$ThrowStatus($$$ERROR($$$GeneralError, "File is not a ZPM module"))
+ }
+
+ // Parse module info (skip full manifest if metadata-only)
+ $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .versionString, 1))
+}
+
+/// Refresh a cache entry by re-parsing its module.xml file.
+/// Called when the content hash indicates the file has changed since it was last cached.
+/// Re-parses filePath into cacheObj and saves it, deferring the full manifest until
+/// GetModuleManifest() asks for it.
+/// A re-parse can change Name and VersionString. Callers must re-check that the entry still describes
+/// the module they looked it up by, as RootNameVersionOpenValidated does.
+/// Returns 0 if the re-parsed Name and Version already belong to another entry under this root.
+ClassMethod RefreshCacheEntry(
+ cacheObj As %IPM.Repo.Filesystem.Cache,
+ filePath As %String,
+ newHash As %String) As %Boolean
+{
+ // Parse the module file (metadata only, re-validate since file changed)
+ do ..ParseModuleFile(filePath, .name, .versionString)
+
+ // Update the cache entry
+ set cacheObj.Name = name
+ set cacheObj.VersionString = versionString
+ set cacheObj.ContentHash = newHash
+ // Drop the manifest along with the flag, so a reader that trusts the stream over ManifestLoaded
+ // can't get superseded bytes.
+ do cacheObj.Manifest.Clear()
+ set cacheObj.ManifestLoaded = 0
+ set cacheObj.LastModified = $zdatetime($ztimestamp, 3)
+
+ set sc = cacheObj.%Save()
+ if $$$ISERR(sc) {
+ // A duplicate Name and Version means another directory already claims this module, which
+ // HandleSaveError reports as a warning; anything else is a real failure and throws.
+ $$$ThrowOnError(cacheObj.HandleSaveError(sc))
+ return 0
+ }
+ return 1
+}
+
+/// Adds a module to the cache.
+/// pContentHash is the SHA-1 of pModuleFileName as computed by the directory walk; an entry whose
+/// stored hash still matches is reused as-is, without re-parsing or re-validating the manifest.
+/// pExisting is the array from LoadExistingEntries and is required: without it an
+/// already-cached directory takes the insert path and fails the CacheItemIndex unique key.
Method AddCacheItem(
pModuleFileName As %String,
pSubDirectory As %String,
+ pContentHash As %String,
Output pName As %String,
- Output pVersionString As %String) As %Status
+ Output pVersionString As %String,
+ Output pCacheId As %String = "",
+ ByRef pExisting) As %Status
{
set tSC = $$$OK
set pName = ""
set pVersionString = ""
+ set pCacheId = ""
try {
- // Get list of what's in module.xml
- set tSC = $system.OBJ.Load(pModuleFileName,"-d",,.tLoadedList,1)
- if $$$ISERR(tSC) {
- // Wasn't a valid file. We'll just continue.
- set tSC = $$$OK
- quit
- }
-
- if ($length(tLoadedList,",") > 1) {
- // Contained multiple documents - tricky! We'll just continue.
- quit
+ // The walk reports an empty hash for a file it could not read. Hash it here rather than
+ // treating the module as changed on every rebuild.
+ if (pContentHash = "") {
+ set pContentHash = $$$lcase(##class(%File).SHA1Hash(pModuleFileName, 1))
+ // Still empty means unreadable. An empty hash matches the empty hash of a previous failed
+ // attempt, so the entry would look unchanged and be served indefinitely without this.
+ if (pContentHash = "") {
+ do ##class(%IPM.General.LogManager).Warning("Could not read "_pModuleFileName_"; keeping the cached entry for it.",1)
+ }
}
- set tExt = $zconvert($piece($get(tLoadedList),".",*),"U")
- if (tExt '= "ZPM") {
- quit
- }
+ set tRow = $get(pExisting(##class(%IPM.Repo.Filesystem.Cache).PathHash(pSubDirectory)))
+ if (tRow '= "") {
+ set $listbuild(tRowId,tRowHash,tRowName,tRowVersion) = tRow
- kill tStream,tName,tVersionString
- set tParseSC = ..GetModuleStreamFromFile(pModuleFileName,.tStream,.pName,.pVersionString)
- if $$$ISERR(tParseSC) {
- // Log as a warning, but keep going.
- do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(tParseSC),1)
- quit
- }
+ if (tRowHash = pContentHash) {
+ // File unchanged, so reuse the cached values without opening the object or parsing.
+ set pName = tRowName
+ set pVersionString = tRowVersion
+ set pCacheId = tRowId
+ return $$$OK
+ }
- // Create cache item.
- if ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexExists(..Root,pSubDirectory) {
- set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).CacheItemIndexOpen(..Root,pSubDirectory,,.tSC)
+ set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).%OpenId(tRowId,,.tSC)
$$$ThrowOnError(tSC)
} else {
+ // New cache entry - need to create, parse and validate
set tCacheItem = ##class(%IPM.Repo.Filesystem.Cache).%New()
set tCacheItem.Root = ..Root
set tCacheItem.SubDirectory = pSubDirectory
}
+
+ // Parse the module file (metadata only, validate since file is new or changed)
+ try {
+ do ..ParseModuleFile(pModuleFileName, .pName, .pVersionString)
+ } catch ex {
+ // Wasn't a valid file or failed to parse. Log as warning and continue
+ do ##class(%IPM.General.LogManager).Warning("Failed to parse module manifest in "_pModuleFileName_": "_$system.Status.GetErrorText(ex.AsStatus()),1)
+
+ // Report an already-cached entry as still present. The file exists, it is only momentarily
+ // unparseable, say mid-save or with unresolved merge markers. Leaving it out of the
+ // caller's visited set would have CleanupStaleEntries evict a healthy entry.
+ if (tRow '= "") {
+ set pName = tRowName
+ set pVersionString = tRowVersion
+ set pCacheId = tRowId
+ }
+ return $$$OK
+ }
+
+ // Update cache item
set tCacheItem.Name = pName
set tCacheItem.VersionString = pVersionString
- do tCacheItem.Manifest.CopyFrom(tStream)
+ set tCacheItem.ContentHash = pContentHash
+ // Drop the manifest along with the flag, as in RefreshCacheEntry.
+ do tCacheItem.Manifest.Clear()
+ set tCacheItem.ManifestLoaded = 0
set tCacheItem.LastModified = $zdatetime($ztimestamp,3)
set tSaveSC = tCacheItem.%Save()
if $$$ISERR(tSaveSC) {
set tSC = tCacheItem.HandleSaveError(tSaveSC)
}
+
+ // Return cache ID for tracking
+ set pCacheId = tCacheItem.%Id()
} catch e {
set tSC = e.AsStatus()
}
quit tSC
}
-Method GetModuleStreamFromFile(
+/// Delete cache entries for files that no longer exist on filesystem
+Method CleanupStaleEntries(
+ ByRef visitedIds,
+ verbose As %Integer = 0) As %Status
+{
+ set sc = $$$OK
+ try {
+ // Query all cache entries for this root
+ set stmt = ##class(%SQL.Statement).%New()
+ set sc = stmt.%Prepare("SELECT ID, Name, VersionString, SubDirectory FROM %IPM_Repo_Filesystem.Cache WHERE Root = ?")
+ $$$ThrowOnError(sc)
+
+ set result = stmt.%Execute(..Root)
+ if (result.%SQLCODE < 0) {
+ set sc = $$$ERROR($$$SQLCode,result.%SQLCODE,result.%Message)
+ quit
+ }
+
+ // Collect before deleting, so no row is removed from under the open cursor.
+ set stale = 0
+ while result.%Next(.fetchSC) {
+ $$$ThrowOnError(fetchSC)
+ set cacheId = result.%Get("ID")
+ if '$data(visitedIds(cacheId)) {
+ set stale($increment(stale)) = $listbuild(cacheId,result.%Get("Name"),result.%Get("VersionString"),result.%Get("SubDirectory"))
+ }
+ }
+ $$$ThrowOnError(fetchSC)
+
+ set deletedCount = 0
+ for i = 1:1:stale {
+ set cacheId = $listget(stale(i),1)
+ set name = $listget(stale(i),2)
+ set version = $listget(stale(i),3)
+ set subDir = $listget(stale(i),4)
+
+ write:verbose !,"Removing stale cache entry: ",name," ",version," (file deleted from ",..Root,subDir,")"
+
+ set delSC = ##class(%IPM.Repo.Filesystem.Cache).%DeleteId(cacheId)
+ if $$$ISERR(delSC) {
+ set sc = $$$ADDSC(sc, delSC)
+ } else {
+ set deletedCount = deletedCount + 1
+ }
+ }
+
+ if (deletedCount > 0) && verbose {
+ write !,"Cleaned up ",deletedCount," stale cache entries"
+ }
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+/// Extracts the full XML section into the Manifest stream and sets ManifestLoaded.
+/// Deferred until a caller actually needs the manifest, so list, search and version resolution
+/// never pay for it.
+ClassMethod LoadManifestForCacheEntry(
+ cacheObj As %IPM.Repo.Filesystem.Cache,
+ filePath As %String) As %Status
+{
+ set sc = $$$OK
+ try {
+ // Extract full manifest using GetModuleStreamFromFile (not metadata-only)
+ $$$ThrowOnError(..GetModuleStreamFromFile(filePath, .stream, .name, .version, 0))
+
+ // Update cache entry with manifest
+ do cacheObj.Manifest.Clear()
+ do cacheObj.Manifest.CopyFrom(stream)
+ set cacheObj.ManifestLoaded = 1
+ set cacheObj.LastModified = $zdatetime($ztimestamp, 3)
+ $$$ThrowOnError(cacheObj.%Save())
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+ClassMethod GetModuleStreamFromFile(
pFilename As %String,
Output pStream As %Stream.GlobalCharacter,
Output pName As %String,
- Output pVersion As %String) As %Status
+ Output pVersion As %String,
+ pMetadataOnly As %Boolean = 0) As %Status
{
set tSC = $$$OK
+ set pStream = ""
set pName = ""
set pVersion = ""
try {
@@ -255,18 +504,30 @@ Method GetModuleStreamFromFile(
set tSC = tSourceStream.LinkToFile(pFilename)
$$$ThrowOnError(tSC)
- set pStream = ##class(%Stream.GlobalCharacter).%New()
set tMetaStream = ##class(%Stream.GlobalCharacter).%New()
- // Extract section of document
- set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"ModuleDocumentTransform")
- set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tSourceStream,tCompiledTransform,.pStream)
- $$$ThrowOnError(tSC)
+ // Extract section of document (skip if metadata-only)
+ if 'pMetadataOnly {
+ // Two transforms mean two passes, so buffer the source in memory to avoid re-reading the
+ // file. A metadata-only call makes one pass and reads the file directly.
+ set tInputStream = ##class(%Stream.GlobalCharacter).%New()
+ set tSC = tInputStream.CopyFrom(tSourceStream)
+ $$$ThrowOnError(tSC)
+ do tInputStream.Rewind()
+
+ set pStream = ##class(%Stream.GlobalCharacter).%New()
+ set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"ModuleDocumentTransform")
+ set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tInputStream,tCompiledTransform,.pStream)
+ $$$ThrowOnError(tSC)
+
+ do tInputStream.Rewind()
+ } else {
+ set tInputStream = tSourceStream
+ }
// Extract Name and Version
- do tSourceStream.Rewind()
set tCompiledTransform = ##class(%IPM.Repo.XSLTProvider).GetCompiledTransformForXData($classname(),"MetadataExtractionTransform")
- set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tSourceStream,tCompiledTransform,.tMetaStream)
+ set tSC = ##class(%XML.XSLT.Transformer).TransformStreamWithCompiledXSL(tInputStream,tCompiledTransform,.tMetaStream)
$$$ThrowOnError(tSC)
set tMetaStream.LineTerminator = $select($$$isWINDOWS:$char(13,10),$$$isUNIX:$char(10))
@@ -309,54 +570,6 @@ XData MetadataExtractionTransform
}
-ClassMethod ScanDirectory(
- pRoot As %String,
- ByRef pFilenameList,
- pSub As %String = "",
- pDepth As %Integer = "",
- pVerbose As %Boolean = 0,
- Output pMaxDepth As %Integer = 0) As %Status [ Internal ]
-{
- set tSC = $$$OK
- try {
- set pRoot = ##class(%File).NormalizeDirectory(pRoot)
- set tDirArray($increment(tDirArray)) = ""
-
- set i = 0
- do {
- set i = i + 1
- if (pDepth > 0) && ($length(tDirArray(i),"/") > pDepth) {
- continue
- }
-
- set tStmt = ##class(%SQL.Statement).%New()
- set tSC = tStmt.%PrepareClassQuery("%Library.File","FileSet")
- if $$$ISERR(tSC) {
- quit
- }
-
- set tFullDir = ##class(%File).NormalizeDirectory(pRoot_tDirArray(i))
- set tRes = tStmt.%Execute(tFullDir,"module.xml",,1)
- while tRes.%Next() {
- if (tRes.%Get("Type") = "D") {
- write:pVerbose !,"Scanning directory: ",tRes.%Get("ItemName")
- set tDirArray($increment(tDirArray)) = tDirArray(i)_$case(tDirArray(i),"":"",:"/")_tRes.%Get("ItemName")
- } else {
- write:pVerbose !,"Found file: ",tRes.%Get("Name")
- set pFilenameList($increment(pFilenameList)) = tRes.%Get("Name")
- set pFilenameList(pFilenameList,"sub") = tDirArray(i)
- if (pDepth > pMaxDepth) {
- set pMaxDepth = $length(tDirArray(i),"/")
- }
- }
- }
- } while (i < tDirArray)
- } catch e {
- set tSC = e.AsStatus()
- }
- quit tSC
-}
-
XData LockFileMapping
{
@@ -378,6 +591,9 @@ Storage Default
Depth
+
+CacheLastRebuilt
+
FilesystemRepoDefinitionDefaultData
%Storage.Persistent
diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls
index f2476ae9e..bcd09509b 100644
--- a/src/cls/IPM/Repo/Filesystem/PackageService.cls
+++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls
@@ -24,9 +24,22 @@ Method HasModule(pModuleReference As %IPM.Storage.ModuleInfo) As %Boolean
Method GetModuleManifest(pModuleReference As %IPM.Storage.ModuleInfo) As %Stream.Object
{
- set tModule = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..Root,pModuleReference.Name,pModuleReference.VersionString,,.tStatus)
- $$$ThrowOnError(tStatus)
- quit tModule.Manifest
+ // Use validated open to ensure cache is fresh
+ set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status)
+ $$$ThrowOnError(status)
+
+ // Lazy load manifest if not already populated (ManifestLoaded=0)
+ // This defers expensive XML parsing until manifest is actually needed
+ if 'module.ManifestLoaded {
+ // Reconstruct file path
+ set dirPath = ##class(%File).NormalizeFilename(module.SubDirectory, module.Root)
+ set filePath = ##class(%File).NormalizeFilename("module.xml", dirPath)
+
+ set status = ##class(%IPM.Repo.Filesystem.Definition).LoadManifestForCacheEntry(module, filePath)
+ $$$ThrowOnError(status)
+ }
+
+ quit module.Manifest
}
Method GetModule(
@@ -47,11 +60,11 @@ Method GetModule(
Method GetModuleDirectory(pModuleReference As %IPM.Storage.ModuleInfo) As %String
{
- // Get the module ...
- set tModule = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..Root,pModuleReference.Name,pModuleReference.VersionString,,.tStatus)
- $$$ThrowOnError(tStatus)
+ // Use validated open to ensure cache is fresh
+ set module = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..Root, pModuleReference.Name, pModuleReference.VersionString, , .status)
+ $$$ThrowOnError(status)
- quit ##class(%File).NormalizeDirectory(tModule.Root_tModule.SubDirectory)
+ quit ##class(%File).NormalizeDirectory(module.Root_module.SubDirectory)
}
/// Returns 1 if the service supports a particular method.
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index 97d2a3363..746695b99 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -523,7 +523,7 @@ Method OnItemRelativePath(pItemName As %String) As %String
Method GetSyncDirectory() As %String
{
if $extract(..ResourceReference.Name) = "/" {
- set dir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name)
+ set dir = ##class(%IPM.Utils.File).NormalizePath(..ResourceReference.Name)
if $extract(dir, *) = "/" {
set dir = $extract(dir, 1, *-1)
}
@@ -538,9 +538,9 @@ Method GetSyncDirectory() As %String
set prefix = ..ResourceReference.Module.SourcesRoot
}
if prefix '= "" {
- set dir = ##class(%IPM.Storage.FileHash).NormalizePath(prefix _ "/" _ dir)
+ set dir = ##class(%IPM.Utils.File).NormalizePath(prefix _ "/" _ dir)
} else {
- set dir = ##class(%IPM.Storage.FileHash).NormalizePath(dir)
+ set dir = ##class(%IPM.Utils.File).NormalizePath(dir)
}
}
quit dir
diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls
index a254e6667..260e12628 100644
--- a/src/cls/IPM/ResourceProcessor/Test.cls
+++ b/src/cls/IPM/ResourceProcessor/Test.cls
@@ -384,7 +384,7 @@ Method SupportsSync() As %Boolean
/// Sync walks this directory directly so new test classes are detected before compilation.
Method GetSyncDirectory() As %String
{
- quit ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name)
+ quit ##class(%IPM.Utils.File).NormalizePath(..ResourceReference.Name)
}
/// Returns 1 if className falls within this resource's declared Package (or is its
@@ -424,7 +424,7 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand
// Record changed TestCase subclasses for SyncRunTests.
// relPath is relative to module root (e.g. "tests/unit/SyncTest/Tests/Trivial.cls").
- set resourceDir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name)
+ set resourceDir = ##class(%IPM.Utils.File).NormalizePath(..ResourceReference.Name)
set relPath = ""
for {
set relPath = $order(modifiedPaths(relPath))
diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls
index 3779809b3..464345b9a 100644
--- a/src/cls/IPM/Storage/FileHash.cls
+++ b/src/cls/IPM/Storage/FileHash.cls
@@ -4,7 +4,7 @@ Class %IPM.Storage.FileHash Extends %Persistent
Property ModuleName As %String(MAXLEN = 255) [ Required ];
/// Path relative to the module root, normalized: forward slashes, no leading slash, no double slashes.
-/// Callers must normalize via NormalizePath before storing or looking up.
+/// Callers must normalize via %IPM.Utils.File:NormalizePath before storing or looking up.
Property RelativePath As %String(MAXLEN = "") [ Required ];
/// SHA-1 of RelativePath (hex), maintained by RelativePathSet. Subscripting ModulePathIndex on
@@ -524,38 +524,6 @@ ClassMethod GetStoredPaths(moduleName As %String, Output paths)
}
}
-/// Normalize a relative path: forward slashes, collapse //, drop "." segments, strip leading slash.
-///
-/// Collapsing "." segments is what makes . work: such a module composes
-/// every resource path as SourcesRoot_"/"_Directory (see %IPM.ResourceProcessor.Default.Document and
-/// .Package OnItemRelativePath / GetSyncDirectory), so its paths arrive here as "./cls/Foo/Bar.cls"
-/// while the directory walk and the stored baseline key the same file as "cls/Foo/Bar.cls". Both
-/// forms reduce to one here so the sync reverse index matches changed paths under such a resource.
-ClassMethod NormalizePath(path As %String) As %String
-{
- set path = $translate(path, "\", "/")
- while path [ "//" {
- set path = $replace(path, "//", "/")
- }
- while path [ "/./" {
- set path = $replace(path, "/./", "/")
- }
- while $extract(path) = "/" {
- set path = $extract(path, 2, *)
- }
- // A leading "./" survives the loop above (it has no slash in front of it), and repeats when
- // SourcesRoot is something like "./." — so strip until none is left.
- while $extract(path, 1, 2) = "./" {
- set path = $extract(path, 3, *)
- }
- // SourcesRoot="." with no Directory reduces the whole path to "."; that is the module root, not
- // a file, so it normalizes to the empty relative path.
- if path = "." {
- set path = ""
- }
- quit path
-}
-
/// Canonical form of a server document name for use as an array subscript: the name as given, with
/// the extension upper-cased.
///
@@ -576,8 +544,9 @@ ClassMethod CanonicalDocName(docName As %String) As %String
}
/// Walk each directory in scanDirs(relDir)="" under moduleRoot and hash all files.
-/// Returns files(relPath)=fullPath and hashes(relPath)=sha1hex (lowercase).
-/// Uses a single Python os.walk call across all directories for speed; falls back to SQL BFS.
+/// Returns files(relPath)=fullPath and hashes(relPath)=sha1hex (lowercase), keyed relative to
+/// moduleRoot. Resolves the module-relative scan dirs to absolute existing directories and defers
+/// the walk itself to %IPM.Utils.File, which the filesystem repository cache scan shares.
ClassMethod WalkAndHashDirs(moduleRoot As %String, ByRef scanDirs, Output files, Output hashes) As %Status
{
set sc = $$$OK
@@ -586,7 +555,6 @@ ClassMethod WalkAndHashDirs(moduleRoot As %String, ByRef scanDirs, Output files,
set moduleRoot = ##class(%File).NormalizeDirectory(moduleRoot)
// Collect existing absolute directories to walk.
- kill absDirList
set dirCount = 0
set relDir = ""
for {
@@ -600,25 +568,8 @@ ClassMethod WalkAndHashDirs(moduleRoot As %String, ByRef scanDirs, Output files,
}
quit:dirCount=0
- // Single Python call across all directories — avoids per-dir interpreter overhead.
- set dirsJson = "["
- for i = 1:1:dirCount {
- if i > 1 { set dirsJson = dirsJson _ "," }
- set dirsJson = dirsJson _ """" _ $replace(absDirList(i), "\", "\\") _ """"
- }
- set dirsJson = dirsJson _ "]"
-
- set walkSC = ..WalkAndHashFilesPython(dirsJson, moduleRoot, .files, .hashes)
- if $$$ISERR(walkSC) {
- // SQL fallback: walk each directory individually.
- kill files, hashes
- for i = 1:1:dirCount {
- kill dirFiles, dirHashes
- $$$ThrowOnError(..WalkAndHashFilesSQL(absDirList(i), moduleRoot, .dirFiles, .dirHashes))
- merge files = dirFiles
- merge hashes = dirHashes
- }
- }
+ // No depth cap and no filename filter: sync tracks every file under a declared directory.
+ set sc = ##class(%IPM.Utils.File).WalkDirectories(.absDirList, moduleRoot, 0, "", 1, .files, .hashes)
} catch e {
set sc = e.AsStatus()
}
@@ -639,7 +590,7 @@ ClassMethod DeduplicateScanDirs(ByRef scanDirs)
for {
set keptDir = $order(kept(keptDir))
quit:keptDir=""
- set keptPrefix = ..NormalizePath(keptDir _ "/")
+ set keptPrefix = ##class(%IPM.Utils.File).NormalizePath(keptDir _ "/")
if $extract(relDir, 1, $length(keptPrefix)) = keptPrefix {
set covered = 1
quit
@@ -653,112 +604,6 @@ ClassMethod DeduplicateScanDirs(ByRef scanDirs)
merge scanDirs = kept
}
-ClassMethod WalkAndHashFilesPython(dirsJson As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ]
-{
- set sc = $$$OK
- try {
- set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot)
- set jsonStr = ..WalkAndHashFilesPythonImpl(dirsJson, relativeToRoot)
- set result = ##class(%DynamicArray).%FromJSON(jsonStr)
- set count = result.%Size()
- for i = 0:1:(count - 1) {
- set entry = result.%Get(i)
- set relPath = entry.%Get("rel")
- set files(relPath) = entry.%Get("full")
- set hashes(relPath) = entry.%Get("hash")
- }
- } catch e {
- set sc = e.AsStatus()
- }
- quit sc
-}
-
-ClassMethod WalkAndHashFilesPythonImpl(dirsJson As %String, relativeToRoot As %String) As %String [ Language = python ]
-{
-import os
-import json
-import hashlib
-import concurrent.futures
-
-SKIP_DIRS = {'.git', '__pycache__', 'node_modules'}
-
-dirs = json.loads(dirsJson)
-root_len = len(relativeToRoot.rstrip(os.sep)) + 1
-
-# Walk all directories, collecting all files.
-file_list = []
-for d in dirs:
- for dirpath, dirnames, filenames in os.walk(d):
- dirnames[:] = [x for x in dirnames if x not in SKIP_DIRS]
- for fname in filenames:
- full_path = os.path.join(dirpath, fname)
- rel_path = full_path[root_len:].replace('\\', '/')
- if rel_path:
- file_list.append((rel_path, full_path))
-
-def hash_file(full_path):
- try:
- h = hashlib.sha1()
- with open(full_path, 'rb') as f:
- while chunk := f.read(65536):
- h.update(chunk)
- return h.hexdigest()
- except (OSError, PermissionError):
- return ""
-
-# Hash all files in parallel — SHA1 releases the GIL, so threads overlap I/O wait.
-with concurrent.futures.ThreadPoolExecutor(max_workers=24) as executor:
- hash_results = list(executor.map(hash_file, [f for _, f in file_list]))
-
-return json.dumps([
- {"rel": rel, "full": full, "hash": h}
- for (rel, full), h in zip(file_list, hash_results)
-])
-}
-
-/// SQL-based BFS fallback that walks and hashes without Python.
-ClassMethod WalkAndHashFilesSQL(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ]
-{
- set sc = $$$OK
- try {
- set dir = ##class(%File).NormalizeDirectory(dir)
- set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot)
- kill walkQueue
- set walkHead = 1, walkTail = 1
- set walkQueue(walkTail) = dir
- for {
- quit:(walkHead > walkTail)
- set walkDir = walkQueue(walkHead)
- set walkHead = walkHead + 1
-
- set rs = ##class(%SQL.Statement).%ExecDirect(,
- "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)",
- walkDir, "*", "", 0)
- while rs.%Next() {
- set entryType = rs.%Get("Type")
- set entryPath = rs.%Get("Name")
- if entryType = "D" {
- set childDirName = ##class(%File).GetFilename(entryPath)
- if ",.git,__pycache__,node_modules," [ (","_childDirName_",") {
- continue
- }
- set walkTail = walkTail + 1
- set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath)
- continue
- }
- set relPath = ..NormalizePath($extract(entryPath, $length(relativeToRoot) + 1, *))
- if relPath '= "" {
- set files(relPath) = entryPath
- set hashes(relPath) = $$$lcase(##class(%File).SHA1Hash(entryPath, 1))
- }
- }
- }
- } catch e {
- set sc = e.AsStatus()
- }
- quit sc
-}
-
Storage Default
{
diff --git a/src/cls/IPM/Utils/File.cls b/src/cls/IPM/Utils/File.cls
index ae976f70a..29b6f055a 100644
--- a/src/cls/IPM/Utils/File.cls
+++ b/src/cls/IPM/Utils/File.cls
@@ -2,6 +2,9 @@
Class %IPM.Utils.File
{
+/// Ceiling on how far WalkDirectoriesSQL will descend, well past any real source tree.
+Parameter MAXWALKDEPTH = 64;
+
/// Create this directory and all the parent directories if they do not exist. This differs from
/// CreateDirectory as that method only creates one new directory where as
/// this will create the entire chain of directories. Returns true if it succeeds and false otherwise.
@@ -241,6 +244,300 @@ ClassMethod Exists(
return 0
}
+/// Normalize a relative path: forward slashes, collapse //, drop "." segments, strip leading slash.
+///
+/// Dot collapsing is what makes . work. Such a module composes resource
+/// paths as SourcesRoot_"/"_Directory, so a file arrives as "./cls/Foo.cls" while the directory walk
+/// keys it as "cls/Foo.cls". Both forms reduce to one here so the sync reverse index matches.
+ClassMethod NormalizePath(path As %String) As %String
+{
+ set path = $translate(path, "\", "/")
+ while path [ "//" {
+ set path = $replace(path, "//", "/")
+ }
+ while path [ "/./" {
+ set path = $replace(path, "/./", "/")
+ }
+ while $extract(path) = "/" {
+ set path = $extract(path, 2, *)
+ }
+ // A leading "./" has no slash in front of it, so the loop above misses it. Repeats for "./.".
+ while $extract(path, 1, 2) = "./" {
+ set path = $extract(path, 3, *)
+ }
+ // A bare "." is the module root, not a file.
+ if path = "." {
+ set path = ""
+ }
+ quit path
+}
+
+/// Walk one or more directories and report the files found, optionally with content hashes.
+/// Shared by sync's change detection and the filesystem repository cache scan.
+/// dirs(n) = absolute directory to walk; callers pass only directories that exist.
+/// relativeTo is the root that result keys are made relative to.
+/// maxDepth limits how far below each walked directory to descend; 0 is unlimited.
+/// A file exactly maxDepth levels down is kept; anything deeper is not visited.
+/// filenameFilter, when non-empty, keeps only files with that name (e.g. "module.xml"),
+/// compared case-insensitively on Windows and case-sensitively elsewhere, matching the filesystem.
+/// files(relPath) = fullPath, and hashes(relPath) = lowercase SHA-1 hex
+/// (empty when computeHashes is 0). Keys are normalized: forward slashes, no leading slash.
+/// Directories named .git, __pycache__ or node_modules are never descended into.
+/// Symlinked directories are skipped by the embedded Python walk but followed by the SQL fallback,
+/// since no %Library.File API distinguishes a symlink from a directory.
+ClassMethod WalkDirectories(
+ ByRef dirs,
+ relativeTo As %String,
+ maxDepth As %Integer = 0,
+ filenameFilter As %String = "",
+ computeHashes As %Boolean = 1,
+ Output files,
+ Output hashes) As %Status
+{
+ set sc = $$$OK
+ kill files, hashes
+ try {
+ set relativeTo = ##class(%Library.File).NormalizeDirectory(relativeTo)
+
+ set dirCount = 0
+ set key = ""
+ for {
+ set key = $order(dirs(key),1,dir)
+ quit:key=""
+ set dirCount = dirCount + 1
+ set dirList(dirCount) = ##class(%Library.File).NormalizeDirectory(dir)
+ }
+ quit:dirCount=0
+
+ // A single Python call across all directories avoids per-directory interpreter overhead.
+ set dirsJson = "["
+ for i = 1:1:dirCount {
+ set dirsJson = dirsJson _ $select(i>1:",",1:"") _ """" _ $replace(dirList(i), "\", "\\") _ """"
+ }
+ set dirsJson = dirsJson _ "]"
+
+ set pythonSC = ..WalkDirectoriesPython(dirsJson, relativeTo, maxDepth, filenameFilter, computeHashes, .files, .hashes)
+ if $$$ISOK(pythonSC) {
+ quit
+ }
+
+ // Fall back to a SQL-driven breadth-first walk, one directory at a time.
+ // Warn, because the fallback is far slower and otherwise indistinguishable from a slow disk.
+ set warning = "Falling back to a slower SQL directory walk. The embedded Python walk of "_relativeTo_" could not be completed or read back: "_$system.Status.GetOneErrorText(pythonSC)
+ do ##class(%IPM.General.LogManager).Warning(warning,1)
+
+ kill files, hashes
+ for i = 1:1:dirCount {
+ $$$ThrowOnError(..WalkDirectoriesSQL(dirList(i), relativeTo, maxDepth, filenameFilter, computeHashes, .files, .hashes))
+ }
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+ClassMethod WalkDirectoriesPython(
+ dirsJson As %String,
+ relativeTo As %String,
+ maxDepth As %Integer,
+ filenameFilter As %String,
+ computeHashes As %Boolean,
+ Output files,
+ Output hashes) As %Status [ Private ]
+{
+ set sc = $$$OK
+ try {
+ // One JSON object per line in a stream rather than one JSON document in a string. A single
+ // string caps out around 3.6MB, which a large source tree exceeds.
+ set outputStream = ##class(%Stream.GlobalCharacter).%New()
+ // Python writes "\n"; the default terminator is CRLF, which would make ReadLine() return the
+ // whole stream as a single line.
+ set outputStream.LineTerminator = $char(10)
+ set expected = ..WalkDirectoriesPythonImpl(dirsJson, relativeTo, maxDepth, filenameFilter, computeHashes, outputStream)
+
+ do outputStream.Rewind()
+ set parsed = 0
+ while 'outputStream.AtEnd {
+ set line = outputStream.ReadLine()
+ continue:line=""
+ set entry = ##class(%DynamicObject).%FromJSON(line)
+ set relPath = entry.%Get("rel")
+ set files(relPath) = entry.%Get("full")
+ set hashes(relPath) = entry.%Get("hash")
+ set parsed = parsed + 1
+ }
+ // Python cannot check the %Status of its stream writes, so verify the count instead. A short
+ // stream returns an error, which sends the caller to the SQL walk rather than a partial list.
+ if (parsed '= expected) {
+ $$$ThrowStatus($$$ERROR($$$GeneralError,"Python directory walk found "_expected_" files but only "_parsed_" were readable from the stream"))
+ }
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+/// os.walk() is C-based and far faster than a SQL FileSet walk; hashing runs on a thread pool
+/// because SHA-1 releases the GIL, so threads overlap I/O wait.
+/// Writes one JSON object per line to outputStream and returns how many were written.
+ClassMethod WalkDirectoriesPythonImpl(
+ dirsJson As %String,
+ relativeTo As %String,
+ maxDepth As %Integer,
+ filenameFilter As %String,
+ computeHashes As %Boolean,
+ outputStream As %Stream.GlobalCharacter) As %Integer [ Language = python, Private ]
+{
+import os
+import json
+import hashlib
+import concurrent.futures
+
+SKIP_DIRS = {'.git', '__pycache__', 'node_modules'}
+
+dirs = json.loads(dirsJson)
+max_depth = int(maxDepth)
+compute_hashes = int(computeHashes)
+root_len = len(relativeTo.rstrip(os.sep)) + 1
+
+# Windows filenames are case-insensitive, so "Module.xml" must match a "module.xml" filter there.
+# Elsewhere those are two distinct files, so the comparison stays exact.
+fold_case = os.name == 'nt'
+name_filter = filenameFilter.lower() if fold_case else filenameFilter
+
+file_list = []
+
+for d in dirs:
+ dir_len = len(d.rstrip(os.sep))
+ for dirpath, dirnames, filenames in os.walk(d):
+ dirnames[:] = [x for x in dirnames if x not in SKIP_DIRS]
+
+ # Depth of dirpath below the walked directory; 0 for the directory itself. Recomputed each
+ # iteration because os.walk() yields absolute paths rather than incremental navigation.
+ rel_dir = dirpath[dir_len:].lstrip(os.sep)
+ current_depth = rel_dir.count(os.sep) + 1 if rel_dir else 0
+
+ # Stop descending once the cap is reached; files at exactly the cap are still kept.
+ if max_depth > 0 and current_depth >= max_depth:
+ dirnames[:] = []
+
+ for fname in filenames:
+ if name_filter and (fname.lower() if fold_case else fname) != name_filter:
+ continue
+ full_path = os.path.join(dirpath, fname)
+ rel_path = full_path[root_len:].replace('\\', '/')
+ if not rel_path:
+ continue
+ file_list.append((rel_path, full_path))
+
+def hash_file(full_path):
+ try:
+ h = hashlib.sha1()
+ with open(full_path, 'rb') as f:
+ while chunk := f.read(65536):
+ h.update(chunk)
+ return h.hexdigest()
+ except (OSError, PermissionError):
+ return ""
+
+if compute_hashes:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=24) as executor:
+ hash_results = list(executor.map(hash_file, [f for _, f in file_list]))
+else:
+ hash_results = [""] * len(file_list)
+
+# Buffer into ~1MB chunks: one Write() per file would cross into IRIS more times than the walk
+# itself costs. json.dumps() escapes newlines, so a record can never span lines.
+buffer = []
+buffer_len = 0
+for (rel, full), h in zip(file_list, hash_results):
+ record = json.dumps({"rel": rel, "full": full, "hash": h})
+ buffer.append(record)
+ buffer_len += len(record) + 1
+ if buffer_len >= 1000000:
+ outputStream.Write("\n".join(buffer) + "\n")
+ buffer = []
+ buffer_len = 0
+if buffer:
+ outputStream.Write("\n".join(buffer) + "\n")
+
+return len(file_list)
+}
+
+/// Breadth-first walk of a single directory using %Library.File_FileSet, for use when embedded
+/// Python is unavailable. Adds to the caller's files/hashes rather than resetting them.
+ClassMethod WalkDirectoriesSQL(
+ dir As %String,
+ relativeTo As %String,
+ maxDepth As %Integer,
+ filenameFilter As %String,
+ computeHashes As %Boolean,
+ ByRef files,
+ ByRef hashes) As %Status [ Internal ]
+{
+ set sc = $$$OK
+ try {
+ // Windows matches filenames case-insensitively; other platforms do not. See WalkDirectories.
+ set foldCase = $$$isWINDOWS
+ set nameFilter = $select(foldCase:$$$lcase(filenameFilter),1:filenameFilter)
+
+ set walkHead = 1, walkTail = 1
+ set walkQueue(1) = ##class(%Library.File).NormalizeDirectory(dir)
+ set walkQueue(1,"depth") = 0
+ for {
+ quit:walkHead>walkTail
+ set walkDir = walkQueue(walkHead)
+ set walkDepth = walkQueue(walkHead,"depth")
+ set walkHead = walkHead + 1
+
+ // FileSet cannot tell a symlink from a directory, so an unlimited walk through a symlink
+ // loop would descend forever, each pass yielding a longer but distinct path. The ceiling is
+ // set far above any real source tree, so a legitimate walk never reaches it.
+ if (walkDepth > ..#MAXWALKDEPTH) {
+ $$$ThrowStatus($$$ERROR($$$GeneralError,"Directory walk exceeded "_..#MAXWALKDEPTH_" levels below "_dir_"; check for a symbolic link cycle"))
+ }
+
+ // The 4th argument is includedirs. Without it there are no Type = "D" rows to descend into.
+ set rs = ##class(%SQL.Statement).%ExecDirect(,
+ "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)",
+ walkDir, "*", "", 1)
+ if rs.%SQLCODE < 0 {
+ $$$ThrowStatus($$$ERROR($$$SQLCode,rs.%SQLCODE,rs.%Message))
+ }
+ while rs.%Next() {
+ set entryPath = rs.%Get("Name")
+ if rs.%Get("Type") = "D" {
+ if (maxDepth > 0) && ((walkDepth + 1) > maxDepth) {
+ continue
+ }
+ if ",.git,__pycache__,node_modules," [ (","_##class(%Library.File).GetFilename(entryPath)_",") {
+ continue
+ }
+ set walkTail = walkTail + 1
+ set walkQueue(walkTail) = ##class(%Library.File).NormalizeDirectory(entryPath)
+ set walkQueue(walkTail,"depth") = walkDepth + 1
+ continue
+ }
+ if (nameFilter '= "") {
+ set entryName = ##class(%Library.File).GetFilename(entryPath)
+ if $select(foldCase:$$$lcase(entryName),1:entryName) '= nameFilter {
+ continue
+ }
+ }
+ set relPath = ..NormalizePath($extract(entryPath, $length(relativeTo) + 1, *))
+ if (relPath = "") {
+ continue
+ }
+ set files(relPath) = entryPath
+ set hashes(relPath) = $select(computeHashes:$$$lcase(##class(%Library.File).SHA1Hash(entryPath, 1)),1:"")
+ }
+ }
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
/// Creates a new temporary directory and returns its normalized path.
/// baseDirectory optionally specifies the parent; defaults to the OS temp directory.
/// $zu(140,17) returns a negative integer error code on failure (e.g., -28 = no space left on device).
diff --git a/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls
new file mode 100644
index 000000000..8ff230f86
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/FilesystemRepo.cls
@@ -0,0 +1,528 @@
+Class Test.PM.Integration.FilesystemRepo Extends Test.PM.Integration.Base
+{
+
+Parameter REPONAME = "fs-cache-test";
+
+/// Working copy of the _data/fs-cache-test fixture. Several tests edit module.xml in place, so they
+/// run against a copy rather than the git-tracked fixture on the bind mount.
+Property RepoPath As %String;
+
+/// Native-filesystem copy of the fixture, made once, and the source every restore copies from.
+Property PristineDir As %String;
+
+XData ModuleB300
+{
+
+
+
+
+ module-b
+ 3.0.0
+ module
+
+
+
+}
+
+Method OnBeforeAllTests() As %Status
+{
+ set ..PristineDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory()_"fs-cache-test-pristine-"_$job)
+ $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..PristineDir))
+ if '##class(%File).CopyDir(..GetModuleDir("fs-cache-test"),..PristineDir,1) {
+ quit $$$ERROR($$$GeneralError,"Failed to copy the fs-cache-test fixture to the pristine directory")
+ }
+
+ // The repo definition stores Root, so this path has to stay the same for every test.
+ set ..RepoPath = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory()_"fs-cache-test-"_$job)
+ $$$ThrowOnError(..RestoreFixture())
+
+ set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -fs -path "_..RepoPath)
+ do $$$AssertStatusOK(sc,"Created fs-cache-test repo successfully.")
+ quit sc
+}
+
+Method OnAfterAllTests() As %Status
+{
+ // Remove test repository
+ set sc = ##class(%IPM.Main).Shell("repo -delete -name "_..#REPONAME)
+ do $$$AssertStatusOK(sc,"Deleted fs-cache-test repo successfully.")
+ do ##class(%File).RemoveDirectoryTree(..RepoPath)
+ do ##class(%File).RemoveDirectoryTree(..PristineDir)
+ quit sc
+}
+
+Method OnAfterOneTest(testName As %String) As %Status
+{
+ // Uninstall all modules after each test for clean state
+ set sc = ##class(%IPM.Main).Shell("uninstall -all")
+ do $$$AssertStatusOK(sc,"Uninstalled all modules after test.")
+
+ // Tests that edit or add module.xml files leave the working copy dirty, so it is rebuilt from
+ // the pristine copy and the cache purged to match.
+ do $$$AssertStatusOK(..RestoreFixture(),"Restored the fixture after "_testName)
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.openSC)
+ if $$$ISOK(openSC) {
+ do $$$AssertStatusOK(repoDef.BuildCache(1,0),"Purged and rebuilt the cache after "_testName)
+ }
+ quit sc
+}
+
+/// Replace the working copy of the fixture with a fresh copy of the pristine one.
+Method RestoreFixture() As %Status [ Private ]
+{
+ do ##class(%File).RemoveDirectoryTree(..RepoPath)
+ $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..RepoPath))
+ if '##class(%File).CopyDir(..PristineDir,..RepoPath,1) {
+ quit $$$ERROR($$$GeneralError,"Failed to restore the fs-cache-test fixture")
+ }
+ quit $$$OK
+}
+
+/// Test that cache is built correctly when repo is created
+Method TestCacheBuiltCorrectly()
+{
+ // Get the repo definition
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition")
+ do $$$AssertTrue($isobject(repoDef),"Repo definition exists")
+
+ // Verify cache entries exist for both modules
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry for module-a 1.0.0 exists")
+ do $$$AssertTrue($isobject(cacheA),"module-a 1.0.0 found in cache")
+
+ // Verify initial cache state - manifest not loaded, but validation passed during cache build
+ // ManifestLoaded=0 means only metadata (name/version) was extracted, not full manifest (lazy loading)
+ do $$$AssertEquals(cacheA.ManifestLoaded,0,"ManifestLoaded is initially 0 (lazy loading)")
+
+ set cacheB1 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry for module-b 1.0.0 exists")
+ do $$$AssertTrue($isobject(cacheB1),"module-b 1.0.0 found in cache")
+
+ set cacheB2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","2.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry for module-b 2.0.0 exists")
+ do $$$AssertTrue($isobject(cacheB2),"module-b 2.0.0 found in cache")
+
+ // Verify CacheLastRebuilt is set
+ do $$$AssertNotEquals(repoDef.CacheLastRebuilt,"","CacheLastRebuilt timestamp is set")
+
+ // Test installing a module to verify cache works
+ set sc = ##class(%IPM.Main).Shell("install module-a 1.0.0")
+ do $$$AssertStatusOK(sc,"Successfully installed module-a from cache")
+}
+
+/// Test lazy loading - manifests are not fully loaded until accessed
+Method TestLazyLoadingVerification()
+{
+ // Rebuild cache to ensure clean state with ManifestLoaded=0
+ set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache")
+ do $$$AssertStatusOK(sc,"Rebuilt cache for clean state")
+
+ // Get cache entry
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Opened cache entry for module-a")
+
+ // Verify ManifestLoaded flag is initially false (lazy loading optimization)
+ do $$$AssertEquals(cacheA.ManifestLoaded,0,"ManifestLoaded flag is initially false")
+
+ // Accessing basic metadata should work without loading full manifest
+ do $$$AssertEquals(cacheA.Name,"module-a","Can access Name without full manifest load")
+ do $$$AssertEquals(cacheA.VersionString,"1.0.0","Can access Version without full manifest load")
+
+ // Now trigger full manifest load by accessing the manifest
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition")
+
+ set packageService = repoDef.GetPackageService()
+
+ // Create ModuleInfo object for GetModuleManifest call
+ set moduleInfo = ##class(%IPM.Storage.ModuleInfo).%New()
+ set moduleInfo.Name = "module-a"
+ set moduleInfo.VersionString = "1.0.0"
+
+ set manifest = packageService.GetModuleManifest(moduleInfo)
+ do $$$AssertTrue($isobject(manifest),"Got module manifest")
+
+ // After accessing manifest, the ManifestLoaded flag should be set
+ set cacheAReloaded = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertEquals(cacheAReloaded.ManifestLoaded,1,"ManifestLoaded flag is now true after accessing manifest")
+}
+
+/// Test manual cache rebuild with -rebuild-cache command
+Method TestManualCacheRebuild()
+{
+ // Set a known stale timestamp so we can verify the rebuild updates it without sleeping
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition")
+ set repoDef.CacheLastRebuilt = "1970-01-01 00:00:00"
+ $$$ThrowOnError(repoDef.%Save())
+
+ // Rebuild cache using command
+ set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache")
+ do $$$AssertStatusOK(sc,"Cache rebuild command executed successfully")
+
+ // Verify timestamp was updated from the stale value
+ set repoDefAfter = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition after rebuild")
+ do $$$AssertNotEquals(repoDefAfter.CacheLastRebuilt,"1970-01-01 00:00:00","CacheLastRebuilt timestamp was updated")
+
+ // Verify cache entries still exist
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry for module-a still exists after rebuild")
+
+ // Test installing a module to verify cache still works
+ set sc = ##class(%IPM.Main).Shell("install module-b 2.0.0")
+ do $$$AssertStatusOK(sc,"Successfully installed module-b after cache rebuild")
+}
+
+/// Read a file's raw bytes. Throws if the file can't be opened.
+/// Binary rather than character: these tests hash the file, so a read/write round trip has to
+/// reproduce the original bytes exactly. A character stream rewrites CRLF as LF, and the fixtures
+/// check out with CRLF wherever git's autocrlf applies.
+Method ReadFile(path As %String) As %String [ Private ]
+{
+ set file = ##class(%Stream.FileBinary).%New()
+ $$$ThrowOnError(file.LinkToFile(path))
+ set content = ""
+ while 'file.AtEnd {
+ set content = content_file.Read()
+ }
+ quit content
+}
+
+/// Replace a file's contents with the given bytes. See ReadFile for why this is binary.
+Method WriteFile(
+ path As %String,
+ content As %String) As %Status [ Private ]
+{
+ set sc = $$$OK
+ try {
+ set file = ##class(%Stream.FileBinary).%New()
+ $$$ThrowOnError(file.LinkToFile(path))
+ do file.Clear()
+ do file.Write(content)
+ $$$ThrowOnError(file.%Save())
+ } catch e {
+ set sc = e.AsStatus()
+ }
+ quit sc
+}
+
+/// Test cache freshness detection via ContentHash tracking: an edited module.xml gets a different
+/// hash, so the next validated open re-parses the entry.
+Method TestCacheFreshnessDetection()
+{
+ // Get the original cache entry
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Opened original cache entry")
+ set originalHash = cacheA.ContentHash
+ do $$$AssertNotEquals(originalHash,"","Original ContentHash is set")
+
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-a/1.0.0",..RepoPath))
+ set originalContent = ..ReadFile(moduleFile)
+ do $$$AssertNotEquals(originalContent,"","Read the original module.xml content")
+
+ try {
+ // Change the content, not just the timestamp: content hashing only reacts to real edits.
+ set modifiedContent = $replace(originalContent,"module","module"_$char(10)_" freshness probe")
+ do $$$AssertNotEquals(modifiedContent,originalContent,"Prepared modified module.xml content")
+ do $$$AssertStatusOK(..WriteFile(moduleFile,modifiedContent),"Rewrote module.xml with changed content")
+
+ // Access the cache entry via validated open (this checks ContentHash)
+ set cacheARefreshed = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry was refreshed based on ContentHash")
+ do $$$AssertNotEquals(cacheARefreshed.ContentHash,originalHash,"ContentHash was updated after the content changed")
+ do $$$AssertEquals(cacheARefreshed.ManifestLoaded,0,"Changed manifest is flagged for reparse")
+
+ // Verify we can still install the module
+ do $$$AssertStatusOK(##class(%IPM.Main).Shell("install module-a 1.0.0"),"Successfully installed module after cache refresh")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(),"No error while testing cache freshness")
+ }
+}
+
+/// A module.xml that cannot be parsed must not cost the module its cache entry. The file is still
+/// present, so nothing has been deleted and the stale-entry cleanup has no business evicting it.
+Method TestUnparseableFileKeepsCacheEntry()
+{
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition")
+
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-a/1.0.0",..RepoPath))
+
+ try {
+ do $$$AssertStatusOK(..WriteFile(moduleFile,""),"Wrote a truncated module.xml")
+
+ // purge=0 is the path an install takes, and the only one that runs stale-entry cleanup.
+ do $$$AssertStatusOK(repoDef.BuildCache(0,0),"Rebuilt cache over the unparseable file")
+
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheA),"module-a 1.0.0 survived a rebuild over an unparseable module.xml")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(),"No error while testing an unparseable module.xml")
+ }
+}
+
+/// Editing in place changes which module the cache entry describes. Resolving the old
+/// version has to fail rather than quietly hand back the new one.
+Method TestVersionBumpInPlaceDoesNotReturnWrongModule()
+{
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-a/1.0.0",..RepoPath))
+ set originalContent = ..ReadFile(moduleFile)
+
+ try {
+ set bumpedContent = $replace(originalContent,"1.0.0","1.1.0")
+ do $$$AssertNotEquals(bumpedContent,originalContent,"Prepared a version-bumped module.xml")
+ do $$$AssertStatusOK(..WriteFile(moduleFile,bumpedContent),"Wrote the version-bumped module.xml")
+
+ set stale = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusNotOK(sc,"Resolving 1.0.0 fails once the file on disk describes 1.1.0")
+ do $$$AssertTrue('$isobject(stale),"No cache entry is handed back for the version that is gone")
+
+ set bumped = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.1.0",,.sc)
+ do $$$AssertTrue($isobject(bumped),"The refreshed entry describes module-a 1.1.0")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(),"No error while testing an in-place version bump")
+ }
+}
+
+/// The SQL walk only runs when embedded Python is unavailable, so it needs its own coverage. It has
+/// to agree with the Python walk that built this repo's cache, since either one can serve a rebuild.
+Method TestSQLWalkMatchesPythonWalk()
+{
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).WalkDirectoriesSQL(..RepoPath,..RepoPath,0,"module.xml",1,.sqlFiles,.sqlHashes),"SQL walk succeeded")
+
+ set dirs(1) = ..RepoPath
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).WalkDirectories(.dirs,..RepoPath,0,"module.xml",1,.pyFiles,.pyHashes),"Python walk succeeded")
+
+ for relPath = "module-a/1.0.0/module.xml","module-b/1.0.0/module.xml","module-b/2.0.0/module.xml" {
+ do $$$AssertTrue($data(sqlFiles(relPath)),"SQL walk found "_relPath)
+ do $$$AssertEquals($get(pyFiles(relPath)),$get(sqlFiles(relPath)),"Both walkers resolved "_relPath_" to the same full path")
+ do $$$AssertEquals($get(pyHashes(relPath)),$get(sqlHashes(relPath)),"Both walkers hashed "_relPath_" identically")
+ }
+
+ // The cache was built by the Python walk, so its stored hash is the answer the SQL walk has to reach.
+ set cacheA = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Opened the cache entry for module-a 1.0.0")
+ do $$$AssertEquals($get(sqlHashes("module-a/1.0.0/module.xml")),cacheA.ContentHash,"SQL walk reproduced the cached ContentHash")
+}
+
+/// Two directories under one root claiming the same Name and Version collide on the RootNameVersion
+/// index. The re-parse cannot be saved, so the lookup has to fail rather than return the unsaved row.
+Method TestDuplicateNameVersionFailsValidatedOpen()
+{
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-b/2.0.0",..RepoPath))
+ set originalContent = ..ReadFile(moduleFile)
+
+ try {
+ set duplicateContent = $replace(originalContent,"2.0.0","1.0.0")
+ do $$$AssertNotEquals(duplicateContent,originalContent,"Prepared a module.xml duplicating module-b 1.0.0")
+ do $$$AssertStatusOK(..WriteFile(moduleFile,duplicateContent),"Wrote the duplicate module.xml")
+
+ set duplicate = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..RepoPath,"module-b","2.0.0",,.sc)
+ do $$$AssertStatusNotOK(sc,"Resolving 2.0.0 fails once its directory duplicates 1.0.0")
+ do $$$AssertTrue('$isobject(duplicate),"No cache entry is handed back for the duplicate")
+
+ // The entry the duplicate collided with is untouched.
+ do $$$AssertTrue(##class(%IPM.Repo.Filesystem.Cache).RootNameVersionExists(..RepoPath,"module-b","1.0.0"),"module-b 1.0.0 is still cached")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(),"No error while testing a duplicated Name and Version")
+ }
+}
+
+/// A root that has gone missing must fail the rebuild rather than empty the cache. An unmounted share
+/// should not look like a repository whose modules were all deleted.
+Method TestMissingRootDoesNotEmptyCache()
+{
+ set tempRoot = ##class(%IPM.Utils.File).CreateTempDirectory()
+
+ try {
+ set moduleDir = ##class(%File).NormalizeFilename("module-b/3.0.0",tempRoot)
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).CreateDirectoryChain(moduleDir),"Created a module directory under a temporary root")
+
+ set xdata = ##class(%Dictionary.XDataDefinition).%OpenId($classname()_"||ModuleB300",,.sc)
+ do $$$AssertStatusOK(sc,"Opened XData block")
+ set file = ##class(%Stream.FileCharacter).%New()
+ $$$ThrowOnError(file.LinkToFile(##class(%File).NormalizeFilename("module.xml",moduleDir)))
+ $$$ThrowOnError(file.CopyFrom(xdata.Data))
+ $$$ThrowOnError(file.%Save())
+
+ do $$$AssertStatusOK(##class(%IPM.Main).Shell("repo -n fs-missing-root -fs -path "_tempRoot),"Created a repo over the temporary root")
+
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen("fs-missing-root",,.sc)
+ do $$$AssertStatusOK(sc,"Opened the temporary repo definition")
+ do $$$AssertTrue(##class(%IPM.Repo.Filesystem.Cache).RootNameVersionExists(repoDef.Root,"module-b","3.0.0"),"module-b 3.0.0 is cached while the root exists")
+
+ do $$$AssertTrue(##class(%File).RemoveDirectoryTree(tempRoot),"Removed the temporary root")
+
+ do $$$AssertStatusNotOK(repoDef.BuildCache(0,0),"Rebuilding against a missing root reports an error")
+ do $$$AssertTrue(##class(%IPM.Repo.Filesystem.Cache).RootNameVersionExists(repoDef.Root,"module-b","3.0.0"),"The cache entry survived a rebuild against a missing root")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(),"No error while testing a missing root")
+ }
+
+ do $$$AssertStatusOK(##class(%IPM.Main).Shell("repo -delete -name fs-missing-root"),"Cleaned up the temporary repo")
+ // Already removed by the test body on the happy path; this covers an early failure.
+ do ##class(%IPM.Utils.File).RemoveDirectoryTree(tempRoot)
+}
+
+/// Test adding a new version to the filesystem and auto-discovery on install
+Method TestAutoDiscoverNewVersion()
+{
+ // Create new directory for module-b 3.0.0
+ set newVersionDir = ##class(%File).NormalizeFilename("module-b/3.0.0",..RepoPath)
+ set created = ##class(%File).CreateDirectory(newVersionDir)
+ do $$$AssertTrue(created,"Created directory for module-b 3.0.0")
+
+ // Create module.xml file from XData block
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",newVersionDir)
+
+ // Read XData content
+ set xdata = ##class(%Dictionary.XDataDefinition).%OpenId($classname()_"||ModuleB300",,.sc)
+ do $$$AssertStatusOK(sc,"Opened XData block")
+
+ set file = ##class(%Stream.FileCharacter).%New()
+ set sc = file.LinkToFile(moduleFile)
+ do $$$AssertStatusOK(sc,"Linked to new module file")
+ do file.CopyFrom(xdata.Data)
+ set sc = file.%Save()
+ do $$$AssertStatusOK(sc,"Saved new module.xml file")
+
+ // NOTE: No manual cache rebuild here - install should auto-discover the new version
+ // This tests the auto-rebuild feature (cache rebuilt before dependency resolution)
+
+ // Verify new version is NOT yet in cache
+ set cacheB3 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc)
+ do $$$AssertTrue('$isobject(cacheB3),"module-b 3.0.0 not yet in cache before install")
+
+ // Install should auto-rebuild cache and find the new version
+ set sc = ##class(%IPM.Main).Shell("install module-b 3.0.0")
+ do $$$AssertStatusOK(sc,"Successfully installed module-b 3.0.0 with auto-rebuild")
+
+ // Verify new version is NOW in cache (after install auto-rebuilt)
+ set cacheB3After = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Cache entry for module-b 3.0.0 exists after install")
+ do $$$AssertTrue($isobject(cacheB3After),"module-b 3.0.0 found in cache after auto-rebuild")
+}
+
+/// Test that a stale cache entry is removed when its directory is deleted and cache is rebuilt.
+Method TestStaleEntryCleanup()
+{
+ // Create a temporary module-b 3.0.0 directory with a valid module.xml
+ set newVersionDir = ##class(%File).NormalizeFilename("module-b/3.0.0",..RepoPath)
+ set created = ##class(%File).CreateDirectory(newVersionDir)
+ do $$$AssertTrue(created,"Created directory for module-b 3.0.0")
+
+ set xdata = ##class(%Dictionary.XDataDefinition).%OpenId($classname()_"||ModuleB300",,.sc)
+ do $$$AssertStatusOK(sc,"Opened XData block")
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",newVersionDir)
+ set file = ##class(%Stream.FileCharacter).%New()
+ $$$ThrowOnError(file.LinkToFile(moduleFile))
+ do file.CopyFrom(xdata.Data)
+ $$$ThrowOnError(file.%Save())
+
+ // Rebuild cache to pick up the new directory
+ set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache")
+ do $$$AssertStatusOK(sc,"Rebuilt cache to populate module-b 3.0.0 entry")
+
+ set cacheB3 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheB3),"module-b 3.0.0 exists in cache before deletion")
+
+ // Delete the directory to create a stale cache entry
+ set deleted = ##class(%File).RemoveDirectoryTree(newVersionDir)
+ do $$$AssertTrue(deleted,"Deleted module-b 3.0.0 directory")
+
+ // Rebuild cache to trigger stale entry cleanup
+ set sc = ##class(%IPM.Main).Shell("repo -n "_..#REPONAME_" -rebuild-cache")
+ do $$$AssertStatusOK(sc,"Rebuilt cache after deletion")
+
+ // Verify stale entry was removed
+ set cacheB3After = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","3.0.0",,.sc)
+ do $$$AssertTrue('$isobject(cacheB3After),"module-b 3.0.0 removed from cache after deletion")
+
+ // Verify other versions still exist
+ set cacheB2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-b","2.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheB2),"module-b 2.0.0 still exists in cache")
+}
+
+/// Test depth specification - create repo with depth and verify it only scans to that depth
+Method TestDepthSpecification()
+{
+ // Test structure:
+ // fs-cache-test-depth/module-shallow/1.0.0/module.xml (2 levels deep)
+ // fs-cache-test-depth/nested/module-deep/1.0.0/module.xml (3 levels deep)
+
+ // Create a new repo with depth=2 (should only find modules up to 2 levels deep)
+ set depthRepoPath = ..GetModuleDir("fs-cache-test-depth")
+ set sc = ##class(%IPM.Main).Shell("repo -n fs-cache-depth -fs -path "_depthRepoPath_" -depth 2")
+ do $$$AssertStatusOK(sc,"Created repo with depth=2")
+
+ // Get repo definition and verify depth is set
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen("fs-cache-depth",,.sc)
+ do $$$AssertStatusOK(sc,"Opened depth repo definition")
+ do $$$AssertEquals(repoDef.Depth,2,"Depth is set to 2")
+
+ // Verify that only modules at depth 2 are cached (not nested deeper)
+ // module-shallow at depth 2 should be found
+ set cacheShallow = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-shallow","1.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheShallow),"module-shallow at depth 2 found in cache")
+
+ // module-deep at depth 3 should NOT be found (exceeds depth limit)
+ set cacheDeep = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-deep","1.0.0",,.sc)
+ do $$$AssertTrue('$isobject(cacheDeep),"module-deep at depth 3 not found in cache (exceeds depth)")
+
+ // Clean up depth repo
+ set sc = ##class(%IPM.Main).Shell("repo -delete -name fs-cache-depth")
+ do $$$AssertStatusOK(sc,"Cleaned up depth test repo")
+
+ // Test repo without depth specification (depth=0, unlimited)
+ set sc = ##class(%IPM.Main).Shell("repo -n fs-cache-unlimited -fs -path "_depthRepoPath)
+ do $$$AssertStatusOK(sc,"Created repo with unlimited depth")
+
+ set repoDefUnlimited = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen("fs-cache-unlimited",,.sc)
+ do $$$AssertStatusOK(sc,"Opened unlimited depth repo definition")
+ do $$$AssertEquals(repoDefUnlimited.Depth,0,"Depth is 0 (unlimited)")
+
+ // Both shallow and deep modules should be found with unlimited depth
+ set cacheShallow2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-shallow","1.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheShallow2),"module-shallow found with unlimited depth")
+
+ set cacheDeep2 = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(depthRepoPath,"module-deep","1.0.0",,.sc)
+ do $$$AssertTrue($isobject(cacheDeep2),"module-deep found with unlimited depth")
+
+ // Clean up
+ set sc = ##class(%IPM.Main).Shell("repo -delete -name fs-cache-unlimited")
+ do $$$AssertStatusOK(sc,"Cleaned up unlimited depth test repo")
+}
+
+/// A touch that leaves the content identical must not invalidate the cache entry: this is the
+/// difference between content hashing and the mtime tracking it replaced.
+Method TestTouchDoesNotReparse()
+{
+ set moduleFile = ##class(%File).NormalizeFilename("module.xml",##class(%File).NormalizeFilename("module-a/1.0.0",..RepoPath))
+
+ // Load the manifest first, so there is a set ManifestLoaded flag to watch survive the touch.
+ set repoDef = ##class(%IPM.Repo.Filesystem.Definition).ServerDefinitionKeyOpen(..#REPONAME,,.sc)
+ do $$$AssertStatusOK(sc,"Opened repo definition")
+ set packageService = repoDef.GetPackageService()
+ set moduleInfo = ##class(%IPM.Storage.ModuleInfo).%New()
+ set moduleInfo.Name = "module-a"
+ set moduleInfo.VersionString = "1.0.0"
+ do $$$AssertTrue($isobject(packageService.GetModuleManifest(moduleInfo)),"Got module manifest")
+
+ set cacheBefore = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpen(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Opened cache entry before the touch")
+ do $$$AssertEquals(cacheBefore.ManifestLoaded,1,"Manifest is loaded before the touch")
+ set hashBefore = cacheBefore.ContentHash
+
+ // Rewrite byte-identical content: this bumps the file's mtime but not its content hash.
+ set content = ..ReadFile(moduleFile)
+ do $$$AssertStatusOK(..WriteFile(moduleFile,content),"Rewrote module.xml with identical content")
+
+ set cacheAfter = ##class(%IPM.Repo.Filesystem.Cache).RootNameVersionOpenValidated(..RepoPath,"module-a","1.0.0",,.sc)
+ do $$$AssertStatusOK(sc,"Validated open succeeded after the touch")
+ do $$$AssertEquals(cacheAfter.ContentHash,hashBefore,"ContentHash is unchanged by a touch")
+ do $$$AssertEquals(cacheAfter.ManifestLoaded,1,"Touch did not discard the loaded manifest")
+}
+
+}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml
new file mode 100644
index 000000000..9f22c7649
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/module-shallow/1.0.0/module.xml
@@ -0,0 +1,10 @@
+
+
+
+
+ module-shallow
+ 1.0.0
+ module
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml
new file mode 100644
index 000000000..0737afb0e
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test-depth/nested/module-deep/1.0.0/module.xml
@@ -0,0 +1,10 @@
+
+
+
+
+ module-deep
+ 1.0.0
+ module
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml
new file mode 100644
index 000000000..e9025b5ed
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-a/1.0.0/module.xml
@@ -0,0 +1,10 @@
+
+
+
+
+ module-a
+ 1.0.0
+ module
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml
new file mode 100644
index 000000000..2a7c846c4
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/1.0.0/module.xml
@@ -0,0 +1,10 @@
+
+
+
+
+ module-b
+ 1.0.0
+ module
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml
new file mode 100644
index 000000000..08f2cbd30
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/fs-cache-test/module-b/2.0.0/module.xml
@@ -0,0 +1,10 @@
+
+
+
+
+ module-b
+ 2.0.0
+ module
+
+
+
diff --git a/tests/unit_tests/Test/PM/Unit/FileHash.cls b/tests/unit_tests/Test/PM/Unit/FileHash.cls
index f06fd0f23..de0255c9f 100644
--- a/tests/unit_tests/Test/PM/Unit/FileHash.cls
+++ b/tests/unit_tests/Test/PM/Unit/FileHash.cls
@@ -175,11 +175,70 @@ Method TestPendingTestsRunOnNextTestSync()
Method TestNormalizePath()
{
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src\cls\Foo\Bar.cls"), "src/cls/Foo/Bar.cls", "Backslashes -> forward slashes")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src//cls///Foo.cls"), "src/cls/Foo.cls", "Consecutive slashes collapsed")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("/src/cls/Foo.cls"), "src/cls/Foo.cls", "Leading slash stripped")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("//src/Foo.cls"), "src/Foo.cls", "Multiple leading slashes stripped")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src/cls/Foo.cls"), "src/cls/Foo.cls", "Already-normalized path unchanged")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("src\cls\Foo\Bar.cls"), "src/cls/Foo/Bar.cls", "Backslashes -> forward slashes")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("src//cls///Foo.cls"), "src/cls/Foo.cls", "Consecutive slashes collapsed")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("/src/cls/Foo.cls"), "src/cls/Foo.cls", "Leading slash stripped")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("//src/Foo.cls"), "src/Foo.cls", "Multiple leading slashes stripped")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("src/cls/Foo.cls"), "src/cls/Foo.cls", "Already-normalized path unchanged")
+}
+
+/// Write content to a new file, creating its directory if needed.
+Method WriteFileAt(
+ path As %String,
+ content As %String) [ Private ]
+{
+ $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(##class(%File).GetDirectory(path)))
+ set file = ##class(%Stream.FileCharacter).%New()
+ $$$ThrowOnError(file.LinkToFile(path))
+ do file.Write(content)
+ $$$ThrowOnError(file.%Save())
+}
+
+/// %IPM.Utils.File:WalkDirectories backs both sync's change detection and the filesystem
+/// repository cache scan, so depth capping, the filename filter and the skipped directories all
+/// have callers depending on them.
+Method TestWalkDirectories()
+{
+ set root = ##class(%IPM.Utils.File).CreateTempDirectory()
+ try {
+ // module.xml in the root and in a/ share content, so they must also share a hash.
+ do ..WriteFileAt(root_"module.xml", "same")
+ do ..WriteFileAt(root_"a/module.xml", "same")
+ do ..WriteFileAt(root_"a/notes.txt", "not a manifest")
+ do ..WriteFileAt(root_"a/b/module.xml", "deep")
+ do ..WriteFileAt(root_".git/module.xml", "vcs")
+
+ set dirs(1) = root
+
+ // Unlimited depth, filtered to module.xml
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).WalkDirectories(.dirs, root, 0, "module.xml", 1, .files, .hashes), "Walked the tree filtering on module.xml")
+ do $$$AssertTrue($data(files("module.xml")), "Found module.xml in the root")
+ do $$$AssertTrue($data(files("a/module.xml")), "Found module.xml one level down")
+ do $$$AssertTrue($data(files("a/b/module.xml")), "Found module.xml two levels down")
+ do $$$AssertTrue('$data(files(".git/module.xml")), ".git was not descended into")
+ do $$$AssertTrue('$data(files("a/notes.txt")), "Filename filter excluded notes.txt")
+
+ do $$$AssertEquals($length(hashes("module.xml")), 40, "Hash is 40 hex characters")
+ do $$$AssertEquals($zconvert(hashes("module.xml"), "L"), hashes("module.xml"), "Hash is lowercase hex")
+ do $$$AssertEquals(hashes("a/module.xml"), hashes("module.xml"), "Identical content hashes identically")
+ do $$$AssertNotEquals(hashes("a/b/module.xml"), hashes("module.xml"), "Different content hashes differently")
+
+ // Depth cap: a file exactly at the cap is kept, anything below is never visited.
+ kill files, hashes
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).WalkDirectories(.dirs, root, 1, "module.xml", 1, .files, .hashes), "Walked the tree with depth capped at 1")
+ do $$$AssertTrue($data(files("module.xml")), "Root module.xml still found at depth 1")
+ do $$$AssertTrue($data(files("a/module.xml")), "File exactly at the depth cap is kept")
+ do $$$AssertTrue('$data(files("a/b/module.xml")), "File below the depth cap is not visited")
+
+ // No filter and no hashing: every file, empty hashes.
+ kill files, hashes
+ do $$$AssertStatusOK(##class(%IPM.Utils.File).WalkDirectories(.dirs, root, 0, "", 0, .files, .hashes), "Walked the tree unfiltered without hashing")
+ do $$$AssertTrue($data(files("a/notes.txt")), "Empty filter keeps non-manifest files")
+ do $$$AssertEquals(hashes("a/notes.txt"), "", "Hashing off leaves hashes empty")
+ } catch e {
+ do $$$AssertStatusOK(e.AsStatus(), "No error while walking directories")
+ }
+ do ##class(%IPM.Utils.File).RemoveDirectoryTree(root)
}
/// A module with . composes every resource path as SourcesRoot_"/"_Directory,
@@ -188,21 +247,21 @@ Method TestNormalizePath()
/// never matches a changed path and edits under such a resource are silently dropped.
Method TestNormalizePathDotSegments()
{
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("./cls/Foo.cls"), "cls/Foo.cls", "Leading ./ stripped (SourcesRoot='.')")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath(".\cls\Foo.cls"), "cls/Foo.cls", "Leading .\ stripped after slash translation")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath(".//cls/Foo.cls"), "cls/Foo.cls", "Leading ./ with doubled slash")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("././cls/Foo.cls"), "cls/Foo.cls", "Repeated leading ./ (SourcesRoot='./.')")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src/./cls/Foo.cls"), "src/cls/Foo.cls", "Interior ./ segment dropped")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("/./cls/Foo.cls"), "cls/Foo.cls", "Leading /./ collapsed")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("./cls/"), "cls/", "Trailing slash preserved (GetSyncDirectory prefix scans)")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("./cls/Foo.cls"), "cls/Foo.cls", "Leading ./ stripped (SourcesRoot='.')")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath(".\cls\Foo.cls"), "cls/Foo.cls", "Leading .\ stripped after slash translation")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath(".//cls/Foo.cls"), "cls/Foo.cls", "Leading ./ with doubled slash")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("././cls/Foo.cls"), "cls/Foo.cls", "Repeated leading ./ (SourcesRoot='./.')")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("src/./cls/Foo.cls"), "src/cls/Foo.cls", "Interior ./ segment dropped")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("/./cls/Foo.cls"), "cls/Foo.cls", "Leading /./ collapsed")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("./cls/"), "cls/", "Trailing slash preserved (GetSyncDirectory prefix scans)")
// SourcesRoot="." with no Directory reduces to the module root itself, not a file.
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("."), "", "Bare '.' is the module root, not a path")
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("./"), "", "'./' is the module root, not a path")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("."), "", "Bare '.' is the module root, not a path")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("./"), "", "'./' is the module root, not a path")
// ".." is not a path this pipeline should ever see and is deliberately left alone rather than
// resolved, so a stray one stays visibly wrong instead of silently escaping the module root.
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("../cls/Foo.cls"), "../cls/Foo.cls", "Parent-dir segments are not resolved")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("../cls/Foo.cls"), "../cls/Foo.cls", "Parent-dir segments are not resolved")
// A dot inside a name is not a "." segment.
- do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("cls/.hidden/Foo.cls"), "cls/.hidden/Foo.cls", "Dot-prefixed directory name preserved")
+ do $$$AssertEquals(##class(%IPM.Utils.File).NormalizePath("cls/.hidden/Foo.cls"), "cls/.hidden/Foo.cls", "Dot-prefixed directory name preserved")
}
/// Document names reach the sync pipeline from two sources that disagree on extension case: