From 777c47ad7d16b8516b9cb1806cd19b2b276d64fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 28 Oct 2026 22:01:33 +0100 Subject: [PATCH 01/12] scripts: remove unused imports --- pym/bob/scripts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pym/bob/scripts.py b/pym/bob/scripts.py index f58cb429..8274b85a 100644 --- a/pym/bob/scripts.py +++ b/pym/bob/scripts.py @@ -7,7 +7,6 @@ from .errors import BobError from .state import finalize from .tty import colorize, Unbuffered, setColorMode, cleanup -from .utils import asHexStr, hashPath, getPlatformTag, EventLoopWrapper import argparse import logging import sys From 31951098b2716253d1e5c00e585a650a7940fe16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Fri, 9 Oct 2026 17:16:23 +0200 Subject: [PATCH 02/12] builder: don't block main loop when hashing workspace Run the hasher in an executor thread. This should make Bob more responsive when hashing a large workspace. --- pym/bob/builder.py | 25 +++++++++++++------------ pym/bob/intermediate.py | 4 ++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index c9a255b9..72211a88 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -52,18 +52,19 @@ async def gatherTasks(tasks): await asyncio.wait(tasks) return [ t.result() for t in tasks ] -def hashWorkspace(step): +async def hashWorkspace(step): + loop = asyncio.get_event_loop() + cachePath = os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin") # Cache is also used by share module - return hashDirectory(step.getStoragePath(), - os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin")) + return await loop.run_in_executor(None, hashDirectory, step.getStoragePath(), cachePath) class HashOnce: def __init__(self, step): self.step = step self.result = None - def hashWorkspace(self): + async def hashWorkspace(self): if self.result is None: - self.result = hashWorkspace(self.step) + self.result = await hashWorkspace(self.step) return self.result def invalidate(self): self.result = None @@ -1202,7 +1203,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None): inputChanged = checkoutBuildOnlyStateChanged(checkoutState, oldCheckoutState) rehash = currentResultHash.hashWorkspace - if checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash): + if await checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash): if checkoutBuildOnlyStateCompatible(checkoutState, oldCheckoutState): with stepExec(checkoutStep, "UPDATE", "{} {}".format(prettySrcPath, overridesString)) as a: @@ -1247,7 +1248,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)): checkoutReason = "dependency changed" elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \ - and (oldResultHash != currentResultHash.hashWorkspace()): + and (oldResultHash != await currentResultHash.hashWorkspace()): checkoutReason = "workspace changed" if checkoutReason: @@ -1345,7 +1346,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): # We always have to rehash the directory as the user might have # changed the source code manually. - checkoutHash = currentResultHash.hashWorkspace() + checkoutHash = await currentResultHash.hashWorkspace() # Generate audit trail before setResultHash() to force re-generation if # the audit trail fails. @@ -1413,7 +1414,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId): # We always rehash the directory in development mode as the # user might have compiled the package manually. if not self.__cleanBuild: - BobState().setResultHash(prettyBuildPath, hashWorkspace(buildStep)) + BobState().setResultHash(prettyBuildPath, await hashWorkspace(buildStep)) else: with stepExec(buildStep, "BUILD", prettyBuildPath, (ALWAYS, NORMAL)) as fullAction: # Squash state because running the step will change the @@ -1425,7 +1426,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId): with stepExec(buildStep, "BUILD", prettyBuildPath, INFO) as buildAction: visibleAction = fullAction if fullAction.visible else buildAction await self._runShell(buildStep, "build", visibleAction, created, self.__cleanBuild) - buildHash = hashWorkspace(buildStep) + buildHash = await hashWorkspace(buildStep) await self._generateAudit(buildStep, depth, buildHash, buildBuildId) BobState().setResultHash(prettyBuildPath, buildHash) BobState().setVariantId(prettyBuildPath, buildDigest[0]) @@ -1626,7 +1627,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId): # Verify integrity of downloaded package to protect against # data corruption. - packageHash = hashWorkspace(packageStep) + packageHash = await hashWorkspace(packageStep) if Audit.fromFile(audit).getArtifact().getResultHash() != packageHash: raise BuildError("Corrupt downloaded artifact! Extracted content hash does not match audit trail.") @@ -1686,7 +1687,7 @@ async def _cookPackageStep(self, packageStep, depth, packageBuildId): with stepExec(packageStep, "PACKAGE", prettyPackagePath, INFO) as packageAction: visibleAction = fullAction if fullAction.visible else packageAction await self._runShell(packageStep, "package", visibleAction, True) - packageHash = hashWorkspace(packageStep) + packageHash = await hashWorkspace(packageStep) packageDigest = await self.__getIncrementalVariantId(packageStep) workspaceChanged = True self.__statistic.packagesBuilt += 1 diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index ed3d75db..ec98d641 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -308,14 +308,14 @@ def deserialize(state): def getScmDirectories(self): return { d : (bytes.fromhex(h), p) for (d, (h, p)) in self.__data['scmDirectories'].items() } - def mayUpdate(self, inputChanged, oldHash, rehash): + async def mayUpdate(self, inputChanged, oldHash, rehash): if any((s.isLocal() and not s.isDeterministic()) for s in self.getScmList()): return True if not self.getUpdateScript(): return False if not self.isUpdateDeterministic() or inputChanged: return True - return rehash() != oldHash + return await rehash() != oldHash async def getDigestCoro(self, calculate, hasher=DigestHasher, fingerprint=None, platform=b'', relaxTools=False): From 7c041e5a9852671c0480057c16249997b3eb67b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 28 Oct 2026 22:09:21 +0100 Subject: [PATCH 03/12] utils: store hashed directory modes in little endian The hash sum of directory entries holds the file modes. Use a defined endianess (little endian) to make the result system independent. Previously the hash sum of an identical directory tree would have differed between little and big endian systems. --- pym/bob/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index ba7c913e..bbb0e83c 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -588,7 +588,7 @@ def __hashDir(self, prefix, path=b''): entries = sorted(entries, key=lambda x: x[1]) dirList = [ - (struct.pack("=L", s.st_mode) + self.__hashEntry(prefix, e, s) + f) + (struct.pack(" Date: Wed, 28 Oct 2026 22:15:39 +0100 Subject: [PATCH 04/12] utils: account for parent dir in hashDirectory The calls to hashDirectory() and hashPath() should calculate the same size. --- pym/bob/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index bbb0e83c..b738d53b 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -597,6 +597,14 @@ def __hashDir(self, prefix, path=b''): return m.digest() def hashDirectory(self, path): + path = os.fsencode(path) + try: + # The size of the top directory is accounted too + s = os.stat(path) + if stat.S_ISDIR(s.st_mode): self.size += s.st_size + except OSError: + pass # will be reported by __hashDir + self.__index.open() try: return self.__hashDir(os.fsencode(path)) From 42500e39e1aee99c6fdf1613510097c0fa1aea27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 28 Oct 2026 22:32:02 +0100 Subject: [PATCH 05/12] utils: gracefully handle hash cache failures The "index" file for a hashed directory tree is just an optimization. If it cannot be read, it's better to just ignore the cache instead of failing the build. Likewise, if writing the cache fails, keep going on. Nothing bad happens. It's just that the next hashing of the same tree will be slower. --- pym/bob/utils.py | 55 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index b738d53b..a2b4e285 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -433,7 +433,9 @@ def __init__(self, cachePath): def open(self): self.__inPos = 0 self.__inPosOld = 0 + self.__inFile = None self.__outFile = None + self.__writeFailed = False self.__current = DirHasher.FileIndex.Stat() try: if os.path.exists(self.__cachePath): @@ -453,17 +455,33 @@ def open(self): self.__inFile = None self.__mismatch = True except OSError as e: - raise BuildError("Error opening hash cache: " + str(e)) - - def close(self): - try: + logging.getLogger(__name__).warning("Cannot open hash cache: %s", str(e)) if self.__inFile: self.__inFile.close() - if self.__outFile: + self.__inFile = None + self.__mismatch = True + + def close(self): + if self.__inFile: + self.__inFile.close() + if self.__outFile: + try: self.__outFile.close() replacePath(self.__outFile.name, self.__cachePath) - except OSError as e: - raise BuildError("Error closing hash cache: " + str(e)) + except OSError as e: + self.__abortWrite(e) + + def __abortWrite(self, err): + """Give up writing the cache. The old cache is kept untouched.""" + logging.getLogger(__name__).warning("Cannot write hash cache: %s", str(err)) + self.__writeFailed = True + if self.__outFile: + try: + self.__outFile.close() + os.unlink(self.__outFile.name) + except OSError: + pass + self.__outFile = None def __readEntry(self): if not self.__inFile: return False @@ -477,8 +495,10 @@ def __readEntry(self): self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen return True - def __writeEntry(self, name, st, digest): - if not self.__outFile: + def __startWrite(self): + """Start the new index. All entries before __current are kept.""" + if self.__outFile or self.__writeFailed: return + try: self.__outFile = NamedTemporaryFile(mode="wb", dir=self.__cacheDir, delete=False) if self.__inFile: pos = self.__inFile.tell() @@ -487,10 +507,19 @@ def __writeEntry(self, name, st, digest): self.__inFile.seek(pos) else: self.__outFile.write(DirHasher.FileIndex.SIGNATURE) - self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns, - st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size, - digest, len(name))) - self.__outFile.write(name) + except OSError as e: + self.__abortWrite(e) + + def __writeEntry(self, name, st, digest): + self.__startWrite() + if not self.__outFile: return + try: + self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns, + st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size, + digest, len(name))) + self.__outFile.write(name) + except OSError as e: + self.__abortWrite(e) def __match(self, name, st): while self.__current.name < name: From a8d3614833e037b448d1bf2eb46f552b745169a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 29 Oct 2026 21:53:35 +0100 Subject: [PATCH 06/12] utils: don't cache hashes of very recent files File time stamps have a limited granularity. A file that is modified again in the same file system clock tick after it was hashed would keep its stat data. Such "racily clean" entries must not be trusted. Any file whose change-time is not older than the current file system time is thus not recorded in the index. --- pym/bob/utils.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index a2b4e285..09126834 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -5,7 +5,7 @@ from .errors import BuildError, ParseError from binascii import hexlify -from tempfile import NamedTemporaryFile, TemporaryDirectory +from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile import collections.abc import hashlib import logging @@ -253,9 +253,16 @@ def isWindows(): __isWindows = True else: __isWindows = False + + def getChangeTime(st): + return st.st_ctime_ns else: __isWindows = True + def getChangeTime(st): + # The st_ctime field is the creation time on Windows + return st.st_mtime_ns + def _replacePathWin32(src, dst): # Workaround for spurious PermissionError's on Windows. i = 0 @@ -437,6 +444,7 @@ def open(self): self.__outFile = None self.__writeFailed = False self.__current = DirHasher.FileIndex.Stat() + self.__changeTimeLimit = self.__calcChangeTimeLimit() try: if os.path.exists(self.__cachePath): self.__inFile = open(self.__cachePath, "rb") @@ -471,6 +479,23 @@ def close(self): except OSError as e: self.__abortWrite(e) + def __calcChangeTimeLimit(self): + """Determine the current time of the file system clock. + + File time stamps have a limited granularity. A file that is + modified again in the same clock tick after it was hashed would + keep its stat data. Such "racily clean" entries must not be + trusted. Any file whose change-time is not older than the current file + system time is thus not recorded in the index. Creating a file gets + us the current time in the right granularity, even on network file + systems where the clock of the server is used. + """ + try: + with TemporaryFile(dir=self.__cacheDir or ".") as f: + return getChangeTime(os.fstat(f.fileno())) + except OSError: + return -(2**63) + def __abortWrite(self, err): """Give up writing the cache. The old cache is kept untouched.""" logging.getLogger(__name__).warning("Cannot write hash cache: %s", str(err)) @@ -513,6 +538,10 @@ def __startWrite(self): def __writeEntry(self, name, st, digest): self.__startWrite() if not self.__outFile: return + # Entries that are of the same age than the current file system + # time must be dropped. Changes in the same "file system tick" + # cannot be distinguished. + if getChangeTime(st) >= self.__changeTimeLimit: return try: self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns, st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size, From 0ab6a6bf61d0a84d933458c95fba1e4cf1c44d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 29 Oct 2026 22:07:26 +0100 Subject: [PATCH 07/12] utils: fix stale entries handling The hash index implementation kept entries of deleted files. These may accumulate over time. __match() now tracks whether the current entry has already matched a file. If it has to skip an entry that never matched, it marks the index as changed and starts the new index right away, while the copy position still stops before the stale entry. Related to that there was a potential bug in the handling of the last index entry. When a new file sorted after the last index entry, __readEntry() hit end-of-file without moving __inPosOld. The copied prefix then stopped one entry short, so the last matching entry was silently left out. Fortunately that only cost a cache miss on the next run, keeping the result still correct. --- pym/bob/utils.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index 09126834..b2eb2780 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -444,6 +444,7 @@ def open(self): self.__outFile = None self.__writeFailed = False self.__current = DirHasher.FileIndex.Stat() + self.__consumed = False self.__changeTimeLimit = self.__calcChangeTimeLimit() try: if os.path.exists(self.__cachePath): @@ -458,15 +459,18 @@ def open(self): "Wrong signature at '%s': %s", self.__cachePath, sig) self.__inFile.close() self.__inFile = None + self.__current = None self.__mismatch = True else: self.__inFile = None + self.__current = None self.__mismatch = True except OSError as e: logging.getLogger(__name__).warning("Cannot open hash cache: %s", str(e)) if self.__inFile: self.__inFile.close() self.__inFile = None + self.__current = None self.__mismatch = True def close(self): @@ -509,16 +513,22 @@ def __abortWrite(self, err): self.__outFile = None def __readEntry(self): - if not self.__inFile: return False + """Read the next entry into __current. + + Sets __current to None at the end of the index. In any case + __inPosOld is the end of all previous entries. + """ + self.__inPosOld = self.__inPos raw = self.__inFile.read(DirHasher.FileIndex.CACHE_ENTRY_SIZE) - if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: return False + if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: + self.__current = None + return e = self.__current (e.ctime, e.mtime, e.dev, e.ino, e.mode, e.size, e.digest, nameLen) = struct.unpack(DirHasher.FileIndex.CACHE_ENTRY_FMT, raw) e.name = self.__inFile.read(nameLen) - self.__inPosOld = self.__inPos self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen - return True + self.__consumed = False def __startWrite(self): """Start the new index. All entries before __current are kept.""" @@ -551,10 +561,17 @@ def __writeEntry(self, name, st, digest): self.__abortWrite(e) def __match(self, name, st): - while self.__current.name < name: - if not self.__readEntry(): break + while (self.__current is not None) and (self.__consumed or + self.__current.name < name): + if not self.__consumed and not self.__mismatch: + # Stale entry of a removed file. Drop it from the index. + self.__mismatch = True + self.__startWrite() + self.__readEntry() e = self.__current - res = ((e.name == name) and (e.ctime == st.st_ctime_ns) and + if (e is None) or (e.name != name): return False + self.__consumed = True + res = ((e.ctime == st.st_ctime_ns) and (e.mtime == st.st_mtime_ns) and (e.dev == st.st_dev) and (e.ino == maskIno(st.st_ino)) and (e.mode == st.st_mode) and (e.size == st.st_size)) From 1f127bfb7b9d5fae392d99c7e9d9c469456beb20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 29 Oct 2026 22:26:03 +0100 Subject: [PATCH 08/12] utils: add rust implementation for directory hasher On fast NVMe disks, hashing large directory trees is limited by the CPU. Doing that in parallel will greatly reduce the time that is needed for that. The rust python will spawn as many threads as CPUs are available and do the hashing in parallel. On the other hand, re-scanning a clean tree is bound by the file system meta data throughput. A parallel approach saves significant time here as well. This discovery phase is done in a thread pool too. This cuts the scan of a clean Linux source tree (91k files, 2.6 GB) with hot page cache and index from >250ms to ~60ms. If the tree is not in the page cache, hashing without index is 6x faster (17.8s -> 3.0s). See #281. --- .gitignore | 3 + pym/bob/utils.py | 31 +- pyproject.toml | 8 +- src/dirhasher/Cargo.lock | 322 +++++++ src/dirhasher/Cargo.toml | 20 + src/dirhasher/src/hasher.rs | 882 ++++++++++++++++++++ src/dirhasher/src/hasher/unix/helpers.rs | 24 + src/dirhasher/src/hasher/windows/helpers.rs | 22 + src/dirhasher/src/lib.rs | 85 ++ test/unit/test_utils_dirhasher.py | 57 +- 10 files changed, 1431 insertions(+), 23 deletions(-) create mode 100644 src/dirhasher/Cargo.lock create mode 100644 src/dirhasher/Cargo.toml create mode 100644 src/dirhasher/src/hasher.rs create mode 100644 src/dirhasher/src/hasher/unix/helpers.rs create mode 100644 src/dirhasher/src/hasher/windows/helpers.rs create mode 100644 src/dirhasher/src/lib.rs diff --git a/.gitignore b/.gitignore index 0edf74b3..b10020b7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ __pycache__/ /dist/ /htmlcov /pym/bob/version.py +/pym/bob/*.pyd +/pym/bob/*.so +/src/dirhasher/target/ diff --git a/pym/bob/utils.py b/pym/bob/utils.py index b2eb2780..d41aa4a5 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -682,7 +682,7 @@ def hashDirectory(self, path): self.__index.open() try: - return self.__hashDir(os.fsencode(path)) + return (self.__hashDir(path), self.size) finally: self.__index.close() @@ -701,17 +701,34 @@ def hashPath(self, path): self.__index.close() +def _hashDirWithSize(path, index, ignoreDirs): + global _hashDirWithSize + try: + from ._dirhasher import hash_directory_with_size + _hashDirWithSize = hash_directory_with_size + except ImportError: + _hashDirWithSize = lambda path, index, ignoreDirs: DirHasher(index, ignoreDirs).hashDirectory(path) + + return _hashDirWithSize(path, index, ignoreDirs) + +def _hashPath(path, index, ignoreDirs): + global _hashPath + try: + from ._dirhasher import hash_path + _hashPath = hash_path + except ImportError: + _hashPath = lambda path, index, ignoreDirs: DirHasher(index, ignoreDirs).hashPath(path) + + return _hashPath(path, index, ignoreDirs) + def hashDirectory(path, index=None, ignoreDirs=None): - return DirHasher(index, ignoreDirs).hashDirectory(path) + return _hashDirWithSize(path, index, ignoreDirs)[0] def hashDirectoryWithSize(path, index=None, ignoreDirs=None): - h = DirHasher(index, ignoreDirs) - retHash = h.hashDirectory(path) - retSize = h.size - return retHash, retSize + return _hashDirWithSize(path, index, ignoreDirs) def hashPath(path, index=None, ignoreDirs=None): - return DirHasher(index, ignoreDirs).hashPath(path) + return _hashPath(path, index, ignoreDirs) def binStat(path): st = os.stat(path) diff --git a/pyproject.toml b/pyproject.toml index d0600313..d5ec8714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,13 @@ [build-system] -requires = ["setuptools >= 40.0", "setuptools-scm", "Sphinx"] +requires = ["setuptools >= 40.0", "setuptools-scm", "Sphinx", "setuptools-rust"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "pym/bob/version.py" local_scheme = "no-local-version" + +[[tool.setuptools-rust.ext-modules]] +# See https://setuptools-rust.readthedocs.io/en/latest/reference.html +path = "src/dirhasher/Cargo.toml" +target = "bob._dirhasher" +optional = true diff --git a/src/dirhasher/Cargo.lock b/src/dirhasher/Cargo.lock new file mode 100644 index 00000000..43fd63d3 --- /dev/null +++ b/src/dirhasher/Cargo.lock @@ -0,0 +1,322 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bob-dirhasher" +version = "0.1.0" +dependencies = [ + "pyo3", + "sha1", + "tempfile", + "threadpool", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/src/dirhasher/Cargo.toml b/src/dirhasher/Cargo.toml new file mode 100644 index 00000000..19ec8b43 --- /dev/null +++ b/src/dirhasher/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bob-dirhasher" +version = "0.1.0" +edition = "2024" + +[dependencies] +pyo3 = "0.29.3" +sha1 = "0.11.0" +tempfile = "3" +threadpool = "1.8.1" + +[profile.release-lto] +inherits = "release" +lto = true + +[lib] +# See https://github.com/PyO3/pyo3 for details +name = "_dirhasher" # private module to be nested into Python package +crate-type = ["cdylib"] +path = "src/lib.rs" diff --git a/src/dirhasher/src/hasher.rs b/src/dirhasher/src/hasher.rs new file mode 100644 index 00000000..c091f778 --- /dev/null +++ b/src/dirhasher/src/hasher.rs @@ -0,0 +1,882 @@ +use sha1::{Digest, Sha1}; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File, FileType, Metadata}; +use std::io::{Error, ErrorKind, Read, Write}; +use std::path::{MAIN_SEPARATOR_STR, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; +use tempfile::NamedTempFile; +use threadpool; + +#[cfg(unix)] +mod unix { + pub mod helpers; + pub use std::os::unix::fs::MetadataExt; +} + +#[cfg(unix)] +use unix::*; + +#[cfg(windows)] +mod windows { + pub mod helpers; + pub use std::os::windows::fs::MetadataExt; +} + +#[cfg(windows)] +use windows::*; + +use helpers::*; // import the helpers from the unix/windows submodule + +pub type Sha1Sum = [u8; 20]; + +pub enum HashSum { + Empty, + Sha1(Sha1Sum), + RDev([u8; 8]), +} + +type Inode = usize; + +/// A warning in the style of the Python logging module: a format string with `%s` placeholders and +/// the matching arguments. +pub struct Warning { + pub msg: &'static str, + pub args: Vec, +} + +impl Warning { + /// Warning about a failed I/O operation on `path`. + fn io(msg: &'static str, path: &Path, err: &Error) -> Self { + Warning { + msg, + args: vec![format!("{}: {}", path.display(), err)], + } + } + + fn stat(path: &Path, err: &Error) -> Self { + Warning { + msg: "Cannot stat '%s': %s", + args: vec![path.display().to_string(), err.to_string()], + } + } +} + +/// Signals that the hashing was aborted by the host. +pub struct Aborted; + +/// Interface to the caller of the hasher. All methods are invoked on the calling thread. +pub trait Host { + /// Report a warning. An error aborts the hashing. + fn warn(&mut self, warning: Warning) -> Result<(), Aborted>; + + /// Invoked roughly every `POLL_INTERVAL` while hashing. An error aborts the hashing. + fn poll(&mut self) -> Result<(), Aborted>; +} + +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Messages from the worker threads to the calling thread. +enum Msg { + ReadDir(ReadDirResult), + Hashed(Inode, Sha1Sum), + Warning(Warning), +} + +/// Compact subset of `Metadata` with just the fields that are needed. +struct Stat { + file_type: FileType, + mode: u32, + size: u64, + ctime: i64, + mtime: i64, + dev: u64, + ino: u64, + rdev: u64, +} + +#[cfg(windows)] +fn win_time_to_unix_ns(t: u64) -> i64 { + // epoch offset - number of 100 ns ticks between 1601 and 1970 + ((t as i64) - 116444736000000000i64) * 100 +} + +#[cfg(windows)] +/// Convert file metadata info Python compatible "st_mode". +/// +/// Mimics _Py_attribute_data_to_stat() and win32_xstat_impl() from the CPython implementation. +fn win_mode_from_metadata(md: &Metadata, file_name: &OsStr) -> u32 { + let mut mode: u32 = if md.is_dir() { + 0o040000 // S_IFDIR + } else if md.is_symlink() { + 0o120000 // S_IFLNK + } else { + 0o100000 // S_IFREG + }; + + // FILE_ATTRIBUTE_DIRECTORY - this is set for symlinks on directories too + if (md.file_attributes() & 0x10) != 0 { + mode |= 0o111; + } + + // FILE_ATTRIBUTE_READONLY + if (md.file_attributes() & 0x1) != 0 { + mode |= 0o444; + } else { + mode |= 0o666; + } + + // Python makes certain files "executable". + if !md.is_dir() { + if let Some((_, ext)) = file_name.to_string_lossy().rsplit_once(".") { + let ext = ext.to_lowercase(); + if ext == ".exe" || ext == ".bat" || ext == ".cmd" || ext == ".com" { + mode |= 0o111; + } + } + } + + mode +} + +impl Stat { + #[cfg(unix)] + fn get_change_time(&self) -> i64 { + self.ctime + } + + #[cfg(windows)] + fn get_change_time(&self) -> i64 { + self.mtime + } +} + +impl Stat { + #[cfg(unix)] + fn from(md: &Metadata, _: &OsStr) -> Self { + Stat { + file_type: md.file_type(), + mode: md.mode(), + size: md.size(), + ctime: md.ctime() * 1_000_000_000 + md.ctime_nsec(), + mtime: md.mtime() * 1_000_000_000 + md.mtime_nsec(), + dev: md.dev(), + ino: md.ino(), + rdev: md.rdev(), + } + } + + #[cfg(windows)] + fn from(md: &Metadata, path: &OsStr) -> Self { + Stat { + file_type: md.file_type(), + mode: win_mode_from_metadata(md, path), + size: md.file_size(), + ctime: win_time_to_unix_ns(md.creation_time()), + mtime: win_time_to_unix_ns(md.last_write_time()), + // Fortunately the following information is also 0 in the Python implementation. We're + // using os.DirEntry.stat() and this does not populate these fields either. + dev: 0, // md.volume_serial_number() is unfortunately not a stable API + ino: 0, // md.file_index() neither + rdev: 0, // unused on Windows + } + } +} + +struct DirEntry { + name: OsString, + ino: Inode, +} + +type DirEntries = Vec; + +struct VNode { + stat: Stat, + children: DirEntries, + hash: Option, +} + +impl VNode { + fn new(stat: Stat) -> Self { + VNode { + stat, + children: Vec::new(), + hash: None, + } + } +} + +/// Determine the current time of the file system clock. +/// +/// File time stamps have a limited granularity. A file that is modified again in the same clock +/// tick after it was hashed would keep its stat data. Such "racily clean" entries must not be +/// trusted. Any file whose ctime is not older than the current file system time is thus not +/// recorded in the index. The ctime is used because it cannot be set by the user. Creating a file +/// gets us the current time in the right granularity, even on network file systems where the clock +/// of the server is used. +fn calc_ctime_limit(index_path: &Path) -> i64 { + // Use the directory of the index file. A bare file name has an empty parent, which means the + // current directory. If there is no parent at all, use the global temporary directory as + // fallback. + let dir = match index_path.parent() { + Some(p) if p.is_empty() => PathBuf::from("."), + Some(p) => p.to_path_buf(), + None => std::env::temp_dir(), + }; + let Ok(tmp_file) = tempfile::tempfile_in(dir) else { + return i64::MIN; + }; + + let Ok(metadata) = tmp_file.metadata() else { + return i64::MIN; + }; + + Stat::from(&metadata, index_path.as_os_str()).get_change_time() +} + +/// Hash a file in a worker thread. Stops early if the hashing was `cancelled`. +fn hash_file(path: &Path, cancelled: &AtomicBool, tx: &mpsc::Sender) -> Sha1Sum { + let mut hasher = Sha1::new(); + + match File::open(path) { + Ok(mut file) => { + let mut buffer = [0u8; 128 * 1024]; + while !cancelled.load(Ordering::Relaxed) { + match file.read(&mut buffer) { + Ok(0) => break, + Ok(len) => hasher.update(&buffer[..len]), + Err(err) => { + if err.kind() != ErrorKind::Interrupted { + let warning = Warning::io("Cannot hash file: %s", path, &err); + let _ = tx.send(Msg::Warning(warning)); + break; + } + } + } + } + } + Err(err) => { + let _ = tx.send(Msg::Warning(Warning::io( + "Cannot hash file: %s", + path, + &err, + ))); + } + } + + return hasher.finalize().into(); +} + +fn hash_link(path: &Path, host: &mut dyn Host) -> Result { + let mut hasher = Sha1::new(); + match path.read_link() { + Ok(destination) => hasher.update(os_string_to_bytes(&destination.into_os_string())), + Err(err) => host.warn(Warning::io("Cannot hash link: %s", path, &err))?, + } + Ok(hasher.finalize().into()) +} + +trait DirHasherIndex { + fn check(&mut self, path: &Path, stat: &Stat) -> Option; + fn update(&mut self, path: &Path, stat: &Stat, hash: &Sha1Sum); + /// Write back the updated index. + fn finish(&self) -> Result<(), Warning>; +} + +struct NullIndex {} + +impl DirHasherIndex for NullIndex { + fn check(&mut self, _path: &Path, _stat: &Stat) -> Option { + None + } + fn update(&mut self, _path: &Path, _stat: &Stat, _hash: &Sha1Sum) {} + fn finish(&self) -> Result<(), Warning> { + Ok(()) + } +} + +struct FileIndexEntry<'a> { + data: &'a [u8], +} + +const INDEX_SIGNATURE: &[u8; 4] = b"BOB2"; +const INDEX_ENTRY_SIZE: usize = 66; +const INDEX_OFF_CTIME: usize = 0; +const INDEX_OFF_MTIME: usize = 8; +const INDEX_OFF_DEV: usize = 16; +const INDEX_OFF_INO: usize = 24; +const INDEX_OFF_MODE: usize = 32; +const INDEX_OFF_SIZE: usize = 36; +const INDEX_OFF_DIGEST: usize = 44; +const INDEX_OFF_NAMELEN: usize = 64; + +impl FileIndexEntry<'_> { + fn new(data: &[u8]) -> Option> { + if data.len() < INDEX_ENTRY_SIZE { + return None; + } + let name_len_bytes: [u8; 2] = data[INDEX_OFF_NAMELEN..][0..2].try_into().unwrap(); + let name_len: usize = u16::from_ne_bytes(name_len_bytes).into(); + let len = INDEX_ENTRY_SIZE + name_len; + if data.len() < len { + return None; + } + Some(FileIndexEntry { data: &data[..len] }) + } + + fn get_name(&self) -> &[u8] { + &self.data[INDEX_ENTRY_SIZE..] + } + + fn len(&self) -> usize { + self.data.len() + } + + fn cmp_stat(&self, stat: &Stat) -> bool { + let ctime = i64::from_ne_bytes(self.data[INDEX_OFF_CTIME..][0..8].try_into().unwrap()); + let mtime = i64::from_ne_bytes(self.data[INDEX_OFF_MTIME..][0..8].try_into().unwrap()); + let dev = u64::from_ne_bytes(self.data[INDEX_OFF_DEV..][0..8].try_into().unwrap()); + let ino = u64::from_ne_bytes(self.data[INDEX_OFF_INO..][0..8].try_into().unwrap()); + let mode = u32::from_ne_bytes(self.data[INDEX_OFF_MODE..][0..4].try_into().unwrap()); + let size = u64::from_ne_bytes(self.data[INDEX_OFF_SIZE..][0..8].try_into().unwrap()); + + return stat.ctime == ctime + && stat.mtime == mtime + && stat.dev == dev + && stat.ino == ino + && stat.mode == mode + && stat.size == size; + } + + fn hash_sum(&self) -> Sha1Sum { + self.data[INDEX_OFF_DIGEST..][0..20].try_into().unwrap() + } +} + +struct FileIndex { + index_path: PathBuf, + ctime_limit: i64, + old_index: Vec, + new_index: Vec, + old_pos: usize, + mismatch: bool, +} + +impl FileIndex { + fn new(index_path: &Path) -> (FileIndex, Option) { + let mut warning = None; + let index = match fs::read(index_path) { + Ok(data) => data, + Err(err) => { + if err.kind() != ErrorKind::NotFound { + warning = Some(Warning::io("Cannot open hash cache: %s", index_path, &err)); + }; + Vec::new() + } + }; + + let mismatch = index.len() < INDEX_SIGNATURE.len() || index[0..4] != *INDEX_SIGNATURE; + let mut new_index: Vec = Vec::with_capacity(index.len()); + new_index.extend_from_slice(INDEX_SIGNATURE); + + let index = FileIndex { + index_path: index_path.to_path_buf(), + ctime_limit: calc_ctime_limit(index_path), + old_index: index, + new_index: new_index, + old_pos: if mismatch { 0 } else { 4 }, + mismatch: mismatch, + }; + (index, warning) + } + + fn write_back(&self) -> std::io::Result<()> { + if !self.mismatch { + return Ok(()); + } + let mut file = NamedTempFile::new_in( + self.index_path + .parent() + .ok_or(Error::new(ErrorKind::NotADirectory, "No parent directory!"))?, + )?; + let _ = file.write_all(&self.new_index)?; + file.persist(&self.index_path)?; + Ok(()) + } + + fn search_entry(&mut self, name: &OsStr) -> Option> { + let name = os_str_to_bytes(name); + loop { + let entry = FileIndexEntry::new(&self.old_index[self.old_pos..])?; + let entry_name = entry.get_name(); + if entry_name == name { + self.old_pos += entry.len(); + return Some(entry); + } else if entry_name < name { + self.old_pos += entry.len(); + self.mismatch = true; + } else { + self.mismatch = true; + return None; + } + } + } + + fn add_entry(&mut self, name: &OsStr, stat: &Stat, hash_sum: &Sha1Sum) { + // Entries that are of the same age than the current file system time must be dropped. + // Changes in the same "file system tick" cannot be distinguished. Entries that are newer + // have been recently modified. Their checksum is most probably wrong. + if stat.ctime >= self.ctime_limit { + return; + } + + self.new_index.extend_from_slice(&stat.ctime.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.mtime.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.dev.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.ino.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.mode.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.size.to_ne_bytes()); + self.new_index.extend_from_slice(hash_sum); + + let name_bytes = os_str_to_bytes(name); + let name_len: u16 = name_bytes.len().try_into().unwrap(); + self.new_index.extend_from_slice(&name_len.to_ne_bytes()); + self.new_index.extend_from_slice(&name_bytes); + } +} + +impl DirHasherIndex for FileIndex { + fn check(&mut self, path: &Path, stat: &Stat) -> Option { + let entry_path = path.as_os_str(); + if let Some(cached_entry) = self.search_entry(entry_path) { + if cached_entry.cmp_stat(stat) { + return Some(cached_entry.hash_sum()); + } + } + + // Either there was no cache entry or it did not match. + self.mismatch = true; + None + } + + fn update(&mut self, path: &Path, stat: &Stat, hash: &Sha1Sum) { + if self.mismatch { + self.add_entry(path.as_os_str(), stat, hash); + } + } + + fn finish(&self) -> Result<(), Warning> { + self.write_back() + .map_err(|err| Warning::io("Cannot write hash cache: %s", &self.index_path, &err)) + } +} + +/// Reusable path buffer for the tree traversal. Holds the prefix, a separator +/// and the relative path. Children are appended and truncated again. +struct PathBuilder { + buf: Vec, + stack: Vec, + prefix_len: usize, + base: usize, +} + +impl PathBuilder { + fn new(prefix: &Path) -> Self { + let mut buf = prefix.as_os_str().as_encoded_bytes().to_vec(); + let prefix_len = buf.len(); + if !prefix.is_empty() && !prefix.ends_with(MAIN_SEPARATOR_STR) { + buf.extend_from_slice(OsStr::new(MAIN_SEPARATOR_STR).as_encoded_bytes()); + } + let base = buf.len(); + PathBuilder { + buf, + stack: Vec::new(), + prefix_len, + base, + } + } + + /// Path relative to the prefix. + fn rel(&self) -> &Path { + // SAFETY: + // - all bytes are from OsStr::as_encoded_bytes() + // - no byte sequences is split + Path::new(unsafe { OsStr::from_encoded_bytes_unchecked(&self.buf[self.base..]) }) + } + + /// Full path including the prefix. + fn full(&self) -> &Path { + let end = if self.buf.len() == self.base { + self.prefix_len + } else { + self.buf.len() + }; + // SAFETY: + // - all bytes are from OsStr::as_encoded_bytes() + // - no byte sequences is split + Path::new(unsafe { OsStr::from_encoded_bytes_unchecked(&self.buf[..end]) }) + } + + /// Add new path segment. Assumes trailing separator for directories. + fn push(&mut self, name: &OsStr) { + self.stack.push(self.buf.len()); + self.buf.extend_from_slice(name.as_encoded_bytes()); + } + + /// Pop last path segment + fn pop(&mut self) { + let len = self.stack.pop().expect("path stack must not be empty"); + self.buf.truncate(len); + } +} + +struct ReadDirEntry { + name: OsString, + stat: Stat, +} + +struct ReadDirResult { + path: PathBuf, + ino: Inode, + entries: Vec, +} + +struct DirHasher<'a> { + host: &'a mut dyn Host, + index: Box, + size: u64, + ignore_dirs: Vec, + ignore_files: Vec, + vnodes: Vec, + pool: threadpool::ThreadPool, + /// Number of worker jobs whose result is still outstanding. + pending: u32, + tx: mpsc::Sender, + rx: mpsc::Receiver, + cancelled: Arc, + next_poll: Instant, +} + +impl<'a> DirHasher<'a> { + fn new( + host: &'a mut dyn Host, + index_path: Option<&Path>, + add_ignore_dirs: Option>, + ) -> Result { + let mut ignore_dirs: Vec = + vec![".git".into(), ".portage-cache".into(), ".svn".into()]; + if let Some(add_ignore_dirs) = add_ignore_dirs { + ignore_dirs.extend(add_ignore_dirs); + } + let index: Box = match index_path { + Some(index_path) => { + let (index, warning) = FileIndex::new(index_path); + if let Some(warning) = warning { + host.warn(warning)?; + } + Box::new(index) + } + None => Box::new(NullIndex {}), + }; + let (tx, rx) = mpsc::channel::(); + Ok(DirHasher { + host, + index, + size: 0, + ignore_dirs, + ignore_files: vec!["BaseDirList.txt".into()], + vnodes: Vec::new(), + pool: threadpool::Builder::new().build(), + pending: 0, + tx, + rx, + cancelled: Arc::new(AtomicBool::new(false)), + next_poll: Instant::now() + POLL_INTERVAL, + }) + } + + /// Poll the host if `POLL_INTERVAL` has elapsed since the last time. + fn maybe_poll(&mut self) -> Result<(), Aborted> { + let now = Instant::now(); + if now >= self.next_poll { + self.host.poll()?; + self.next_poll = now + POLL_INTERVAL; + } + Ok(()) + } + + fn handle_msg(&mut self, msg: Msg) -> Result<(), Aborted> { + match msg { + Msg::ReadDir(res) => { + self.pending -= 1; + self.process_readdir(res); + } + Msg::Hashed(ino, hash) => { + self.pending -= 1; + self.vnodes[ino].hash = Some(HashSum::Sha1(hash)); + } + Msg::Warning(warning) => self.host.warn(warning)?, + } + Ok(()) + } + + /// Handle the messages of the workers until all jobs are finished. + fn wait_pending(&mut self) -> Result<(), Aborted> { + while self.pending > 0 { + let timeout = self.next_poll.saturating_duration_since(Instant::now()); + match self.rx.recv_timeout(timeout) { + Ok(msg) => self.handle_msg(msg)?, + Err(mpsc::RecvTimeoutError::Timeout) => (), + Err(mpsc::RecvTimeoutError::Disconnected) => { + unreachable!("sender is owned by self") + } + } + self.maybe_poll()?; + } + Ok(()) + } + + fn schedule_readdir(&mut self, path: PathBuf, ino: Inode) { + let tx = self.tx.clone(); + let cancelled = self.cancelled.clone(); + self.pending += 1; + self.pool.execute(move || { + if cancelled.load(Ordering::Relaxed) { + return; + } + let mut entries = Vec::new(); + match fs::read_dir(&path) { + Ok(iter) => { + for entry in iter { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + let warning = Warning::io("Cannot list directory: %s", &path, &err); + let _ = tx.send(Msg::Warning(warning)); + continue; + } + }; + let metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(err) => { + let _ = tx.send(Msg::Warning(Warning::stat(&entry.path(), &err))); + continue; + } + }; + let name = entry.file_name(); + let stat = Stat::from(&metadata, name.as_os_str()); + entries.push(ReadDirEntry { name, stat }); + } + } + Err(err) => { + let warning = Warning::io("Cannot list directory: %s", &path, &err); + let _ = tx.send(Msg::Warning(warning)); + } + }; + // The receiver is gone if the hashing was aborted. + let _ = tx.send(Msg::ReadDir(ReadDirResult { path, ino, entries })); + }); + } + + fn process_readdir(&mut self, res: ReadDirResult) { + let mut dir: DirEntries = Vec::with_capacity(res.entries.len()); + + for entry in res.entries { + let mut name = entry.name; + if entry.stat.file_type.is_dir() { + if self.ignore_dirs.contains(&name) { + continue; + } + name.push(MAIN_SEPARATOR_STR); + } else if self.ignore_files.contains(&name) { + continue; + } + + let ino: Inode = self.vnodes.len(); + if entry.stat.file_type.is_dir() { + self.schedule_readdir(res.path.join(&name), ino); + }; + self.vnodes.push(VNode::new(entry.stat)); + dir.push(DirEntry { name, ino }); + } + + dir.sort_unstable_by(|e1, e2| e1.name.cmp(&e2.name)); + self.vnodes[res.ino].children = dir; + } + + /// Discover the file system tree at `path`. The caller decides whether a symlink at `path` + /// itself is followed by passing the respective `metadata`. Symlinks below `path` are never + /// followed. + fn discover(&mut self, path: &Path, metadata: &Metadata) -> Result<(), Aborted> { + let stat = Stat::from(metadata, path.as_os_str()); + if stat.file_type.is_dir() { + let ino: Inode = self.vnodes.len(); + self.vnodes.push(VNode::new(stat)); + self.schedule_readdir(PathBuf::from(path), ino); + self.wait_pending()?; + } else { + self.vnodes.push(VNode::new(stat)); + }; + + Ok(()) + } + + /// Start collecting hash sums on the discovered file system tree. If unkown files are + /// encountered, schedule their hashing in the thread pool. + fn collect_node(&mut self, path: &mut PathBuilder, ino: Inode) -> Result<(), Aborted> { + self.size += self.vnodes[ino].stat.size; + + let file_type = self.vnodes[ino].stat.file_type; + if file_type.is_file() { + match self.index.check(path.rel(), &self.vnodes[ino].stat) { + Some(hash) => self.vnodes[ino].hash = Some(HashSum::Sha1(hash)), + None => { + let tx = self.tx.clone(); + let cancelled = self.cancelled.clone(); + let path = path.full().to_path_buf(); + self.pending += 1; + self.pool.execute(move || { + if cancelled.load(Ordering::Relaxed) { + return; + } + let hash = hash_file(&path, &cancelled, &tx); + let _ = tx.send(Msg::Hashed(ino, hash)); + }); + } + } + } else if file_type.is_dir() { + for i in 0..self.vnodes[ino].children.len() { + let child_ino = self.vnodes[ino].children[i].ino; + path.push(&self.vnodes[ino].children[i].name); + self.collect_node(path, child_ino)?; + path.pop(); + } + } else if file_type.is_symlink() { + let hash = match self.index.check(path.rel(), &self.vnodes[ino].stat) { + Some(hash) => hash, + None => hash_link(path.full(), &mut *self.host)?, + }; + self.vnodes[ino].hash = Some(HashSum::Sha1(hash)); + } else if is_block_device(&file_type) || is_char_device(&file_type) { + let hash = self.vnodes[ino].stat.rdev.to_le_bytes(); + self.vnodes[ino].hash = Some(HashSum::RDev(hash)); + } else { + self.vnodes[ino].hash = Some(HashSum::Empty); + if !is_fifo(&file_type) { + self.host.warn(Warning { + msg: "Unknown file: %s", + args: vec![path.full().display().to_string()], + })?; + } + } + + Ok(()) + } + + /// Calculate the final hash sum. All VNodes except directories have a hash sum already. + fn finalize_node(&mut self, path: &mut PathBuilder, ino: Inode) { + let file_type = self.vnodes[ino].stat.file_type; + if file_type.is_dir() { + let mut hasher = Sha1::new(); + for i in 0..self.vnodes[ino].children.len() { + let child_ino = self.vnodes[ino].children[i].ino; + path.push(&self.vnodes[ino].children[i].name); + self.finalize_node(path, child_ino); + path.pop(); + hasher.update(&self.vnodes[child_ino].stat.mode.to_le_bytes()); + match self.vnodes[child_ino].hash.as_ref().unwrap() { + HashSum::Empty => (), + HashSum::Sha1(hash) => hasher.update(hash), + HashSum::RDev(hash) => hasher.update(hash), + }; + hasher.update(os_string_to_bytes(&self.vnodes[ino].children[i].name)); + } + self.vnodes[ino].hash = Some(HashSum::Sha1(hasher.finalize().into())); + } + + if file_type.is_file() || file_type.is_symlink() { + if let Some(HashSum::Sha1(hash)) = self.vnodes[ino].hash { + self.index.update(path.rel(), &self.vnodes[ino].stat, &hash); + } + } + } + + /// Hash the tree at `path` and write back the updated index. + fn hash_tree(&mut self, path: &Path, metadata: &Metadata) -> Result { + self.discover(path, metadata)?; + self.collect_node(&mut PathBuilder::new(path), 0)?; + self.wait_pending()?; + self.finalize_node(&mut PathBuilder::new(Path::new("")), 0); + if let Err(warning) = self.index.finish() { + self.host.warn(warning)?; + } + Ok(self.vnodes[0].hash.take().unwrap()) + } + + fn hash_dir(mut self, path: &Path) -> Result<(Vec, u64), Aborted> { + // A symlink to a directory is followed, like os.scandir() does in Python. + let metadata = path.metadata().and_then(|md| { + if md.is_dir() { + Ok(md) + } else { + Err(Error::from(ErrorKind::NotADirectory)) + } + }); + let metadata = match metadata { + Ok(metadata) => metadata, + Err(err) => { + self.host + .warn(Warning::io("Cannot list directory: %s", path, &err))?; + return Ok((Sha1::digest(b"").to_vec(), 0)); + } + }; + + let HashSum::Sha1(hash) = self.hash_tree(path, &metadata)? else { + unreachable!("directories have a SHA1 hash"); + }; + Ok((hash.to_vec(), self.size)) + } + + fn hash_path(mut self, path: &Path) -> Result, Aborted> { + let metadata = match path.symlink_metadata() { + Ok(metadata) => metadata, + Err(err) => { + self.host.warn(Warning::stat(path, &err))?; + return Ok(Vec::new()); + } + }; + + let hash = match self.hash_tree(path, &metadata)? { + HashSum::Empty => Vec::new(), + HashSum::Sha1(hash) => hash.to_vec(), + HashSum::RDev(hash) => hash.to_vec(), + }; + Ok(hash) + } +} + +impl Drop for DirHasher<'_> { + fn drop(&mut self) { + // Let the outstanding worker jobs finish quickly if the hashing was aborted. + self.cancelled.store(true, Ordering::Relaxed); + } +} + +pub fn hash_dir( + path: &Path, + index: Option<&Path>, + ignore_dirs: Option>, + host: &mut dyn Host, +) -> Result<(Vec, u64), Aborted> { + DirHasher::new(host, index, ignore_dirs)?.hash_dir(path) +} + +pub fn hash_path( + path: &Path, + index: Option<&Path>, + ignore_dirs: Option>, + host: &mut dyn Host, +) -> Result, Aborted> { + DirHasher::new(host, index, ignore_dirs)?.hash_path(path) +} diff --git a/src/dirhasher/src/hasher/unix/helpers.rs b/src/dirhasher/src/hasher/unix/helpers.rs new file mode 100644 index 00000000..ad110f18 --- /dev/null +++ b/src/dirhasher/src/hasher/unix/helpers.rs @@ -0,0 +1,24 @@ +use std::ffi::{OsStr, OsString}; +use std::fs::FileType; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::FileTypeExt; + +pub fn is_block_device(ft: &FileType) -> bool { + ft.is_block_device() +} + +pub fn is_char_device(ft: &FileType) -> bool { + ft.is_char_device() +} + +pub fn is_fifo(ft: &FileType) -> bool { + ft.is_fifo() +} + +pub fn os_str_to_bytes(s: &OsStr) -> &[u8] { + s.as_bytes() +} + +pub fn os_string_to_bytes(s: &OsString) -> &[u8] { + os_str_to_bytes(s.as_os_str()) +} diff --git a/src/dirhasher/src/hasher/windows/helpers.rs b/src/dirhasher/src/hasher/windows/helpers.rs new file mode 100644 index 00000000..7f41a93c --- /dev/null +++ b/src/dirhasher/src/hasher/windows/helpers.rs @@ -0,0 +1,22 @@ +use std::ffi::{OsStr, OsString}; +use std::fs::FileType; + +pub fn is_block_device(_: &FileType) -> bool { + false +} + +pub fn is_char_device(_: &FileType) -> bool { + false +} + +pub fn is_fifo(_: &FileType) -> bool { + false +} + +pub fn os_str_to_bytes(s: &OsStr) -> &[u8] { + s.to_str().expect("must not fail").as_bytes() +} + +pub fn os_string_to_bytes(s: &OsString) -> &[u8] { + os_str_to_bytes(s.as_os_str()) +} diff --git a/src/dirhasher/src/lib.rs b/src/dirhasher/src/lib.rs new file mode 100644 index 00000000..4b7d4fda --- /dev/null +++ b/src/dirhasher/src/lib.rs @@ -0,0 +1,85 @@ +use pyo3::prelude::*; + +mod hasher; + +#[pymodule] +mod _dirhasher { + use crate::hasher::{Aborted, Host, Warning}; + use pyo3::prelude::*; + use pyo3::types::{PyBytes, PyTuple}; + use std::ffi::OsString; + use std::path::PathBuf; + + /// Connects the hasher with Python while the GIL is released. The GIL is re-acquired briefly to + /// log warnings and to handle signals. The first Python exception aborts the hashing. + #[derive(Default)] + struct PyHost { + err: Option, + } + + impl PyHost { + fn catch(&mut self, res: PyResult<()>) -> Result<(), Aborted> { + res.map_err(|err| { + self.err = Some(err); + Aborted + }) + } + + fn into_result(self, res: Result) -> PyResult { + res.map_err(|Aborted| self.err.expect("exception recorded on abort")) + } + } + + impl Host for PyHost { + /// Forward the warning to the logger of the Python implementation. + fn warn(&mut self, mut warning: Warning) -> Result<(), Aborted> { + let res = Python::attach(|py| { + let logger = py + .import("logging")? + .call_method1("getLogger", ("bob.utils",))?; + warning.args.insert(0, warning.msg.to_string()); + logger.call_method1("warning", PyTuple::new(py, warning.args)?)?; + Ok(()) + }); + self.catch(res) + } + + /// Run the Python signal handlers, e.g. to raise KeyboardInterrupt on Ctrl-C. + fn poll(&mut self) -> Result<(), Aborted> { + let res = Python::attach(|py| py.check_signals()); + self.catch(res) + } + } + + #[pyfunction] + fn hash_directory_with_size<'py>( + py: Python<'py>, + path: PathBuf, + index: Option, + ignore_dirs: Option>, + ) -> PyResult<(Bound<'py, PyBytes>, u64)> { + let ignore_dirs: Option> = + ignore_dirs.map(|v| v.into_iter().map(PathBuf::into_os_string).collect()); + let mut host = PyHost::default(); + let res = + py.detach(|| crate::hasher::hash_dir(&path, index.as_deref(), ignore_dirs, &mut host)); + let (hash, size) = host.into_result(res)?; + Ok((PyBytes::new(py, &hash), size)) + } + + #[pyfunction] + fn hash_path<'py>( + py: Python<'py>, + path: PathBuf, + index: Option, + ignore_dirs: Option>, + ) -> PyResult> { + let ignore_dirs: Option> = + ignore_dirs.map(|v| v.into_iter().map(PathBuf::into_os_string).collect()); + let mut host = PyHost::default(); + let res = + py.detach(|| crate::hasher::hash_path(&path, index.as_deref(), ignore_dirs, &mut host)); + let hash = host.into_result(res)?; + Ok(PyBytes::new(py, &hash)) + } +} diff --git a/test/unit/test_utils_dirhasher.py b/test/unit/test_utils_dirhasher.py index ab2784b2..789b58fb 100644 --- a/test/unit/test_utils_dirhasher.py +++ b/test/unit/test_utils_dirhasher.py @@ -4,13 +4,30 @@ # SPDX-License-Identifier: GPL-3.0-or-later from tempfile import NamedTemporaryFile, TemporaryDirectory -from unittest import TestCase +from unittest import TestCase, skipIf from unittest.mock import MagicMock, mock_open, patch import binascii import os import sys -from bob.utils import hashFile, hashDirectory, binStat +from bob.utils import hashFile, binStat, DirHasher + +try: + from bob._dirhasher import hash_directory_with_size, hash_path + def rustHashDirectory(path, index=None, ignoreDirs=None): + return hash_directory_with_size(path, index, ignoreDirs)[0] + def rustHashPath(path, index=None, ignoreDirs=None): + return hash_path(path, index, ignoreDirs) +except ImportError: + rustHashDirectory = None + rustHashPath = None + +def pyHashDirectory(path, index=None, ignoreDirs=None): + return DirHasher(index, ignoreDirs).hashDirectory(path)[0] + +def pyHashPath(path, index=None, ignoreDirs=None): + return DirHasher(index, ignoreDirs).hashPath(path) + class TestHashFile(TestCase): def testBigFile(self): @@ -41,7 +58,7 @@ def makeOsScandir(entries): m.return_value = OsScandirList(entries) return m -class TestHashDir(TestCase): +class CommonHashDirTests: def setUp(self): self.umask = os.umask(0o022) @@ -60,7 +77,7 @@ def testDirAndFile(self): with open(os.path.join(tmp, "dir", "file"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) self.assertEqual(len(sum1), 20) # Result depends on path separator character ('/' vs. '\') if sys.platform == "win32": @@ -69,7 +86,7 @@ def testDirAndFile(self): else: self.assertEqual(sum1, binascii.unhexlify( "640f516de78fba0b6d2ddde4451000f142d06b0d")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertEqual(sum1, sum2) def testRenameDirectory(self): @@ -80,9 +97,9 @@ def testRenameDirectory(self): with open(os.path.join(tmp, "dir", "file"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) os.rename(os.path.join(tmp, "dir"), os.path.join(tmp, "foo")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertNotEqual(sum1, sum2) def testRenameFile(self): @@ -92,9 +109,9 @@ def testRenameFile(self): with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) os.rename(os.path.join(tmp, "foo"), os.path.join(tmp, "bar")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertNotEqual(sum1, sum2) def testRewriteFile(self): @@ -105,20 +122,25 @@ def testRewriteFile(self): with TemporaryDirectory() as tmp: with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp, index) + sum1 = self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'qwer') - sum2 = hashDirectory(tmp, index) + sum2 = self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') self.assertNotEqual(sum1, sum2) + +class TestHashDirPython(CommonHashDirTests, TestCase): + hashDirectory = staticmethod(pyHashDirectory) + hashPath = staticmethod(pyHashPath) + def testBigIno(self): """Test that index handles big inode numbers as found on Windows""" @@ -148,7 +170,7 @@ def testBigIno(self): f.write(b'abc') with patch('os.scandir', entries): - hashDirectory(tmp, index) + self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') @@ -182,7 +204,7 @@ def testOldFile(self): pass with patch('os.scandir', entries): - h = hashDirectory(tmp, index) + h = self.hashDirectory(tmp, index) with patch('os.stat', mock_lstat): b = binStat("whatever") @@ -215,7 +237,7 @@ def testBlockDev(self): with TemporaryDirectory() as indexDir: index = os.path.join(indexDir, "index.bin") with patch('os.scandir', entries): - h = hashDirectory("whatever", index) + h = self.hashDirectory("whatever", index) self.assertEqual(h, b'\xe8\x8e\xad\x9bv\xcbt\xc4\xcd\xa7x\xdb\xde\x96\xab@\x18\xb1\xdcX') @@ -245,7 +267,12 @@ def testChrDev(self): with TemporaryDirectory() as indexDir: index = os.path.join(indexDir, "index.bin") with patch('os.scandir', entries): - h = hashDirectory("whatever", index) + h = self.hashDirectory("whatever", index) self.assertEqual(h, b"\x9b\x98~\xa5\xd5\xc4\x1e\xe29'\x8d\x1e\xe1\x12\xdd\xf4\xa51\xf5d") +@skipIf(rustHashDirectory is None, "rust DirHasher implementation not available") +class TestHashDirRust(CommonHashDirTests, TestCase): + hashDirectory = staticmethod(rustHashDirectory) + hashPath = staticmethod(rustHashPath) + From b95cbc0683af2ff3767e9a39f83e6f36d15a25f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 29 Oct 2026 22:39:55 +0100 Subject: [PATCH 09/12] test: add more dirhasher tests Add additional common tests for both the Python and Rust implementation. Also add tests that compare the result and index file of both implementations for a given tree. They should behave the same. --- test/unit/test_utils_dirhasher.py | 191 +++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 1 deletion(-) diff --git a/test/unit/test_utils_dirhasher.py b/test/unit/test_utils_dirhasher.py index 789b58fb..70e34fb0 100644 --- a/test/unit/test_utils_dirhasher.py +++ b/test/unit/test_utils_dirhasher.py @@ -7,10 +7,12 @@ from unittest import TestCase, skipIf from unittest.mock import MagicMock, mock_open, patch import binascii +import hashlib import os import sys -from bob.utils import hashFile, binStat, DirHasher +import time +from bob.utils import hashFile, binStat, DirHasher, canSymlink try: from bob._dirhasher import hash_directory_with_size, hash_path @@ -136,6 +138,58 @@ def testRewriteFile(self): self.assertNotEqual(sum1, sum2) + def testMissingIndexDir(self): + """Index in non-existing directory just issues a warning""" + with TemporaryDirectory() as tmp: + index = os.path.join(tmp, "missing", "index.bin") + with open(os.path.join(tmp, "file"), "wb") as f: + f.write(b'abc') + with self.assertLogs(level='WARNING') as cm: + if sys.platform == "win32": + expect = b'\x88\xee\x84\x87Y\xb21\xaa\x84\x98\x05\xc9&\xcf#4\x05\xe2\xab\x15' + else: + expect = b'\x039\xa9+\xebb\xaf\xbe\x11>@\xf7\xeeMl}\xb6\xd0\x9a\x90' + self.assertEqual(self.hashDirectory(tmp, index), expect) + self.assertTrue(cm.records[0].getMessage().startswith("Cannot write hash cache:")) + + def testBadIndex(self): + """A corrupted index is ignored""" + with TemporaryDirectory() as tmp: + index = os.path.join(tmp, "index.bin") + with open(index, "wb") as f: + f.write(b'garbage') + + ws = os.path.join(tmp, "dir") + os.mkdir(ws) + with open(os.path.join(ws, "file"), 'wb') as f: + f.write(b'abc') + + if sys.platform == "win32": + expect = b'\x88\xee\x84\x87Y\xb21\xaa\x84\x98\x05\xc9&\xcf#4\x05\xe2\xab\x15' + else: + expect = b'\x039\xa9+\xebb\xaf\xbe\x11>@\xf7\xeeMl}\xb6\xd0\x9a\x90' + self.assertEqual(self.hashDirectory(ws, index), expect) + + with open(index, "rb") as f: + self.assertEqual(f.read(4), b'BOB2') + + def testHashPath(self): + """Files, symlinks and missing paths can be hashed directly""" + with TemporaryDirectory() as tmp: + fn = os.path.join(tmp, "file") + with open(fn, "wb") as f: + f.write(b'abc') + os.symlink("file", os.path.join(tmp, "link")) + self.assertEqual(self.hashPath(fn), binascii.unhexlify( + "a9993e364706816aba3e25717850c26c9cd0d89d")) + self.assertEqual(self.hashPath(os.path.join(tmp, "link")), + hashlib.sha1(b'file').digest()) + self.assertEqual(self.hashPath(tmp), self.hashDirectory(tmp)) + with self.assertLogs(level='WARNING') as cm: + self.assertEqual(self.hashPath(os.path.join(tmp, "nx")), b'') + self.assertTrue(cm.records[0].getMessage().startswith("Cannot stat '")) + + class TestHashDirPython(CommonHashDirTests, TestCase): hashDirectory = staticmethod(pyHashDirectory) @@ -276,3 +330,138 @@ class TestHashDirRust(CommonHashDirTests, TestCase): hashDirectory = staticmethod(rustHashDirectory) hashPath = staticmethod(rustHashPath) +@skipIf(rustHashDirectory is None, "rust DirHasher implementation not available") +class TestHashDirCompat(TestCase): + """Compare rust implementation against Python implementation""" + + def setUp(self): + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = root = os.path.join(self.tmp.name, "root") + def mk(path, content=b''): + path = os.path.join(root, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(content) + # Names that exercise the sort order of files vs. directories + mk("a-b", b'1') + mk("a/x", b'2') + mk("a0", b'3') + os.chmod(os.path.join(root, "a0"), 0o755) + mk("a.c/y", b'4') + mk("big", os.urandom(1024*1024+17)) + mk("sub/deep/er/file", b'deep') + mk("sub/BaseDirList.txt", b'ignored') + mk(".git/config", b'ignored') + mk("ign/file", b'ignored by option') + mk("sub/ign/file", b'ignored by option') + if sys.platform != "win32": + # On Windows, invalid UTF-8 sequences are not allowed + mk(os.fsdecode(b"y\xe4\xf6\xfc-latin1"), b'5') + mk("Umlaut-\u00e4\u00f6\u00fc", b'6') + # Windows does not support FIFOs + os.mkfifo(os.path.join(root, "fifo")) + os.mkdir(os.path.join(root, "empty")) + if canSymlink(): + os.symlink("a/x", os.path.join(root, "link")) + os.symlink("dangling", os.path.join(root, "sub", "dangling")) + os.symlink("sub", os.path.join(root, "dirlink")) + + def check(self, isDir, path, index=False, ignoreDirs=None): + results = [] + impls = ["py", "rust"] + funs = [pyHashDirectory, rustHashDirectory] if isDir else [pyHashPath, rustHashPath] + for name, func in zip(impls, funs): + indexPath = os.path.join(self.tmp.name, name + ".bin") if index else None + results.append(func(path, indexPath, ignoreDirs)) + results.append(func(path, indexPath, ignoreDirs)) # run again with index cache + self.assertEqual(results[0], results[1]) + self.assertEqual(results[0], results[2]) + self.assertEqual(results[0], results[3]) + return results[0] + + def testHashDirectory(self): + d1 = self.check(True, self.root) + d2 = self.check(True, self.root, ignoreDirs=["ign"]) + self.assertNotEqual(d1, d2) + self.check(True, os.path.join(self.root, "dirlink")) + self.check(True, os.path.join(self.root, "sub")) + + def testManyFiles(self): + """Enough files to keep all threads busy and to prefetch files""" + for i in range(500): + d = os.path.join(self.root, "many", str(i % 7)) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, str(i)), "wb") as f: + f.write(os.urandom(i * 97)) + self.check("hashDirectory", self.root) + self.check("hashDirectory", self.root, True) + + def waitClockTick(self): + """Wait until the file system clock has advanced. + + Files that are not older than the file system time are "racily clean" + and are not recorded in the index. Make sure all files are old enough + so that the index content is deterministic. + """ + def change_time(): + with NamedTemporaryFile(dir=self.tmp.name) as f: + if sys.platform == "win32": + return os.fstat(f.fileno()).st_mtime_ns + else: + return os.fstat(f.fileno()).st_ctime_ns + start = change_time() + while change_time() <= start: + time.sleep(0.001) + + def testHashDirectoryIndex(self): + """Both implementations create the same index""" + self.waitClockTick() + d1 = self.check(True, self.root, True) + self.assertEqual(d1, self.check(True, self.root)) + with open(os.path.join(self.tmp.name, "py.bin"), "rb") as f: + pyIndex = f.read() + with open(os.path.join(self.tmp.name, "rust.bin"), "rb") as f: + rustIndex = f.read() + self.assertEqual(pyIndex, rustIndex) + + def checkIndexUpdate(self): + """Hash once with each implementation and compare the updated indices. + + A second run would fix up a wrongly updated index. Hence each + implementation is invoked just once. + """ + pyIndex = os.path.join(self.tmp.name, "py.bin") + rustIndex = os.path.join(self.tmp.name, "rust.bin") + self.assertEqual(pyHashDirectory(self.root, pyIndex), + rustHashDirectory(self.root, rustIndex)) + with open(pyIndex, "rb") as f: + pyIndex = f.read() + with open(rustIndex, "rb") as f: + rustIndex = f.read() + self.assertEqual(pyIndex, rustIndex) + return pyIndex + + def testHashDirectoryIndexUpdate(self): + """Removed and added files update the index identically""" + self.waitClockTick() + i1 = self.checkIndexUpdate() + + # Entries of removed files are dropped + os.unlink(os.path.join(self.root, "a-b")) + i2 = self.checkIndexUpdate() + self.assertNotIn(b"a-b", i2) + + # Appending after the last entry keeps all other entries + with open(os.path.join(self.root, os.fsdecode(b"zzz-end")), "wb") as f: + f.write(b'end') + self.waitClockTick() + i3 = self.checkIndexUpdate() + self.assertTrue(i3.startswith(i2)) + self.assertIn(b"zzz-end", i3) + + def testHashPath(self): + for p in ("big", "link", "fifo", "sub", "dirlink", "sub/dangling"): + self.check(False, os.path.join(self.root, p)) + self.check(False, os.path.join(self.root, p), True) + From feea5b286bcb9858aedad28f3b309700415d8158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 29 Oct 2026 22:15:31 +0100 Subject: [PATCH 10/12] scripts: make sure hasher plugin is always up-to-date When using Bob from sources, changes to the rust directory hasher should lead to a rebuild of the Python plugin. Do that before executing any commands or tests. --- pym/bob/develop/make.py | 26 +++++++++++++++++++++++++- pym/bob/scripts.py | 7 +++++++ test/run-tests.sh | 5 +++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/pym/bob/develop/make.py b/pym/bob/develop/make.py index 1ab2818b..946f017d 100644 --- a/pym/bob/develop/make.py +++ b/pym/bob/develop/make.py @@ -6,7 +6,7 @@ import os import sys -__all__ = ('makeSandboxHelper', 'makeManpages') +__all__ = ('makeSandboxHelper', 'makeManpages', 'makeDirHasher') def findYoungestInDir(path, exclude): ret = 0 @@ -53,6 +53,30 @@ def makeSandboxHelper(): return resultPath +def makeDirHasher(): + if sys.platform == "win32": + cargoName = "_dirhasher.dll" + else: + cargoName = "lib_dirhasher.so" + + import sysconfig + bobRoot = getBobRoot() + resultPath = os.path.join(bobRoot, "pym", "bob", + "_dirhasher" + sysconfig.get_config_var("EXT_SUFFIX")) + inputPath = os.path.join(bobRoot, "src", "dirhasher") + resultDate = findYoungest(resultPath) + inputDate = findYoungest(inputPath) + if inputDate > resultDate: + import shutil + if shutil.which("cargo") is None: + return + + import subprocess + print("Build", resultPath, "...", file=sys.stderr) + subprocess.run(["cargo", "build", "--release"], cwd=inputPath, check=True) + shutil.copy(os.path.join(inputPath, "target", "release", cargoName), resultPath+".new") + os.replace(resultPath+".new", resultPath) + def makeManpages(): bobRoot = getBobRoot() resultPath = os.path.join(bobRoot, "doc", "_build", "man") diff --git a/pym/bob/scripts.py b/pym/bob/scripts.py index 8274b85a..2cdfca78 100644 --- a/pym/bob/scripts.py +++ b/pym/bob/scripts.py @@ -276,6 +276,13 @@ def cmd(): print("bob -C: unable to change directory:", str(e), file=sys.stderr) return 1 + # Make sure native extension is updated + try: + from .develop.make import makeDirHasher + makeDirHasher() + except ImportError: + pass + if args.command in availableCommands: cmd = availableCommands[args.command][1] if DEBUG['prof']: diff --git a/test/run-tests.sh b/test/run-tests.sh index c7cff46c..25be56ce 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -227,9 +227,10 @@ if [[ -n $RUN_COVERAGE ]] ; then fi fi -# Make sure sandbox helper is updated -$RUN_PYTHON3 -c "from bob.develop.make import makeSandboxHelper +# Make sure sandbox helper and native extensions are updated +$RUN_PYTHON3 -c "from bob.develop.make import makeSandboxHelper, makeDirHasher makeSandboxHelper() +makeDirHasher() " # execute everything if nothing was specified From b8568d60d2c3a4968a18a5e603be0f0a48d5de61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 1 Nov 2026 20:03:15 +0100 Subject: [PATCH 11/12] github: switch to Python 3.14 for Windows and sdist --- .github/workflows/workflow.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 350edadd..d3af9eec 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -72,10 +72,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.13 + - name: Set up Python 3.14 uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" cache: pip - name: Setup Java @@ -119,10 +119,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.13 + - name: Set up Python 3.14 uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" cache: pip - name: Install dependencies From ed50060326c8ec7f020eccc8df947598dfd9f11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 1 Nov 2026 20:17:55 +0100 Subject: [PATCH 12/12] github: build Windows wheel Bob now has an optional directory hasher plugin on Windows too. Provide a wheel so that it's usable directly from PyPi. --- .github/workflows/workflow.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index d3af9eec..18252785 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -97,7 +97,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install PyYAML coverage schema python-magic pyparsing sphinx wheel + pip install PyYAML coverage schema python-magic pyparsing sphinx wheel build - name: Run unit tests run: | @@ -112,6 +112,19 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + - name: Build Python package + run: | + python3 -m build --wheel + if (-not (Test-Path pym\bob\version.py -PathType Leaf)) { + throw "setuptools-scm did not run" + } + + - name: Store the binary wheel + uses: actions/upload-artifact@v4 + with: + name: python-dist-windows + path: dist + build-sdist: runs-on: ubuntu-latest timeout-minutes: 5 @@ -147,8 +160,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 needs: - # Depend on all builds even if they don't provide an artifact (e.g. - # windows). This makes sure we publish only if everything is ok. - build-linux - build-windows - build-sdist