diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 350edadd..18252785 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -72,10 +72,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.13 + - name: Set up Python 3.14 uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" cache: pip - name: Setup Java @@ -97,7 +97,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install PyYAML coverage schema python-magic pyparsing sphinx wheel + pip install PyYAML coverage schema python-magic pyparsing sphinx wheel build - name: Run unit tests run: | @@ -112,6 +112,19 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + - name: Build Python package + run: | + python3 -m build --wheel + if (-not (Test-Path pym\bob\version.py -PathType Leaf)) { + throw "setuptools-scm did not run" + } + + - name: Store the binary wheel + uses: actions/upload-artifact@v4 + with: + name: python-dist-windows + path: dist + build-sdist: runs-on: ubuntu-latest timeout-minutes: 5 @@ -119,10 +132,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.13 + - name: Set up Python 3.14 uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" cache: pip - name: Install dependencies @@ -147,8 +160,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 needs: - # Depend on all builds even if they don't provide an artifact (e.g. - # windows). This makes sure we publish only if everything is ok. - build-linux - build-windows - build-sdist diff --git a/.gitignore b/.gitignore index 0edf74b3..b10020b7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ __pycache__/ /dist/ /htmlcov /pym/bob/version.py +/pym/bob/*.pyd +/pym/bob/*.so +/src/dirhasher/target/ diff --git a/pym/bob/builder.py b/pym/bob/builder.py index c9a255b9..72211a88 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -52,18 +52,19 @@ async def gatherTasks(tasks): await asyncio.wait(tasks) return [ t.result() for t in tasks ] -def hashWorkspace(step): +async def hashWorkspace(step): + loop = asyncio.get_event_loop() + cachePath = os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin") # Cache is also used by share module - return hashDirectory(step.getStoragePath(), - os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin")) + return await loop.run_in_executor(None, hashDirectory, step.getStoragePath(), cachePath) class HashOnce: def __init__(self, step): self.step = step self.result = None - def hashWorkspace(self): + async def hashWorkspace(self): if self.result is None: - self.result = hashWorkspace(self.step) + self.result = await hashWorkspace(self.step) return self.result def invalidate(self): self.result = None @@ -1202,7 +1203,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None): inputChanged = checkoutBuildOnlyStateChanged(checkoutState, oldCheckoutState) rehash = currentResultHash.hashWorkspace - if checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash): + if await checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash): if checkoutBuildOnlyStateCompatible(checkoutState, oldCheckoutState): with stepExec(checkoutStep, "UPDATE", "{} {}".format(prettySrcPath, overridesString)) as a: @@ -1247,7 +1248,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)): checkoutReason = "dependency changed" elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \ - and (oldResultHash != currentResultHash.hashWorkspace()): + and (oldResultHash != await currentResultHash.hashWorkspace()): checkoutReason = "workspace changed" if checkoutReason: @@ -1345,7 +1346,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): # We always have to rehash the directory as the user might have # changed the source code manually. - checkoutHash = currentResultHash.hashWorkspace() + checkoutHash = await currentResultHash.hashWorkspace() # Generate audit trail before setResultHash() to force re-generation if # the audit trail fails. @@ -1413,7 +1414,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId): # We always rehash the directory in development mode as the # user might have compiled the package manually. if not self.__cleanBuild: - BobState().setResultHash(prettyBuildPath, hashWorkspace(buildStep)) + BobState().setResultHash(prettyBuildPath, await hashWorkspace(buildStep)) else: with stepExec(buildStep, "BUILD", prettyBuildPath, (ALWAYS, NORMAL)) as fullAction: # Squash state because running the step will change the @@ -1425,7 +1426,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId): with stepExec(buildStep, "BUILD", prettyBuildPath, INFO) as buildAction: visibleAction = fullAction if fullAction.visible else buildAction await self._runShell(buildStep, "build", visibleAction, created, self.__cleanBuild) - buildHash = hashWorkspace(buildStep) + buildHash = await hashWorkspace(buildStep) await self._generateAudit(buildStep, depth, buildHash, buildBuildId) BobState().setResultHash(prettyBuildPath, buildHash) BobState().setVariantId(prettyBuildPath, buildDigest[0]) @@ -1626,7 +1627,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId): # Verify integrity of downloaded package to protect against # data corruption. - packageHash = hashWorkspace(packageStep) + packageHash = await hashWorkspace(packageStep) if Audit.fromFile(audit).getArtifact().getResultHash() != packageHash: raise BuildError("Corrupt downloaded artifact! Extracted content hash does not match audit trail.") @@ -1686,7 +1687,7 @@ async def _cookPackageStep(self, packageStep, depth, packageBuildId): with stepExec(packageStep, "PACKAGE", prettyPackagePath, INFO) as packageAction: visibleAction = fullAction if fullAction.visible else packageAction await self._runShell(packageStep, "package", visibleAction, True) - packageHash = hashWorkspace(packageStep) + packageHash = await hashWorkspace(packageStep) packageDigest = await self.__getIncrementalVariantId(packageStep) workspaceChanged = True self.__statistic.packagesBuilt += 1 diff --git a/pym/bob/develop/make.py b/pym/bob/develop/make.py index 1ab2818b..946f017d 100644 --- a/pym/bob/develop/make.py +++ b/pym/bob/develop/make.py @@ -6,7 +6,7 @@ import os import sys -__all__ = ('makeSandboxHelper', 'makeManpages') +__all__ = ('makeSandboxHelper', 'makeManpages', 'makeDirHasher') def findYoungestInDir(path, exclude): ret = 0 @@ -53,6 +53,30 @@ def makeSandboxHelper(): return resultPath +def makeDirHasher(): + if sys.platform == "win32": + cargoName = "_dirhasher.dll" + else: + cargoName = "lib_dirhasher.so" + + import sysconfig + bobRoot = getBobRoot() + resultPath = os.path.join(bobRoot, "pym", "bob", + "_dirhasher" + sysconfig.get_config_var("EXT_SUFFIX")) + inputPath = os.path.join(bobRoot, "src", "dirhasher") + resultDate = findYoungest(resultPath) + inputDate = findYoungest(inputPath) + if inputDate > resultDate: + import shutil + if shutil.which("cargo") is None: + return + + import subprocess + print("Build", resultPath, "...", file=sys.stderr) + subprocess.run(["cargo", "build", "--release"], cwd=inputPath, check=True) + shutil.copy(os.path.join(inputPath, "target", "release", cargoName), resultPath+".new") + os.replace(resultPath+".new", resultPath) + def makeManpages(): bobRoot = getBobRoot() resultPath = os.path.join(bobRoot, "doc", "_build", "man") diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index ed3d75db..ec98d641 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -308,14 +308,14 @@ def deserialize(state): def getScmDirectories(self): return { d : (bytes.fromhex(h), p) for (d, (h, p)) in self.__data['scmDirectories'].items() } - def mayUpdate(self, inputChanged, oldHash, rehash): + async def mayUpdate(self, inputChanged, oldHash, rehash): if any((s.isLocal() and not s.isDeterministic()) for s in self.getScmList()): return True if not self.getUpdateScript(): return False if not self.isUpdateDeterministic() or inputChanged: return True - return rehash() != oldHash + return await rehash() != oldHash async def getDigestCoro(self, calculate, hasher=DigestHasher, fingerprint=None, platform=b'', relaxTools=False): diff --git a/pym/bob/scripts.py b/pym/bob/scripts.py index f58cb429..2cdfca78 100644 --- a/pym/bob/scripts.py +++ b/pym/bob/scripts.py @@ -7,7 +7,6 @@ from .errors import BobError from .state import finalize from .tty import colorize, Unbuffered, setColorMode, cleanup -from .utils import asHexStr, hashPath, getPlatformTag, EventLoopWrapper import argparse import logging import sys @@ -277,6 +276,13 @@ def cmd(): print("bob -C: unable to change directory:", str(e), file=sys.stderr) return 1 + # Make sure native extension is updated + try: + from .develop.make import makeDirHasher + makeDirHasher() + except ImportError: + pass + if args.command in availableCommands: cmd = availableCommands[args.command][1] if DEBUG['prof']: diff --git a/pym/bob/utils.py b/pym/bob/utils.py index ba7c913e..d41aa4a5 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -5,7 +5,7 @@ from .errors import BuildError, ParseError from binascii import hexlify -from tempfile import NamedTemporaryFile, TemporaryDirectory +from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile import collections.abc import hashlib import logging @@ -253,9 +253,16 @@ def isWindows(): __isWindows = True else: __isWindows = False + + def getChangeTime(st): + return st.st_ctime_ns else: __isWindows = True + def getChangeTime(st): + # The st_ctime field is the creation time on Windows + return st.st_mtime_ns + def _replacePathWin32(src, dst): # Workaround for spurious PermissionError's on Windows. i = 0 @@ -433,8 +440,12 @@ def __init__(self, cachePath): def open(self): self.__inPos = 0 self.__inPosOld = 0 + self.__inFile = None self.__outFile = None + self.__writeFailed = False self.__current = DirHasher.FileIndex.Stat() + self.__consumed = False + self.__changeTimeLimit = self.__calcChangeTimeLimit() try: if os.path.exists(self.__cachePath): self.__inFile = open(self.__cachePath, "rb") @@ -448,37 +459,81 @@ def open(self): "Wrong signature at '%s': %s", self.__cachePath, sig) self.__inFile.close() self.__inFile = None + self.__current = None self.__mismatch = True else: self.__inFile = None + self.__current = None self.__mismatch = True except OSError as e: - raise BuildError("Error opening hash cache: " + str(e)) - - def close(self): - try: + logging.getLogger(__name__).warning("Cannot open hash cache: %s", str(e)) if self.__inFile: self.__inFile.close() - if self.__outFile: + self.__inFile = None + self.__current = None + self.__mismatch = True + + def close(self): + if self.__inFile: + self.__inFile.close() + if self.__outFile: + try: self.__outFile.close() replacePath(self.__outFile.name, self.__cachePath) - except OSError as e: - raise BuildError("Error closing hash cache: " + str(e)) + except OSError as e: + self.__abortWrite(e) + + def __calcChangeTimeLimit(self): + """Determine the current time of the file system clock. + + File time stamps have a limited granularity. A file that is + modified again in the same clock tick after it was hashed would + keep its stat data. Such "racily clean" entries must not be + trusted. Any file whose change-time is not older than the current file + system time is thus not recorded in the index. Creating a file gets + us the current time in the right granularity, even on network file + systems where the clock of the server is used. + """ + try: + with TemporaryFile(dir=self.__cacheDir or ".") as f: + return getChangeTime(os.fstat(f.fileno())) + except OSError: + return -(2**63) + + def __abortWrite(self, err): + """Give up writing the cache. The old cache is kept untouched.""" + logging.getLogger(__name__).warning("Cannot write hash cache: %s", str(err)) + self.__writeFailed = True + if self.__outFile: + try: + self.__outFile.close() + os.unlink(self.__outFile.name) + except OSError: + pass + self.__outFile = None def __readEntry(self): - if not self.__inFile: return False + """Read the next entry into __current. + + Sets __current to None at the end of the index. In any case + __inPosOld is the end of all previous entries. + """ + self.__inPosOld = self.__inPos raw = self.__inFile.read(DirHasher.FileIndex.CACHE_ENTRY_SIZE) - if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: return False + if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: + self.__current = None + return e = self.__current (e.ctime, e.mtime, e.dev, e.ino, e.mode, e.size, e.digest, nameLen) = struct.unpack(DirHasher.FileIndex.CACHE_ENTRY_FMT, raw) e.name = self.__inFile.read(nameLen) - self.__inPosOld = self.__inPos self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen - return True + self.__consumed = False - def __writeEntry(self, name, st, digest): - if not self.__outFile: + def __startWrite(self): + """Start the new index. All entries before __current are kept.""" + if self.__outFile or self.__writeFailed: return + try: self.__outFile = NamedTemporaryFile(mode="wb", dir=self.__cacheDir, delete=False) if self.__inFile: pos = self.__inFile.tell() @@ -487,16 +542,36 @@ def __writeEntry(self, name, st, digest): self.__inFile.seek(pos) else: self.__outFile.write(DirHasher.FileIndex.SIGNATURE) - self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns, - st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size, - digest, len(name))) - self.__outFile.write(name) + except OSError as e: + self.__abortWrite(e) + + def __writeEntry(self, name, st, digest): + self.__startWrite() + if not self.__outFile: return + # Entries that are of the same age than the current file system + # time must be dropped. Changes in the same "file system tick" + # cannot be distinguished. + if getChangeTime(st) >= self.__changeTimeLimit: return + try: + self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns, + st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size, + digest, len(name))) + self.__outFile.write(name) + except OSError as e: + self.__abortWrite(e) def __match(self, name, st): - while self.__current.name < name: - if not self.__readEntry(): break + while (self.__current is not None) and (self.__consumed or + self.__current.name < name): + if not self.__consumed and not self.__mismatch: + # Stale entry of a removed file. Drop it from the index. + self.__mismatch = True + self.__startWrite() + self.__readEntry() e = self.__current - res = ((e.name == name) and (e.ctime == st.st_ctime_ns) and + if (e is None) or (e.name != name): return False + self.__consumed = True + res = ((e.ctime == st.st_ctime_ns) and (e.mtime == st.st_mtime_ns) and (e.dev == st.st_dev) and (e.ino == maskIno(st.st_ino)) and (e.mode == st.st_mode) and (e.size == st.st_size)) @@ -588,7 +663,7 @@ def __hashDir(self, prefix, path=b''): entries = sorted(entries, key=lambda x: x[1]) dirList = [ - (struct.pack("=L", s.st_mode) + self.__hashEntry(prefix, e, s) + f) + (struct.pack("= 40.0", "setuptools-scm", "Sphinx"] +requires = ["setuptools >= 40.0", "setuptools-scm", "Sphinx", "setuptools-rust"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "pym/bob/version.py" local_scheme = "no-local-version" + +[[tool.setuptools-rust.ext-modules]] +# See https://setuptools-rust.readthedocs.io/en/latest/reference.html +path = "src/dirhasher/Cargo.toml" +target = "bob._dirhasher" +optional = true diff --git a/src/dirhasher/Cargo.lock b/src/dirhasher/Cargo.lock new file mode 100644 index 00000000..43fd63d3 --- /dev/null +++ b/src/dirhasher/Cargo.lock @@ -0,0 +1,322 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bob-dirhasher" +version = "0.1.0" +dependencies = [ + "pyo3", + "sha1", + "tempfile", + "threadpool", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/src/dirhasher/Cargo.toml b/src/dirhasher/Cargo.toml new file mode 100644 index 00000000..19ec8b43 --- /dev/null +++ b/src/dirhasher/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bob-dirhasher" +version = "0.1.0" +edition = "2024" + +[dependencies] +pyo3 = "0.29.3" +sha1 = "0.11.0" +tempfile = "3" +threadpool = "1.8.1" + +[profile.release-lto] +inherits = "release" +lto = true + +[lib] +# See https://github.com/PyO3/pyo3 for details +name = "_dirhasher" # private module to be nested into Python package +crate-type = ["cdylib"] +path = "src/lib.rs" diff --git a/src/dirhasher/src/hasher.rs b/src/dirhasher/src/hasher.rs new file mode 100644 index 00000000..c091f778 --- /dev/null +++ b/src/dirhasher/src/hasher.rs @@ -0,0 +1,882 @@ +use sha1::{Digest, Sha1}; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File, FileType, Metadata}; +use std::io::{Error, ErrorKind, Read, Write}; +use std::path::{MAIN_SEPARATOR_STR, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; +use tempfile::NamedTempFile; +use threadpool; + +#[cfg(unix)] +mod unix { + pub mod helpers; + pub use std::os::unix::fs::MetadataExt; +} + +#[cfg(unix)] +use unix::*; + +#[cfg(windows)] +mod windows { + pub mod helpers; + pub use std::os::windows::fs::MetadataExt; +} + +#[cfg(windows)] +use windows::*; + +use helpers::*; // import the helpers from the unix/windows submodule + +pub type Sha1Sum = [u8; 20]; + +pub enum HashSum { + Empty, + Sha1(Sha1Sum), + RDev([u8; 8]), +} + +type Inode = usize; + +/// A warning in the style of the Python logging module: a format string with `%s` placeholders and +/// the matching arguments. +pub struct Warning { + pub msg: &'static str, + pub args: Vec, +} + +impl Warning { + /// Warning about a failed I/O operation on `path`. + fn io(msg: &'static str, path: &Path, err: &Error) -> Self { + Warning { + msg, + args: vec![format!("{}: {}", path.display(), err)], + } + } + + fn stat(path: &Path, err: &Error) -> Self { + Warning { + msg: "Cannot stat '%s': %s", + args: vec![path.display().to_string(), err.to_string()], + } + } +} + +/// Signals that the hashing was aborted by the host. +pub struct Aborted; + +/// Interface to the caller of the hasher. All methods are invoked on the calling thread. +pub trait Host { + /// Report a warning. An error aborts the hashing. + fn warn(&mut self, warning: Warning) -> Result<(), Aborted>; + + /// Invoked roughly every `POLL_INTERVAL` while hashing. An error aborts the hashing. + fn poll(&mut self) -> Result<(), Aborted>; +} + +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Messages from the worker threads to the calling thread. +enum Msg { + ReadDir(ReadDirResult), + Hashed(Inode, Sha1Sum), + Warning(Warning), +} + +/// Compact subset of `Metadata` with just the fields that are needed. +struct Stat { + file_type: FileType, + mode: u32, + size: u64, + ctime: i64, + mtime: i64, + dev: u64, + ino: u64, + rdev: u64, +} + +#[cfg(windows)] +fn win_time_to_unix_ns(t: u64) -> i64 { + // epoch offset - number of 100 ns ticks between 1601 and 1970 + ((t as i64) - 116444736000000000i64) * 100 +} + +#[cfg(windows)] +/// Convert file metadata info Python compatible "st_mode". +/// +/// Mimics _Py_attribute_data_to_stat() and win32_xstat_impl() from the CPython implementation. +fn win_mode_from_metadata(md: &Metadata, file_name: &OsStr) -> u32 { + let mut mode: u32 = if md.is_dir() { + 0o040000 // S_IFDIR + } else if md.is_symlink() { + 0o120000 // S_IFLNK + } else { + 0o100000 // S_IFREG + }; + + // FILE_ATTRIBUTE_DIRECTORY - this is set for symlinks on directories too + if (md.file_attributes() & 0x10) != 0 { + mode |= 0o111; + } + + // FILE_ATTRIBUTE_READONLY + if (md.file_attributes() & 0x1) != 0 { + mode |= 0o444; + } else { + mode |= 0o666; + } + + // Python makes certain files "executable". + if !md.is_dir() { + if let Some((_, ext)) = file_name.to_string_lossy().rsplit_once(".") { + let ext = ext.to_lowercase(); + if ext == ".exe" || ext == ".bat" || ext == ".cmd" || ext == ".com" { + mode |= 0o111; + } + } + } + + mode +} + +impl Stat { + #[cfg(unix)] + fn get_change_time(&self) -> i64 { + self.ctime + } + + #[cfg(windows)] + fn get_change_time(&self) -> i64 { + self.mtime + } +} + +impl Stat { + #[cfg(unix)] + fn from(md: &Metadata, _: &OsStr) -> Self { + Stat { + file_type: md.file_type(), + mode: md.mode(), + size: md.size(), + ctime: md.ctime() * 1_000_000_000 + md.ctime_nsec(), + mtime: md.mtime() * 1_000_000_000 + md.mtime_nsec(), + dev: md.dev(), + ino: md.ino(), + rdev: md.rdev(), + } + } + + #[cfg(windows)] + fn from(md: &Metadata, path: &OsStr) -> Self { + Stat { + file_type: md.file_type(), + mode: win_mode_from_metadata(md, path), + size: md.file_size(), + ctime: win_time_to_unix_ns(md.creation_time()), + mtime: win_time_to_unix_ns(md.last_write_time()), + // Fortunately the following information is also 0 in the Python implementation. We're + // using os.DirEntry.stat() and this does not populate these fields either. + dev: 0, // md.volume_serial_number() is unfortunately not a stable API + ino: 0, // md.file_index() neither + rdev: 0, // unused on Windows + } + } +} + +struct DirEntry { + name: OsString, + ino: Inode, +} + +type DirEntries = Vec; + +struct VNode { + stat: Stat, + children: DirEntries, + hash: Option, +} + +impl VNode { + fn new(stat: Stat) -> Self { + VNode { + stat, + children: Vec::new(), + hash: None, + } + } +} + +/// Determine the current time of the file system clock. +/// +/// File time stamps have a limited granularity. A file that is modified again in the same clock +/// tick after it was hashed would keep its stat data. Such "racily clean" entries must not be +/// trusted. Any file whose ctime is not older than the current file system time is thus not +/// recorded in the index. The ctime is used because it cannot be set by the user. Creating a file +/// gets us the current time in the right granularity, even on network file systems where the clock +/// of the server is used. +fn calc_ctime_limit(index_path: &Path) -> i64 { + // Use the directory of the index file. A bare file name has an empty parent, which means the + // current directory. If there is no parent at all, use the global temporary directory as + // fallback. + let dir = match index_path.parent() { + Some(p) if p.is_empty() => PathBuf::from("."), + Some(p) => p.to_path_buf(), + None => std::env::temp_dir(), + }; + let Ok(tmp_file) = tempfile::tempfile_in(dir) else { + return i64::MIN; + }; + + let Ok(metadata) = tmp_file.metadata() else { + return i64::MIN; + }; + + Stat::from(&metadata, index_path.as_os_str()).get_change_time() +} + +/// Hash a file in a worker thread. Stops early if the hashing was `cancelled`. +fn hash_file(path: &Path, cancelled: &AtomicBool, tx: &mpsc::Sender) -> Sha1Sum { + let mut hasher = Sha1::new(); + + match File::open(path) { + Ok(mut file) => { + let mut buffer = [0u8; 128 * 1024]; + while !cancelled.load(Ordering::Relaxed) { + match file.read(&mut buffer) { + Ok(0) => break, + Ok(len) => hasher.update(&buffer[..len]), + Err(err) => { + if err.kind() != ErrorKind::Interrupted { + let warning = Warning::io("Cannot hash file: %s", path, &err); + let _ = tx.send(Msg::Warning(warning)); + break; + } + } + } + } + } + Err(err) => { + let _ = tx.send(Msg::Warning(Warning::io( + "Cannot hash file: %s", + path, + &err, + ))); + } + } + + return hasher.finalize().into(); +} + +fn hash_link(path: &Path, host: &mut dyn Host) -> Result { + let mut hasher = Sha1::new(); + match path.read_link() { + Ok(destination) => hasher.update(os_string_to_bytes(&destination.into_os_string())), + Err(err) => host.warn(Warning::io("Cannot hash link: %s", path, &err))?, + } + Ok(hasher.finalize().into()) +} + +trait DirHasherIndex { + fn check(&mut self, path: &Path, stat: &Stat) -> Option; + fn update(&mut self, path: &Path, stat: &Stat, hash: &Sha1Sum); + /// Write back the updated index. + fn finish(&self) -> Result<(), Warning>; +} + +struct NullIndex {} + +impl DirHasherIndex for NullIndex { + fn check(&mut self, _path: &Path, _stat: &Stat) -> Option { + None + } + fn update(&mut self, _path: &Path, _stat: &Stat, _hash: &Sha1Sum) {} + fn finish(&self) -> Result<(), Warning> { + Ok(()) + } +} + +struct FileIndexEntry<'a> { + data: &'a [u8], +} + +const INDEX_SIGNATURE: &[u8; 4] = b"BOB2"; +const INDEX_ENTRY_SIZE: usize = 66; +const INDEX_OFF_CTIME: usize = 0; +const INDEX_OFF_MTIME: usize = 8; +const INDEX_OFF_DEV: usize = 16; +const INDEX_OFF_INO: usize = 24; +const INDEX_OFF_MODE: usize = 32; +const INDEX_OFF_SIZE: usize = 36; +const INDEX_OFF_DIGEST: usize = 44; +const INDEX_OFF_NAMELEN: usize = 64; + +impl FileIndexEntry<'_> { + fn new(data: &[u8]) -> Option> { + if data.len() < INDEX_ENTRY_SIZE { + return None; + } + let name_len_bytes: [u8; 2] = data[INDEX_OFF_NAMELEN..][0..2].try_into().unwrap(); + let name_len: usize = u16::from_ne_bytes(name_len_bytes).into(); + let len = INDEX_ENTRY_SIZE + name_len; + if data.len() < len { + return None; + } + Some(FileIndexEntry { data: &data[..len] }) + } + + fn get_name(&self) -> &[u8] { + &self.data[INDEX_ENTRY_SIZE..] + } + + fn len(&self) -> usize { + self.data.len() + } + + fn cmp_stat(&self, stat: &Stat) -> bool { + let ctime = i64::from_ne_bytes(self.data[INDEX_OFF_CTIME..][0..8].try_into().unwrap()); + let mtime = i64::from_ne_bytes(self.data[INDEX_OFF_MTIME..][0..8].try_into().unwrap()); + let dev = u64::from_ne_bytes(self.data[INDEX_OFF_DEV..][0..8].try_into().unwrap()); + let ino = u64::from_ne_bytes(self.data[INDEX_OFF_INO..][0..8].try_into().unwrap()); + let mode = u32::from_ne_bytes(self.data[INDEX_OFF_MODE..][0..4].try_into().unwrap()); + let size = u64::from_ne_bytes(self.data[INDEX_OFF_SIZE..][0..8].try_into().unwrap()); + + return stat.ctime == ctime + && stat.mtime == mtime + && stat.dev == dev + && stat.ino == ino + && stat.mode == mode + && stat.size == size; + } + + fn hash_sum(&self) -> Sha1Sum { + self.data[INDEX_OFF_DIGEST..][0..20].try_into().unwrap() + } +} + +struct FileIndex { + index_path: PathBuf, + ctime_limit: i64, + old_index: Vec, + new_index: Vec, + old_pos: usize, + mismatch: bool, +} + +impl FileIndex { + fn new(index_path: &Path) -> (FileIndex, Option) { + let mut warning = None; + let index = match fs::read(index_path) { + Ok(data) => data, + Err(err) => { + if err.kind() != ErrorKind::NotFound { + warning = Some(Warning::io("Cannot open hash cache: %s", index_path, &err)); + }; + Vec::new() + } + }; + + let mismatch = index.len() < INDEX_SIGNATURE.len() || index[0..4] != *INDEX_SIGNATURE; + let mut new_index: Vec = Vec::with_capacity(index.len()); + new_index.extend_from_slice(INDEX_SIGNATURE); + + let index = FileIndex { + index_path: index_path.to_path_buf(), + ctime_limit: calc_ctime_limit(index_path), + old_index: index, + new_index: new_index, + old_pos: if mismatch { 0 } else { 4 }, + mismatch: mismatch, + }; + (index, warning) + } + + fn write_back(&self) -> std::io::Result<()> { + if !self.mismatch { + return Ok(()); + } + let mut file = NamedTempFile::new_in( + self.index_path + .parent() + .ok_or(Error::new(ErrorKind::NotADirectory, "No parent directory!"))?, + )?; + let _ = file.write_all(&self.new_index)?; + file.persist(&self.index_path)?; + Ok(()) + } + + fn search_entry(&mut self, name: &OsStr) -> Option> { + let name = os_str_to_bytes(name); + loop { + let entry = FileIndexEntry::new(&self.old_index[self.old_pos..])?; + let entry_name = entry.get_name(); + if entry_name == name { + self.old_pos += entry.len(); + return Some(entry); + } else if entry_name < name { + self.old_pos += entry.len(); + self.mismatch = true; + } else { + self.mismatch = true; + return None; + } + } + } + + fn add_entry(&mut self, name: &OsStr, stat: &Stat, hash_sum: &Sha1Sum) { + // Entries that are of the same age than the current file system time must be dropped. + // Changes in the same "file system tick" cannot be distinguished. Entries that are newer + // have been recently modified. Their checksum is most probably wrong. + if stat.ctime >= self.ctime_limit { + return; + } + + self.new_index.extend_from_slice(&stat.ctime.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.mtime.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.dev.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.ino.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.mode.to_ne_bytes()); + self.new_index.extend_from_slice(&stat.size.to_ne_bytes()); + self.new_index.extend_from_slice(hash_sum); + + let name_bytes = os_str_to_bytes(name); + let name_len: u16 = name_bytes.len().try_into().unwrap(); + self.new_index.extend_from_slice(&name_len.to_ne_bytes()); + self.new_index.extend_from_slice(&name_bytes); + } +} + +impl DirHasherIndex for FileIndex { + fn check(&mut self, path: &Path, stat: &Stat) -> Option { + let entry_path = path.as_os_str(); + if let Some(cached_entry) = self.search_entry(entry_path) { + if cached_entry.cmp_stat(stat) { + return Some(cached_entry.hash_sum()); + } + } + + // Either there was no cache entry or it did not match. + self.mismatch = true; + None + } + + fn update(&mut self, path: &Path, stat: &Stat, hash: &Sha1Sum) { + if self.mismatch { + self.add_entry(path.as_os_str(), stat, hash); + } + } + + fn finish(&self) -> Result<(), Warning> { + self.write_back() + .map_err(|err| Warning::io("Cannot write hash cache: %s", &self.index_path, &err)) + } +} + +/// Reusable path buffer for the tree traversal. Holds the prefix, a separator +/// and the relative path. Children are appended and truncated again. +struct PathBuilder { + buf: Vec, + stack: Vec, + prefix_len: usize, + base: usize, +} + +impl PathBuilder { + fn new(prefix: &Path) -> Self { + let mut buf = prefix.as_os_str().as_encoded_bytes().to_vec(); + let prefix_len = buf.len(); + if !prefix.is_empty() && !prefix.ends_with(MAIN_SEPARATOR_STR) { + buf.extend_from_slice(OsStr::new(MAIN_SEPARATOR_STR).as_encoded_bytes()); + } + let base = buf.len(); + PathBuilder { + buf, + stack: Vec::new(), + prefix_len, + base, + } + } + + /// Path relative to the prefix. + fn rel(&self) -> &Path { + // SAFETY: + // - all bytes are from OsStr::as_encoded_bytes() + // - no byte sequences is split + Path::new(unsafe { OsStr::from_encoded_bytes_unchecked(&self.buf[self.base..]) }) + } + + /// Full path including the prefix. + fn full(&self) -> &Path { + let end = if self.buf.len() == self.base { + self.prefix_len + } else { + self.buf.len() + }; + // SAFETY: + // - all bytes are from OsStr::as_encoded_bytes() + // - no byte sequences is split + Path::new(unsafe { OsStr::from_encoded_bytes_unchecked(&self.buf[..end]) }) + } + + /// Add new path segment. Assumes trailing separator for directories. + fn push(&mut self, name: &OsStr) { + self.stack.push(self.buf.len()); + self.buf.extend_from_slice(name.as_encoded_bytes()); + } + + /// Pop last path segment + fn pop(&mut self) { + let len = self.stack.pop().expect("path stack must not be empty"); + self.buf.truncate(len); + } +} + +struct ReadDirEntry { + name: OsString, + stat: Stat, +} + +struct ReadDirResult { + path: PathBuf, + ino: Inode, + entries: Vec, +} + +struct DirHasher<'a> { + host: &'a mut dyn Host, + index: Box, + size: u64, + ignore_dirs: Vec, + ignore_files: Vec, + vnodes: Vec, + pool: threadpool::ThreadPool, + /// Number of worker jobs whose result is still outstanding. + pending: u32, + tx: mpsc::Sender, + rx: mpsc::Receiver, + cancelled: Arc, + next_poll: Instant, +} + +impl<'a> DirHasher<'a> { + fn new( + host: &'a mut dyn Host, + index_path: Option<&Path>, + add_ignore_dirs: Option>, + ) -> Result { + let mut ignore_dirs: Vec = + vec![".git".into(), ".portage-cache".into(), ".svn".into()]; + if let Some(add_ignore_dirs) = add_ignore_dirs { + ignore_dirs.extend(add_ignore_dirs); + } + let index: Box = match index_path { + Some(index_path) => { + let (index, warning) = FileIndex::new(index_path); + if let Some(warning) = warning { + host.warn(warning)?; + } + Box::new(index) + } + None => Box::new(NullIndex {}), + }; + let (tx, rx) = mpsc::channel::(); + Ok(DirHasher { + host, + index, + size: 0, + ignore_dirs, + ignore_files: vec!["BaseDirList.txt".into()], + vnodes: Vec::new(), + pool: threadpool::Builder::new().build(), + pending: 0, + tx, + rx, + cancelled: Arc::new(AtomicBool::new(false)), + next_poll: Instant::now() + POLL_INTERVAL, + }) + } + + /// Poll the host if `POLL_INTERVAL` has elapsed since the last time. + fn maybe_poll(&mut self) -> Result<(), Aborted> { + let now = Instant::now(); + if now >= self.next_poll { + self.host.poll()?; + self.next_poll = now + POLL_INTERVAL; + } + Ok(()) + } + + fn handle_msg(&mut self, msg: Msg) -> Result<(), Aborted> { + match msg { + Msg::ReadDir(res) => { + self.pending -= 1; + self.process_readdir(res); + } + Msg::Hashed(ino, hash) => { + self.pending -= 1; + self.vnodes[ino].hash = Some(HashSum::Sha1(hash)); + } + Msg::Warning(warning) => self.host.warn(warning)?, + } + Ok(()) + } + + /// Handle the messages of the workers until all jobs are finished. + fn wait_pending(&mut self) -> Result<(), Aborted> { + while self.pending > 0 { + let timeout = self.next_poll.saturating_duration_since(Instant::now()); + match self.rx.recv_timeout(timeout) { + Ok(msg) => self.handle_msg(msg)?, + Err(mpsc::RecvTimeoutError::Timeout) => (), + Err(mpsc::RecvTimeoutError::Disconnected) => { + unreachable!("sender is owned by self") + } + } + self.maybe_poll()?; + } + Ok(()) + } + + fn schedule_readdir(&mut self, path: PathBuf, ino: Inode) { + let tx = self.tx.clone(); + let cancelled = self.cancelled.clone(); + self.pending += 1; + self.pool.execute(move || { + if cancelled.load(Ordering::Relaxed) { + return; + } + let mut entries = Vec::new(); + match fs::read_dir(&path) { + Ok(iter) => { + for entry in iter { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + let warning = Warning::io("Cannot list directory: %s", &path, &err); + let _ = tx.send(Msg::Warning(warning)); + continue; + } + }; + let metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(err) => { + let _ = tx.send(Msg::Warning(Warning::stat(&entry.path(), &err))); + continue; + } + }; + let name = entry.file_name(); + let stat = Stat::from(&metadata, name.as_os_str()); + entries.push(ReadDirEntry { name, stat }); + } + } + Err(err) => { + let warning = Warning::io("Cannot list directory: %s", &path, &err); + let _ = tx.send(Msg::Warning(warning)); + } + }; + // The receiver is gone if the hashing was aborted. + let _ = tx.send(Msg::ReadDir(ReadDirResult { path, ino, entries })); + }); + } + + fn process_readdir(&mut self, res: ReadDirResult) { + let mut dir: DirEntries = Vec::with_capacity(res.entries.len()); + + for entry in res.entries { + let mut name = entry.name; + if entry.stat.file_type.is_dir() { + if self.ignore_dirs.contains(&name) { + continue; + } + name.push(MAIN_SEPARATOR_STR); + } else if self.ignore_files.contains(&name) { + continue; + } + + let ino: Inode = self.vnodes.len(); + if entry.stat.file_type.is_dir() { + self.schedule_readdir(res.path.join(&name), ino); + }; + self.vnodes.push(VNode::new(entry.stat)); + dir.push(DirEntry { name, ino }); + } + + dir.sort_unstable_by(|e1, e2| e1.name.cmp(&e2.name)); + self.vnodes[res.ino].children = dir; + } + + /// Discover the file system tree at `path`. The caller decides whether a symlink at `path` + /// itself is followed by passing the respective `metadata`. Symlinks below `path` are never + /// followed. + fn discover(&mut self, path: &Path, metadata: &Metadata) -> Result<(), Aborted> { + let stat = Stat::from(metadata, path.as_os_str()); + if stat.file_type.is_dir() { + let ino: Inode = self.vnodes.len(); + self.vnodes.push(VNode::new(stat)); + self.schedule_readdir(PathBuf::from(path), ino); + self.wait_pending()?; + } else { + self.vnodes.push(VNode::new(stat)); + }; + + Ok(()) + } + + /// Start collecting hash sums on the discovered file system tree. If unkown files are + /// encountered, schedule their hashing in the thread pool. + fn collect_node(&mut self, path: &mut PathBuilder, ino: Inode) -> Result<(), Aborted> { + self.size += self.vnodes[ino].stat.size; + + let file_type = self.vnodes[ino].stat.file_type; + if file_type.is_file() { + match self.index.check(path.rel(), &self.vnodes[ino].stat) { + Some(hash) => self.vnodes[ino].hash = Some(HashSum::Sha1(hash)), + None => { + let tx = self.tx.clone(); + let cancelled = self.cancelled.clone(); + let path = path.full().to_path_buf(); + self.pending += 1; + self.pool.execute(move || { + if cancelled.load(Ordering::Relaxed) { + return; + } + let hash = hash_file(&path, &cancelled, &tx); + let _ = tx.send(Msg::Hashed(ino, hash)); + }); + } + } + } else if file_type.is_dir() { + for i in 0..self.vnodes[ino].children.len() { + let child_ino = self.vnodes[ino].children[i].ino; + path.push(&self.vnodes[ino].children[i].name); + self.collect_node(path, child_ino)?; + path.pop(); + } + } else if file_type.is_symlink() { + let hash = match self.index.check(path.rel(), &self.vnodes[ino].stat) { + Some(hash) => hash, + None => hash_link(path.full(), &mut *self.host)?, + }; + self.vnodes[ino].hash = Some(HashSum::Sha1(hash)); + } else if is_block_device(&file_type) || is_char_device(&file_type) { + let hash = self.vnodes[ino].stat.rdev.to_le_bytes(); + self.vnodes[ino].hash = Some(HashSum::RDev(hash)); + } else { + self.vnodes[ino].hash = Some(HashSum::Empty); + if !is_fifo(&file_type) { + self.host.warn(Warning { + msg: "Unknown file: %s", + args: vec![path.full().display().to_string()], + })?; + } + } + + Ok(()) + } + + /// Calculate the final hash sum. All VNodes except directories have a hash sum already. + fn finalize_node(&mut self, path: &mut PathBuilder, ino: Inode) { + let file_type = self.vnodes[ino].stat.file_type; + if file_type.is_dir() { + let mut hasher = Sha1::new(); + for i in 0..self.vnodes[ino].children.len() { + let child_ino = self.vnodes[ino].children[i].ino; + path.push(&self.vnodes[ino].children[i].name); + self.finalize_node(path, child_ino); + path.pop(); + hasher.update(&self.vnodes[child_ino].stat.mode.to_le_bytes()); + match self.vnodes[child_ino].hash.as_ref().unwrap() { + HashSum::Empty => (), + HashSum::Sha1(hash) => hasher.update(hash), + HashSum::RDev(hash) => hasher.update(hash), + }; + hasher.update(os_string_to_bytes(&self.vnodes[ino].children[i].name)); + } + self.vnodes[ino].hash = Some(HashSum::Sha1(hasher.finalize().into())); + } + + if file_type.is_file() || file_type.is_symlink() { + if let Some(HashSum::Sha1(hash)) = self.vnodes[ino].hash { + self.index.update(path.rel(), &self.vnodes[ino].stat, &hash); + } + } + } + + /// Hash the tree at `path` and write back the updated index. + fn hash_tree(&mut self, path: &Path, metadata: &Metadata) -> Result { + self.discover(path, metadata)?; + self.collect_node(&mut PathBuilder::new(path), 0)?; + self.wait_pending()?; + self.finalize_node(&mut PathBuilder::new(Path::new("")), 0); + if let Err(warning) = self.index.finish() { + self.host.warn(warning)?; + } + Ok(self.vnodes[0].hash.take().unwrap()) + } + + fn hash_dir(mut self, path: &Path) -> Result<(Vec, u64), Aborted> { + // A symlink to a directory is followed, like os.scandir() does in Python. + let metadata = path.metadata().and_then(|md| { + if md.is_dir() { + Ok(md) + } else { + Err(Error::from(ErrorKind::NotADirectory)) + } + }); + let metadata = match metadata { + Ok(metadata) => metadata, + Err(err) => { + self.host + .warn(Warning::io("Cannot list directory: %s", path, &err))?; + return Ok((Sha1::digest(b"").to_vec(), 0)); + } + }; + + let HashSum::Sha1(hash) = self.hash_tree(path, &metadata)? else { + unreachable!("directories have a SHA1 hash"); + }; + Ok((hash.to_vec(), self.size)) + } + + fn hash_path(mut self, path: &Path) -> Result, Aborted> { + let metadata = match path.symlink_metadata() { + Ok(metadata) => metadata, + Err(err) => { + self.host.warn(Warning::stat(path, &err))?; + return Ok(Vec::new()); + } + }; + + let hash = match self.hash_tree(path, &metadata)? { + HashSum::Empty => Vec::new(), + HashSum::Sha1(hash) => hash.to_vec(), + HashSum::RDev(hash) => hash.to_vec(), + }; + Ok(hash) + } +} + +impl Drop for DirHasher<'_> { + fn drop(&mut self) { + // Let the outstanding worker jobs finish quickly if the hashing was aborted. + self.cancelled.store(true, Ordering::Relaxed); + } +} + +pub fn hash_dir( + path: &Path, + index: Option<&Path>, + ignore_dirs: Option>, + host: &mut dyn Host, +) -> Result<(Vec, u64), Aborted> { + DirHasher::new(host, index, ignore_dirs)?.hash_dir(path) +} + +pub fn hash_path( + path: &Path, + index: Option<&Path>, + ignore_dirs: Option>, + host: &mut dyn Host, +) -> Result, Aborted> { + DirHasher::new(host, index, ignore_dirs)?.hash_path(path) +} diff --git a/src/dirhasher/src/hasher/unix/helpers.rs b/src/dirhasher/src/hasher/unix/helpers.rs new file mode 100644 index 00000000..ad110f18 --- /dev/null +++ b/src/dirhasher/src/hasher/unix/helpers.rs @@ -0,0 +1,24 @@ +use std::ffi::{OsStr, OsString}; +use std::fs::FileType; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::FileTypeExt; + +pub fn is_block_device(ft: &FileType) -> bool { + ft.is_block_device() +} + +pub fn is_char_device(ft: &FileType) -> bool { + ft.is_char_device() +} + +pub fn is_fifo(ft: &FileType) -> bool { + ft.is_fifo() +} + +pub fn os_str_to_bytes(s: &OsStr) -> &[u8] { + s.as_bytes() +} + +pub fn os_string_to_bytes(s: &OsString) -> &[u8] { + os_str_to_bytes(s.as_os_str()) +} diff --git a/src/dirhasher/src/hasher/windows/helpers.rs b/src/dirhasher/src/hasher/windows/helpers.rs new file mode 100644 index 00000000..7f41a93c --- /dev/null +++ b/src/dirhasher/src/hasher/windows/helpers.rs @@ -0,0 +1,22 @@ +use std::ffi::{OsStr, OsString}; +use std::fs::FileType; + +pub fn is_block_device(_: &FileType) -> bool { + false +} + +pub fn is_char_device(_: &FileType) -> bool { + false +} + +pub fn is_fifo(_: &FileType) -> bool { + false +} + +pub fn os_str_to_bytes(s: &OsStr) -> &[u8] { + s.to_str().expect("must not fail").as_bytes() +} + +pub fn os_string_to_bytes(s: &OsString) -> &[u8] { + os_str_to_bytes(s.as_os_str()) +} diff --git a/src/dirhasher/src/lib.rs b/src/dirhasher/src/lib.rs new file mode 100644 index 00000000..4b7d4fda --- /dev/null +++ b/src/dirhasher/src/lib.rs @@ -0,0 +1,85 @@ +use pyo3::prelude::*; + +mod hasher; + +#[pymodule] +mod _dirhasher { + use crate::hasher::{Aborted, Host, Warning}; + use pyo3::prelude::*; + use pyo3::types::{PyBytes, PyTuple}; + use std::ffi::OsString; + use std::path::PathBuf; + + /// Connects the hasher with Python while the GIL is released. The GIL is re-acquired briefly to + /// log warnings and to handle signals. The first Python exception aborts the hashing. + #[derive(Default)] + struct PyHost { + err: Option, + } + + impl PyHost { + fn catch(&mut self, res: PyResult<()>) -> Result<(), Aborted> { + res.map_err(|err| { + self.err = Some(err); + Aborted + }) + } + + fn into_result(self, res: Result) -> PyResult { + res.map_err(|Aborted| self.err.expect("exception recorded on abort")) + } + } + + impl Host for PyHost { + /// Forward the warning to the logger of the Python implementation. + fn warn(&mut self, mut warning: Warning) -> Result<(), Aborted> { + let res = Python::attach(|py| { + let logger = py + .import("logging")? + .call_method1("getLogger", ("bob.utils",))?; + warning.args.insert(0, warning.msg.to_string()); + logger.call_method1("warning", PyTuple::new(py, warning.args)?)?; + Ok(()) + }); + self.catch(res) + } + + /// Run the Python signal handlers, e.g. to raise KeyboardInterrupt on Ctrl-C. + fn poll(&mut self) -> Result<(), Aborted> { + let res = Python::attach(|py| py.check_signals()); + self.catch(res) + } + } + + #[pyfunction] + fn hash_directory_with_size<'py>( + py: Python<'py>, + path: PathBuf, + index: Option, + ignore_dirs: Option>, + ) -> PyResult<(Bound<'py, PyBytes>, u64)> { + let ignore_dirs: Option> = + ignore_dirs.map(|v| v.into_iter().map(PathBuf::into_os_string).collect()); + let mut host = PyHost::default(); + let res = + py.detach(|| crate::hasher::hash_dir(&path, index.as_deref(), ignore_dirs, &mut host)); + let (hash, size) = host.into_result(res)?; + Ok((PyBytes::new(py, &hash), size)) + } + + #[pyfunction] + fn hash_path<'py>( + py: Python<'py>, + path: PathBuf, + index: Option, + ignore_dirs: Option>, + ) -> PyResult> { + let ignore_dirs: Option> = + ignore_dirs.map(|v| v.into_iter().map(PathBuf::into_os_string).collect()); + let mut host = PyHost::default(); + let res = + py.detach(|| crate::hasher::hash_path(&path, index.as_deref(), ignore_dirs, &mut host)); + let hash = host.into_result(res)?; + Ok(PyBytes::new(py, &hash)) + } +} diff --git a/test/run-tests.sh b/test/run-tests.sh index c7cff46c..25be56ce 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -227,9 +227,10 @@ if [[ -n $RUN_COVERAGE ]] ; then fi fi -# Make sure sandbox helper is updated -$RUN_PYTHON3 -c "from bob.develop.make import makeSandboxHelper +# Make sure sandbox helper and native extensions are updated +$RUN_PYTHON3 -c "from bob.develop.make import makeSandboxHelper, makeDirHasher makeSandboxHelper() +makeDirHasher() " # execute everything if nothing was specified diff --git a/test/unit/test_utils_dirhasher.py b/test/unit/test_utils_dirhasher.py index ab2784b2..70e34fb0 100644 --- a/test/unit/test_utils_dirhasher.py +++ b/test/unit/test_utils_dirhasher.py @@ -4,13 +4,32 @@ # SPDX-License-Identifier: GPL-3.0-or-later from tempfile import NamedTemporaryFile, TemporaryDirectory -from unittest import TestCase +from unittest import TestCase, skipIf from unittest.mock import MagicMock, mock_open, patch import binascii +import hashlib import os import sys -from bob.utils import hashFile, hashDirectory, binStat +import time +from bob.utils import hashFile, binStat, DirHasher, canSymlink + +try: + from bob._dirhasher import hash_directory_with_size, hash_path + def rustHashDirectory(path, index=None, ignoreDirs=None): + return hash_directory_with_size(path, index, ignoreDirs)[0] + def rustHashPath(path, index=None, ignoreDirs=None): + return hash_path(path, index, ignoreDirs) +except ImportError: + rustHashDirectory = None + rustHashPath = None + +def pyHashDirectory(path, index=None, ignoreDirs=None): + return DirHasher(index, ignoreDirs).hashDirectory(path)[0] + +def pyHashPath(path, index=None, ignoreDirs=None): + return DirHasher(index, ignoreDirs).hashPath(path) + class TestHashFile(TestCase): def testBigFile(self): @@ -41,7 +60,7 @@ def makeOsScandir(entries): m.return_value = OsScandirList(entries) return m -class TestHashDir(TestCase): +class CommonHashDirTests: def setUp(self): self.umask = os.umask(0o022) @@ -60,7 +79,7 @@ def testDirAndFile(self): with open(os.path.join(tmp, "dir", "file"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) self.assertEqual(len(sum1), 20) # Result depends on path separator character ('/' vs. '\') if sys.platform == "win32": @@ -69,7 +88,7 @@ def testDirAndFile(self): else: self.assertEqual(sum1, binascii.unhexlify( "640f516de78fba0b6d2ddde4451000f142d06b0d")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertEqual(sum1, sum2) def testRenameDirectory(self): @@ -80,9 +99,9 @@ def testRenameDirectory(self): with open(os.path.join(tmp, "dir", "file"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) os.rename(os.path.join(tmp, "dir"), os.path.join(tmp, "foo")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertNotEqual(sum1, sum2) def testRenameFile(self): @@ -92,9 +111,9 @@ def testRenameFile(self): with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp) + sum1 = self.hashDirectory(tmp) os.rename(os.path.join(tmp, "foo"), os.path.join(tmp, "bar")) - sum2 = hashDirectory(tmp) + sum2 = self.hashDirectory(tmp) self.assertNotEqual(sum1, sum2) def testRewriteFile(self): @@ -105,20 +124,77 @@ def testRewriteFile(self): with TemporaryDirectory() as tmp: with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'abc') - sum1 = hashDirectory(tmp, index) + sum1 = self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') with open(os.path.join(tmp, "foo"), 'wb') as f: f.write(b'qwer') - sum2 = hashDirectory(tmp, index) + sum2 = self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') self.assertNotEqual(sum1, sum2) + def testMissingIndexDir(self): + """Index in non-existing directory just issues a warning""" + with TemporaryDirectory() as tmp: + index = os.path.join(tmp, "missing", "index.bin") + with open(os.path.join(tmp, "file"), "wb") as f: + f.write(b'abc') + with self.assertLogs(level='WARNING') as cm: + if sys.platform == "win32": + expect = b'\x88\xee\x84\x87Y\xb21\xaa\x84\x98\x05\xc9&\xcf#4\x05\xe2\xab\x15' + else: + expect = b'\x039\xa9+\xebb\xaf\xbe\x11>@\xf7\xeeMl}\xb6\xd0\x9a\x90' + self.assertEqual(self.hashDirectory(tmp, index), expect) + self.assertTrue(cm.records[0].getMessage().startswith("Cannot write hash cache:")) + + def testBadIndex(self): + """A corrupted index is ignored""" + with TemporaryDirectory() as tmp: + index = os.path.join(tmp, "index.bin") + with open(index, "wb") as f: + f.write(b'garbage') + + ws = os.path.join(tmp, "dir") + os.mkdir(ws) + with open(os.path.join(ws, "file"), 'wb') as f: + f.write(b'abc') + + if sys.platform == "win32": + expect = b'\x88\xee\x84\x87Y\xb21\xaa\x84\x98\x05\xc9&\xcf#4\x05\xe2\xab\x15' + else: + expect = b'\x039\xa9+\xebb\xaf\xbe\x11>@\xf7\xeeMl}\xb6\xd0\x9a\x90' + self.assertEqual(self.hashDirectory(ws, index), expect) + + with open(index, "rb") as f: + self.assertEqual(f.read(4), b'BOB2') + + def testHashPath(self): + """Files, symlinks and missing paths can be hashed directly""" + with TemporaryDirectory() as tmp: + fn = os.path.join(tmp, "file") + with open(fn, "wb") as f: + f.write(b'abc') + os.symlink("file", os.path.join(tmp, "link")) + self.assertEqual(self.hashPath(fn), binascii.unhexlify( + "a9993e364706816aba3e25717850c26c9cd0d89d")) + self.assertEqual(self.hashPath(os.path.join(tmp, "link")), + hashlib.sha1(b'file').digest()) + self.assertEqual(self.hashPath(tmp), self.hashDirectory(tmp)) + with self.assertLogs(level='WARNING') as cm: + self.assertEqual(self.hashPath(os.path.join(tmp, "nx")), b'') + self.assertTrue(cm.records[0].getMessage().startswith("Cannot stat '")) + + + +class TestHashDirPython(CommonHashDirTests, TestCase): + hashDirectory = staticmethod(pyHashDirectory) + hashPath = staticmethod(pyHashPath) + def testBigIno(self): """Test that index handles big inode numbers as found on Windows""" @@ -148,7 +224,7 @@ def testBigIno(self): f.write(b'abc') with patch('os.scandir', entries): - hashDirectory(tmp, index) + self.hashDirectory(tmp, index) with open(index, "rb") as f: self.assertEqual(f.read(4), b'BOB2') @@ -182,7 +258,7 @@ def testOldFile(self): pass with patch('os.scandir', entries): - h = hashDirectory(tmp, index) + h = self.hashDirectory(tmp, index) with patch('os.stat', mock_lstat): b = binStat("whatever") @@ -215,7 +291,7 @@ def testBlockDev(self): with TemporaryDirectory() as indexDir: index = os.path.join(indexDir, "index.bin") with patch('os.scandir', entries): - h = hashDirectory("whatever", index) + h = self.hashDirectory("whatever", index) self.assertEqual(h, b'\xe8\x8e\xad\x9bv\xcbt\xc4\xcd\xa7x\xdb\xde\x96\xab@\x18\xb1\xdcX') @@ -245,7 +321,147 @@ def testChrDev(self): with TemporaryDirectory() as indexDir: index = os.path.join(indexDir, "index.bin") with patch('os.scandir', entries): - h = hashDirectory("whatever", index) + h = self.hashDirectory("whatever", index) self.assertEqual(h, b"\x9b\x98~\xa5\xd5\xc4\x1e\xe29'\x8d\x1e\xe1\x12\xdd\xf4\xa51\xf5d") +@skipIf(rustHashDirectory is None, "rust DirHasher implementation not available") +class TestHashDirRust(CommonHashDirTests, TestCase): + hashDirectory = staticmethod(rustHashDirectory) + hashPath = staticmethod(rustHashPath) + +@skipIf(rustHashDirectory is None, "rust DirHasher implementation not available") +class TestHashDirCompat(TestCase): + """Compare rust implementation against Python implementation""" + + def setUp(self): + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = root = os.path.join(self.tmp.name, "root") + def mk(path, content=b''): + path = os.path.join(root, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(content) + # Names that exercise the sort order of files vs. directories + mk("a-b", b'1') + mk("a/x", b'2') + mk("a0", b'3') + os.chmod(os.path.join(root, "a0"), 0o755) + mk("a.c/y", b'4') + mk("big", os.urandom(1024*1024+17)) + mk("sub/deep/er/file", b'deep') + mk("sub/BaseDirList.txt", b'ignored') + mk(".git/config", b'ignored') + mk("ign/file", b'ignored by option') + mk("sub/ign/file", b'ignored by option') + if sys.platform != "win32": + # On Windows, invalid UTF-8 sequences are not allowed + mk(os.fsdecode(b"y\xe4\xf6\xfc-latin1"), b'5') + mk("Umlaut-\u00e4\u00f6\u00fc", b'6') + # Windows does not support FIFOs + os.mkfifo(os.path.join(root, "fifo")) + os.mkdir(os.path.join(root, "empty")) + if canSymlink(): + os.symlink("a/x", os.path.join(root, "link")) + os.symlink("dangling", os.path.join(root, "sub", "dangling")) + os.symlink("sub", os.path.join(root, "dirlink")) + + def check(self, isDir, path, index=False, ignoreDirs=None): + results = [] + impls = ["py", "rust"] + funs = [pyHashDirectory, rustHashDirectory] if isDir else [pyHashPath, rustHashPath] + for name, func in zip(impls, funs): + indexPath = os.path.join(self.tmp.name, name + ".bin") if index else None + results.append(func(path, indexPath, ignoreDirs)) + results.append(func(path, indexPath, ignoreDirs)) # run again with index cache + self.assertEqual(results[0], results[1]) + self.assertEqual(results[0], results[2]) + self.assertEqual(results[0], results[3]) + return results[0] + + def testHashDirectory(self): + d1 = self.check(True, self.root) + d2 = self.check(True, self.root, ignoreDirs=["ign"]) + self.assertNotEqual(d1, d2) + self.check(True, os.path.join(self.root, "dirlink")) + self.check(True, os.path.join(self.root, "sub")) + + def testManyFiles(self): + """Enough files to keep all threads busy and to prefetch files""" + for i in range(500): + d = os.path.join(self.root, "many", str(i % 7)) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, str(i)), "wb") as f: + f.write(os.urandom(i * 97)) + self.check("hashDirectory", self.root) + self.check("hashDirectory", self.root, True) + + def waitClockTick(self): + """Wait until the file system clock has advanced. + + Files that are not older than the file system time are "racily clean" + and are not recorded in the index. Make sure all files are old enough + so that the index content is deterministic. + """ + def change_time(): + with NamedTemporaryFile(dir=self.tmp.name) as f: + if sys.platform == "win32": + return os.fstat(f.fileno()).st_mtime_ns + else: + return os.fstat(f.fileno()).st_ctime_ns + start = change_time() + while change_time() <= start: + time.sleep(0.001) + + def testHashDirectoryIndex(self): + """Both implementations create the same index""" + self.waitClockTick() + d1 = self.check(True, self.root, True) + self.assertEqual(d1, self.check(True, self.root)) + with open(os.path.join(self.tmp.name, "py.bin"), "rb") as f: + pyIndex = f.read() + with open(os.path.join(self.tmp.name, "rust.bin"), "rb") as f: + rustIndex = f.read() + self.assertEqual(pyIndex, rustIndex) + + def checkIndexUpdate(self): + """Hash once with each implementation and compare the updated indices. + + A second run would fix up a wrongly updated index. Hence each + implementation is invoked just once. + """ + pyIndex = os.path.join(self.tmp.name, "py.bin") + rustIndex = os.path.join(self.tmp.name, "rust.bin") + self.assertEqual(pyHashDirectory(self.root, pyIndex), + rustHashDirectory(self.root, rustIndex)) + with open(pyIndex, "rb") as f: + pyIndex = f.read() + with open(rustIndex, "rb") as f: + rustIndex = f.read() + self.assertEqual(pyIndex, rustIndex) + return pyIndex + + def testHashDirectoryIndexUpdate(self): + """Removed and added files update the index identically""" + self.waitClockTick() + i1 = self.checkIndexUpdate() + + # Entries of removed files are dropped + os.unlink(os.path.join(self.root, "a-b")) + i2 = self.checkIndexUpdate() + self.assertNotIn(b"a-b", i2) + + # Appending after the last entry keeps all other entries + with open(os.path.join(self.root, os.fsdecode(b"zzz-end")), "wb") as f: + f.write(b'end') + self.waitClockTick() + i3 = self.checkIndexUpdate() + self.assertTrue(i3.startswith(i2)) + self.assertIn(b"zzz-end", i3) + + def testHashPath(self): + for p in ("big", "link", "fifo", "sub", "dirlink", "sub/dangling"): + self.check(False, os.path.join(self.root, p)) + self.check(False, os.path.join(self.root, p), True) +