Native hasher implementation

.gitignore

@@ -9,3 +9,5 @@ __pycache__/
 /dist/
 /htmlcov
 /pym/bob/version.py
+/pym/bob/*.so
+/src/dirhasher/target/

pym/bob/builder.py

@@ -52,18 +52,19 @@ async def gatherTasks(tasks):
     await asyncio.wait(tasks)
     return [ t.result() for t in tasks ]
 
-def hashWorkspace(step):
+async def hashWorkspace(step):
+    loop = asyncio.get_event_loop()
+    cachePath = os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin")
     # Cache is also used by share module
-    return hashDirectory(step.getStoragePath(),
-        os.path.join(os.path.dirname(step.getWorkspacePath()), "cache.bin"))
+    return await loop.run_in_executor(None, hashDirectory, step.getStoragePath(), cachePath)
 
 class HashOnce:
     def __init__(self, step):
         self.step = step
         self.result = None
-    def hashWorkspace(self):
+    async def hashWorkspace(self):
         if self.result is None:
-            self.result = hashWorkspace(self.step)
+            self.result = await hashWorkspace(self.step)
         return self.result
     def invalidate(self):
         self.result = None
@@ -1202,7 +1203,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
         if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None):
             inputChanged = checkoutBuildOnlyStateChanged(checkoutState, oldCheckoutState)
             rehash = currentResultHash.hashWorkspace
-            if checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash):
+            if await checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash):
                 if checkoutBuildOnlyStateCompatible(checkoutState, oldCheckoutState):
                     with stepExec(checkoutStep, "UPDATE",
                                   "{} {}".format(prettySrcPath, overridesString)) as a:
@@ -1247,7 +1248,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
             elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)):
                 checkoutReason = "dependency changed"
             elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \
-                 and (oldResultHash != currentResultHash.hashWorkspace()):
+                 and (oldResultHash != await currentResultHash.hashWorkspace()):
                 checkoutReason = "workspace changed"
 
             if checkoutReason:
@@ -1345,7 +1346,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
 
         # We always have to rehash the directory as the user might have
         # changed the source code manually.
-        checkoutHash = currentResultHash.hashWorkspace()
+        checkoutHash = await currentResultHash.hashWorkspace()
 
         # Generate audit trail before setResultHash() to force re-generation if
         # the audit trail fails.
@@ -1413,7 +1414,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId):
             # We always rehash the directory in development mode as the
             # user might have compiled the package manually.
             if not self.__cleanBuild:
-                BobState().setResultHash(prettyBuildPath, hashWorkspace(buildStep))
+                BobState().setResultHash(prettyBuildPath, await hashWorkspace(buildStep))
         else:
             with stepExec(buildStep, "BUILD", prettyBuildPath, (ALWAYS, NORMAL)) as fullAction:
                 # Squash state because running the step will change the
@@ -1425,7 +1426,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId):
                 with stepExec(buildStep, "BUILD", prettyBuildPath, INFO) as buildAction:
                     visibleAction = fullAction if fullAction.visible else buildAction
                     await self._runShell(buildStep, "build", visibleAction, created, self.__cleanBuild)
-                buildHash = hashWorkspace(buildStep)
+                buildHash = await hashWorkspace(buildStep)
                 await self._generateAudit(buildStep, depth, buildHash, buildBuildId)
             BobState().setResultHash(prettyBuildPath, buildHash)
             BobState().setVariantId(prettyBuildPath, buildDigest[0])
@@ -1626,7 +1627,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId):
 
                 # Verify integrity of downloaded package to protect against
                 # data corruption.
-                packageHash = hashWorkspace(packageStep)
+                packageHash = await hashWorkspace(packageStep)
                 if Audit.fromFile(audit).getArtifact().getResultHash() != packageHash:
                     raise BuildError("Corrupt downloaded artifact! Extracted content hash does not match audit trail.")
 
@@ -1686,7 +1687,7 @@ async def _cookPackageStep(self, packageStep, depth, packageBuildId):
                 with stepExec(packageStep, "PACKAGE", prettyPackagePath, INFO) as packageAction:
                     visibleAction = fullAction if fullAction.visible else packageAction
                     await self._runShell(packageStep, "package", visibleAction, True)
-                packageHash = hashWorkspace(packageStep)
+                packageHash = await hashWorkspace(packageStep)
                 packageDigest = await self.__getIncrementalVariantId(packageStep)
                 workspaceChanged = True
                 self.__statistic.packagesBuilt += 1

pym/bob/develop/make.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-__all__ = ('makeSandboxHelper', 'makeManpages')
+__all__ = ('makeSandboxHelper', 'makeManpages', 'makeDirHasher')
 
 def findYoungestInDir(path, exclude):
     ret = 0
@@ -53,6 +53,29 @@ def makeSandboxHelper():
 
     return resultPath
 
+def makeDirHasher():
+    # Linux only so far
+    if sys.platform == "win32":
+        return
+
+    import sysconfig
+    bobRoot = getBobRoot()
+    resultPath = os.path.join(bobRoot, "pym", "bob",
+                              "_dirhasher" + sysconfig.get_config_var("EXT_SUFFIX"))
+    inputPath = os.path.join(bobRoot, "src", "dirhasher")
+    resultDate = findYoungest(resultPath)
+    inputDate = findYoungest(inputPath)
+    if inputDate > resultDate:
+        import shutil
+        if shutil.which("cargo") is None:
+            return
+
+        import subprocess
+        print("Build", resultPath, "...", file=sys.stderr)
+        subprocess.run(["cargo", "build", "--release"], cwd=inputPath, check=True)
+        shutil.copy(os.path.join(inputPath, "target", "release", "lib_dirhasher.so"), resultPath+".new")
+        os.rename(resultPath+".new", resultPath)
+
 def makeManpages():
     bobRoot = getBobRoot()
     resultPath = os.path.join(bobRoot, "doc", "_build", "man")

pym/bob/intermediate.py

@@ -308,14 +308,14 @@ def deserialize(state):
     def getScmDirectories(self):
         return { d : (bytes.fromhex(h), p) for (d, (h, p)) in self.__data['scmDirectories'].items() }
 
-    def mayUpdate(self, inputChanged, oldHash, rehash):
+    async def mayUpdate(self, inputChanged, oldHash, rehash):
         if any((s.isLocal() and not s.isDeterministic()) for s in self.getScmList()):
             return True
         if not self.getUpdateScript():
             return False
         if not self.isUpdateDeterministic() or inputChanged:
             return True
-        return rehash() != oldHash
+        return await rehash() != oldHash
 
     async def getDigestCoro(self, calculate, hasher=DigestHasher,
                             fingerprint=None, platform=b'', relaxTools=False):

pym/bob/scripts.py

@@ -7,7 +7,6 @@
 from .errors import BobError
 from .state import finalize
 from .tty import colorize, Unbuffered, setColorMode, cleanup
-from .utils import asHexStr, hashPath, getPlatformTag, EventLoopWrapper
 import argparse
 import logging
 import sys
@@ -277,6 +276,13 @@ def cmd():
                     print("bob -C: unable to change directory:", str(e), file=sys.stderr)
                     return 1
 
+        # Make sure native extension is updated
+        try:
+            from .develop.make import makeDirHasher
+            makeDirHasher()
+        except ImportError:
+            pass
+
         if args.command in availableCommands:
             cmd = availableCommands[args.command][1]
             if DEBUG['prof']: