Patch support

doc/manual/configuration.rst

@@ -1079,6 +1079,41 @@ can use :ref:`bobpaths_string_literals` and :ref:`bobpaths_string_function_calls
 to express a condition (see :ref:`configuration-principle-booleans`). The SCM
 will only be considered if the condition passes.
 
+patches
+^^^^^^^
+
+All checkout SCMs may declare an ordered ``patches`` list.  Bob applies these
+recipe-relative unified diffs after obtaining the SCM and before running the
+``checkoutScript``.  This makes the patch content part of the checkout
+identity, so changing a patch invalidates dependent artifacts.
+
+Each item names either one ``file``, a non-recursive ``files`` glob, or a
+simple quilt-style ``series`` file; blank lines and comments in a series are
+ignored.  Glob matches are applied in lexicographic order and an unmatched
+glob is an error.  ``strip`` defaults to 1::
+
+    patches:
+      - file: patches/0001-fix-build.patch
+      - files: patches/foo*.patch
+      - series: patches/series
+        strip: 1
+
+Bob applies text unified diffs itself, with exact context. It tries each hunk
+at its declared location first; otherwise, it accepts exactly one full-context
+match within 100 lines and reports the offset. It rejects unsupported patch
+formats (including binary, rename, and mode-only patches) and patch files
+outside the recipe directory. Patches are confined to the SCM's ``dir``; they
+cannot be used to modify another checkout.
+
+When changing a declared patch's ``strip`` level, Bob uses the new level while
+reversing the persisted old patch content during an inline checkout update.
+If an overlay application failed, changing its contents retries the corrected
+overlay against the verified retained base checkout. Bob prepares the entire
+overlay before writing files and restores originals if a write fails. Failed
+checkout state without a verified base is moved to the attic before obtaining
+a fresh checkout. Creation patches cannot overwrite existing files, and patch
+targets must not traverse symlinks.
+
 Each SCM supports a number of specific attributes. See the description of each
 SCM below for the details. Additionally, the following synthetic attributes
 exist. They are generated internally and cannot be set in the recipe. They are

pym/bob/audit.py

@@ -6,12 +6,14 @@
 from . import BOB_INPUT_HASH, DEBUG
 from .errors import BuildError, ParseError
 from .scm import GitAudit, SvnAudit, UrlAudit, ImportAudit, auditFromData
+from .scm.scm import overlayFingerprint
 from .utils import asHexStr, hashFile, binStat
 from datetime import datetime, timezone
 import gzip
 import hashlib
 import io
 import json
+import os
 import platform
 import pickle
 import schema
@@ -195,7 +197,12 @@ async def addScm(self, name, workspace, dir, extra):
         scm = Artifact.SCMS.get(name)
         if scm is None:
             raise BuildError("Cannot handle SCM: " + name)
-        self.__data['scms'].append((await scm.fromDir(workspace, dir, extra)).dump())
+        data = (await scm.fromDir(workspace, dir, extra)).dump()
+        if 'patches' in extra:
+            data['patches'] = extra['patches']
+            data['patchFingerprint'] = overlayFingerprint(
+                os.path.join(workspace, extra['patchDirectory']))
+        self.__data['scms'].append(data)
         self.__invalidateId()
 
     def addTool(self, name, toolId):
@@ -400,4 +407,3 @@ def addArg(self, arg):
         audit = Audit.fromFile(arg)
         self.__merge(audit)
         self.__artifact.addArg(audit.getId())
-

pym/bob/builder.py

@@ -11,6 +11,8 @@
 from .invoker import Invoker, InvocationMode, JobserverConfig
 from .languages import StepSpec
 from .scm import getScm
+from .scm.patch import PatchApplyError
+from .scm.scm import overlayFingerprint
 from .state import BobState
 from .stringparser import Env
 from .tty import log, stepMessage, stepAction, stepExec, setProgress, ttyReinit, \
@@ -70,9 +72,15 @@ def invalidate(self):
 
 CHECKOUT_STATE_VARIANT_ID = None    # Key in checkout directory state for step variant-id
 CHECKOUT_STATE_BUILD_ONLY = 1       # Key for checkout state of build-only builds
+CHECKOUT_STATE_PATCH_OVERLAYS = 2   # Managed patch overlay metadata
+CHECKOUT_STATE_PATCH_OVERLAY_FAILED = 3 # Failed overlay switch; attic next run
+CHECKOUT_STATE_PATCH_BASES = 4     # Verified base fingerprints after failed apply
 
 # Keys in checkout getDirectoryState that are not directories
-CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY}
+CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY,
+                         CHECKOUT_STATE_PATCH_OVERLAYS,
+                         CHECKOUT_STATE_PATCH_OVERLAY_FAILED,
+                         CHECKOUT_STATE_PATCH_BASES}
 
 def compareDirectoryState(left, right):
     """Compare two directory states while ignoring the SCM specs.
@@ -86,8 +94,9 @@ def compareDirectoryState(left, right):
     only relevant for build-only builds that have their dedicated functions
     below.
     """
-    left  = { d : v[0] for d, v in left.items()  if d != CHECKOUT_STATE_BUILD_ONLY }
-    right = { d : v[0] for d, v in right.items() if d != CHECKOUT_STATE_BUILD_ONLY }
+    ignored = CHECKOUT_NON_DIR_KEYS - {CHECKOUT_STATE_VARIANT_ID}
+    left  = { d : v[0] for d, v in left.items()  if d not in ignored }
+    right = { d : v[0] for d, v in right.items() if d not in ignored }
     return left == right
 
 def checkoutsFromState(state):
@@ -1226,10 +1235,14 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
 
             if self.__cleanCheckout:
                 # check state of SCMs and invalidate if the directory is dirty
+                overlayState = oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {})
                 for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState):
                     if scmDigest != checkoutState.get(scmDir, (None, None))[0]: continue
                     if not os.path.exists(os.path.join(prettySrcPath, scmDir)): continue
-                    if scmMap[scmDir].status(checkoutStep.getWorkspacePath()).dirty:
+                    scm = scmMap[scmDir]
+                    status = scm.statusWithOverlay(checkoutStep.getWorkspacePath(),
+                                                   overlayState.get(scmDir))
+                    if status.dirty:
                         # Invalidate scmDigest to forcibly move it away in the loop below.
                         # Do not use None here to distinguish it from a non-existent directory.
                         oldCheckoutState[scmDir] = (False, scmSpec)
@@ -1244,6 +1257,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                 checkoutReason = "indeterministic"
             elif not compareDirectoryState(checkoutState, oldCheckoutState):
                 checkoutReason = "recipe changed"
+            elif oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAY_FAILED):
+                checkoutReason = "failed patch overlay"
             elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)):
                 checkoutReason = "dependency changed"
             elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \
@@ -1256,6 +1271,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                 # checkoutsFromState() to return the directories in a top-down
                 # order.
                 atticPaths = AtticTracker()
+                failedOverlays = oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAY_FAILED, [])
+                failedBases = oldCheckoutState.get(CHECKOUT_STATE_PATCH_BASES, {})
                 for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState):
                     scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir))
                     if atticPaths.affected(scmPath):
@@ -1267,21 +1284,41 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                             BobState().setAtticDirectoryState(atticPath, scmSpec)
                         del oldCheckoutState[scmDir]
                         BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
-                    elif scmDigest != checkoutState.get(scmDir, (None, None))[0]:
-                        canSwitch = (scmDir in scmMap) and scmDigest and \
+                    elif (scmDigest != checkoutState.get(scmDir, (None, None))[0] or
+                          scmDir in failedOverlays):
+                        retryFailedOverlay = (scmDir in failedOverlays and
+                                              scmDigest != checkoutState.get(scmDir, (None, None))[0] and
+                                              scmDir in failedBases and os.path.exists(scmPath) and
+                                              overlayFingerprint(scmPath) == failedBases[scmDir])
+                        canSwitch = (scmDir not in failedOverlays or retryFailedOverlay) and \
+                                     (scmDir in scmMap) and scmDigest and \
                                      scmSpec is not None and \
-                                     scmMap[scmDir].canSwitch(getScm(scmSpec)) and \
+                                     scmMap[scmDir].canSwitch(getScm({**scmSpec,
+                                         '__patchOverlay': oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {}).get(scmDir),
+                                         '__patchOverlayFailed': scmDir in failedOverlays,
+                                         '__patchOverlayBase': failedBases.get(scmDir)})) and \
                                      os.path.exists(scmPath)
                         didSwitch = False
                         if canSwitch:
                             didSwitch = await self.__runScmSwitch(checkoutStep,
-                                scmPath, scmMap[scmDir], scmSpec)
+                                scmPath, scmMap[scmDir], {**scmSpec,
+                                    '__patchOverlay': oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {}).get(scmDir),
+                                    '__patchOverlayFailed': scmDir in failedOverlays,
+                                    '__patchOverlayBase': failedBases.get(scmDir)})
 
                         if didSwitch:
                             oldCheckoutState[scmDir] = checkoutState[scmDir]
                             BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
                             continue
 
+                        if canSwitch and scmMap[scmDir].getOverlayState() is not None:
+                            failedOverlays = sorted(set(failedOverlays) | {scmDir})
+                            oldCheckoutState[CHECKOUT_STATE_PATCH_OVERLAY_FAILED] = failedOverlays
+                            oldCheckoutState.get(CHECKOUT_STATE_PATCH_BASES, {}).pop(scmDir, None)
+                            BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
+                            raise BuildError("Managed patch overlay switch failed; checkout left in place. "
+                                "Run again to move it to the attic.")
+
                         if os.path.exists(scmPath):
                             if not self.__attic:
                                 raise BuildError("SCM '{}' inline switch not possible and move to attic disabled '{}'!"
@@ -1329,13 +1366,35 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                     oldCheckoutHash = datetime.datetime.now()
                     BobState().setResultHash(prettySrcPath, oldCheckoutHash)
 
-                with stepExec(checkoutStep, "CHECKOUT",
-                              "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a:
-                    await self._runShell(checkoutStep, "checkout", a, created)
+                try:
+                    with stepExec(checkoutStep, "CHECKOUT",
+                                  "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a:
+                        await self._runShell(checkoutStep, "checkout", a, created)
+                except BobError as e:
+                    # Only a transactional apply failure establishes a base
+                    # for an inline retry. Other failures require an attic
+                    # move before trying a fresh checkout.
+                    failed = sorted(s.getDirectory() for s in scmList
+                                    if s.getOverlayState() is not None)
+                    if failed:
+                        checkoutState[CHECKOUT_STATE_PATCH_OVERLAY_FAILED] = failed
+                        if isinstance(e, PatchApplyError) and e.baseFingerprint is not None:
+                            checkoutState[CHECKOUT_STATE_PATCH_BASES] = {
+                                e.directory: e.baseFingerprint}
+                        # A failed script has no successful recipe variant.
+                        checkoutState.pop(CHECKOUT_STATE_VARIANT_ID, None)
+                        BobState().setDirectoryState(prettySrcPath, checkoutState)
+                    raise
                 self.__statistic.checkouts += 1
                 checkoutExecuted = True
                 currentResultHash.invalidate() # force recalculation
                 # reflect new checkout state
+                overlays = {s.getDirectory(): {
+                    'manifest': s.getOverlayState(),
+                    'fingerprint': overlayFingerprint(os.path.join(prettySrcPath, s.getDirectory()))
+                } for s in scmList if s.getOverlayState() is not None}
+                if overlays:
+                    checkoutState[CHECKOUT_STATE_PATCH_OVERLAYS] = overlays
                 BobState().setDirectoryState(prettySrcPath, checkoutState)
                 BobState().setInputHashes(prettySrcPath, checkoutInputHashes)
                 BobState().setVariantId(prettySrcPath, await self.__getIncrementalVariantId(checkoutStep))

pym/bob/cmds/build/status.py

@@ -27,6 +27,7 @@
     ScmTaint.modified       : NORMAL,       # modified
     ScmTaint.new            : NORMAL,
     ScmTaint.overridden     : DEBUG,
+    ScmTaint.patched        : INFO,       # expected managed overlay
     ScmTaint.switched       : NORMAL,       # modified
     ScmTaint.unknown        : NORMAL,       # cannot tell, could be modified
     ScmTaint.unpushed_main  : NORMAL,       # modified
@@ -106,6 +107,7 @@ def __init__(self, recurse, verbose, showClean, showOverrides, showAttic):
     def __showCheckoutStep(self, pp, checkoutStep):
         workspace = checkoutStep.getWorkspacePath()
         oldCheckoutState = BobState().getDirectoryState(workspace, True)
+        overlayState = oldCheckoutState.get(2, {})
         checkoutState = checkoutStep.getScmDirectories()
         scms = { scm.getDirectory() : scm for scm in checkoutStep.getScmList() }
         result = {}
@@ -118,7 +120,8 @@ def __showCheckoutStep(self, pp, checkoutStep):
 
             if scmDigest == checkoutState.get(scmDir, (None, None))[0]:
                 # The digest still matches -> use recipe values
-                status = scms[scmDir].status(workspace)
+                scm = scms[scmDir]
+                status = scm.statusWithOverlay(workspace, overlayState.get(scmDir))
             elif scmSpec is not None:
                 # New project that kept scm spec -> compare with that and mark
                 # as attic
@@ -292,4 +295,3 @@ def doStatus(argv, bobRoot):
             printer.showAllDirs(args.attic)
     finally:
         BobState().setSynchronous()
-

pym/bob/input.py

@@ -7,7 +7,7 @@
 from .errors import ParseError, BobError
 from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage
 from .pathspec import PackageSet
-from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
+from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, PatchScm, ScmOverride, \
     auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS
 from .state import BobState
 from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression, \
@@ -1491,7 +1491,15 @@ def getPreRunCmds(self):
         return [s.getProperties(False) for s in self.scmList]
 
     def getJenkinsPreRunCmds(self):
-        return [ s.getProperties(True) for s in self.scmList if not s.hasJenkinsPlugin() ]
+        ret = []
+        for scm in self.scmList:
+            if not scm.hasJenkinsPlugin():
+                ret.append(scm.getProperties(True))
+            else:
+                props = scm.getJenkinsPreRunProperties()
+                if props is not None:
+                    ret.append(props)
+        return ret
 
     def getSetupScript(self):
         return self.corePackage.recipe.checkoutSetupScript
@@ -2411,6 +2419,9 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         for scm in self.__checkoutSCMs:
             scm["__source"] = sourceName
             scm["recipe"] = sourceFile
+            if scm.get("patches"):
+                PatchScm.trackPatchFiles(sourceFile, scm["patches"],
+                                         recipeSet.loadBinary)
         self.__checkoutAsserts = recipe.get("checkoutAssert", [])
         i = 0
         for a in self.__checkoutAsserts:
@@ -4861,4 +4872,3 @@ def find_class(self, module, name):
             return getattr(self.__plugins[module], name)
         else:
             return super().find_class(module, name)
-

pym/bob/scm/__init__.py

@@ -10,6 +10,7 @@
 from .imp import ImportScm, ImportAudit
 from .svn import SvnScm, SvnAudit
 from .url import UrlScm, UrlAudit
+from .patch import PatchScm
 import os.path
 import schema
 
@@ -58,28 +59,33 @@ def auditFromData(data):
         raise ParseError("Error while validating audit: {} {}".format(str(e), str(data)))
 
 def getScm(spec, overrides=[], recipeSet=None):
+    patches = spec.get('patches', [])
     scm = spec["scm"]
     if scm == "git":
-        return GitScm(spec, overrides,
+        ret = GitScm(spec, overrides,
             recipeSet and recipeSet.getPolicy('scmIgnoreUser'),
             recipeSet and recipeSet.getPolicy('gitCommitOnBranch'))
     elif scm == "import":
-        return ImportScm(spec, overrides,
+        ret = ImportScm(spec, overrides,
             recipeSet and recipeSet.getPolicy('pruneImportScm'),
             recipeSet and recipeSet.getPolicy('fixImportScmVariant'),
             recipeSet and recipeSet.getProjectRoot())
     elif scm == "svn":
-        return SvnScm(spec, overrides,
+        ret = SvnScm(spec, overrides,
             recipeSet.getPreMirrors() if recipeSet else [],
             recipeSet.getFallbackMirrors() if recipeSet else [])
     elif scm == "cvs":
-        return CvsScm(spec, overrides)
+        ret = CvsScm(spec, overrides)
     elif scm == "url":
-        return UrlScm(spec, overrides,
+        ret = UrlScm(spec, overrides,
             recipeSet and recipeSet.getPolicy('scmIgnoreUser'),
             recipeSet.getPreMirrors() if recipeSet else [],
             recipeSet.getFallbackMirrors() if recipeSet else [],
             recipeSet and recipeSet.getPolicy('defaultFileMode'),
             recipeSet and recipeSet.getPolicy('urlScmSeparateDownload'))
     else:
         raise ParseError("Unknown SCM '{}'".format(scm))
+    return PatchScm(ret, patches, spec.get('__patchOverlay'),
+                    spec.get('__patchOnly', False),
+                    spec.get('__patchOverlayFailed', False), recipeSet,
+                    spec.get('__patchOverlayBase')) if patches else ret

pym/bob/scm/cvs.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from .scm import Scm, ScmTaint, ScmStatus
+from .patch import PatchScm
 import re
 import schema
 import os
@@ -12,7 +13,8 @@
 class CvsScm(Scm):
 
     DEFAULTS = {
-        schema.Optional('dir') : str
+        schema.Optional('dir') : str,
+        schema.Optional('patches') : PatchScm.PATCHES_SHEMA,
     }
 
     __SCHEMA = {