diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index ea837431..3de391dc 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -1079,6 +1079,41 @@ can use :ref:`bobpaths_string_literals` and :ref:`bobpaths_string_function_calls to express a condition (see :ref:`configuration-principle-booleans`). The SCM will only be considered if the condition passes. +patches +^^^^^^^ + +All checkout SCMs may declare an ordered ``patches`` list. Bob applies these +recipe-relative unified diffs after obtaining the SCM and before running the +``checkoutScript``. This makes the patch content part of the checkout +identity, so changing a patch invalidates dependent artifacts. + +Each item names either one ``file``, a non-recursive ``files`` glob, or a +simple quilt-style ``series`` file; blank lines and comments in a series are +ignored. Glob matches are applied in lexicographic order and an unmatched +glob is an error. ``strip`` defaults to 1:: + + patches: + - file: patches/0001-fix-build.patch + - files: patches/foo*.patch + - series: patches/series + strip: 1 + +Bob applies text unified diffs itself, with exact context. It tries each hunk +at its declared location first; otherwise, it accepts exactly one full-context +match within 100 lines and reports the offset. It rejects unsupported patch +formats (including binary, rename, and mode-only patches) and patch files +outside the recipe directory. Patches are confined to the SCM's ``dir``; they +cannot be used to modify another checkout. + +When changing a declared patch's ``strip`` level, Bob uses the new level while +reversing the persisted old patch content during an inline checkout update. +If an overlay application failed, changing its contents retries the corrected +overlay against the verified retained base checkout. Bob prepares the entire +overlay before writing files and restores originals if a write fails. Failed +checkout state without a verified base is moved to the attic before obtaining +a fresh checkout. Creation patches cannot overwrite existing files, and patch +targets must not traverse symlinks. + Each SCM supports a number of specific attributes. See the description of each SCM below for the details. Additionally, the following synthetic attributes exist. They are generated internally and cannot be set in the recipe. They are diff --git a/pym/bob/audit.py b/pym/bob/audit.py index 45f7a980..81037db4 100644 --- a/pym/bob/audit.py +++ b/pym/bob/audit.py @@ -6,12 +6,14 @@ from . import BOB_INPUT_HASH, DEBUG from .errors import BuildError, ParseError from .scm import GitAudit, SvnAudit, UrlAudit, ImportAudit, auditFromData +from .scm.scm import overlayFingerprint from .utils import asHexStr, hashFile, binStat from datetime import datetime, timezone import gzip import hashlib import io import json +import os import platform import pickle import schema @@ -195,7 +197,12 @@ async def addScm(self, name, workspace, dir, extra): scm = Artifact.SCMS.get(name) if scm is None: raise BuildError("Cannot handle SCM: " + name) - self.__data['scms'].append((await scm.fromDir(workspace, dir, extra)).dump()) + data = (await scm.fromDir(workspace, dir, extra)).dump() + if 'patches' in extra: + data['patches'] = extra['patches'] + data['patchFingerprint'] = overlayFingerprint( + os.path.join(workspace, extra['patchDirectory'])) + self.__data['scms'].append(data) self.__invalidateId() def addTool(self, name, toolId): @@ -400,4 +407,3 @@ def addArg(self, arg): audit = Audit.fromFile(arg) self.__merge(audit) self.__artifact.addArg(audit.getId()) - diff --git a/pym/bob/builder.py b/pym/bob/builder.py index c9a255b9..e591fae4 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -11,6 +11,8 @@ from .invoker import Invoker, InvocationMode, JobserverConfig from .languages import StepSpec from .scm import getScm +from .scm.patch import PatchApplyError +from .scm.scm import overlayFingerprint from .state import BobState from .stringparser import Env from .tty import log, stepMessage, stepAction, stepExec, setProgress, ttyReinit, \ @@ -70,9 +72,15 @@ def invalidate(self): CHECKOUT_STATE_VARIANT_ID = None # Key in checkout directory state for step variant-id CHECKOUT_STATE_BUILD_ONLY = 1 # Key for checkout state of build-only builds +CHECKOUT_STATE_PATCH_OVERLAYS = 2 # Managed patch overlay metadata +CHECKOUT_STATE_PATCH_OVERLAY_FAILED = 3 # Failed overlay switch; attic next run +CHECKOUT_STATE_PATCH_BASES = 4 # Verified base fingerprints after failed apply # Keys in checkout getDirectoryState that are not directories -CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY} +CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY, + CHECKOUT_STATE_PATCH_OVERLAYS, + CHECKOUT_STATE_PATCH_OVERLAY_FAILED, + CHECKOUT_STATE_PATCH_BASES} def compareDirectoryState(left, right): """Compare two directory states while ignoring the SCM specs. @@ -86,8 +94,9 @@ def compareDirectoryState(left, right): only relevant for build-only builds that have their dedicated functions below. """ - left = { d : v[0] for d, v in left.items() if d != CHECKOUT_STATE_BUILD_ONLY } - right = { d : v[0] for d, v in right.items() if d != CHECKOUT_STATE_BUILD_ONLY } + ignored = CHECKOUT_NON_DIR_KEYS - {CHECKOUT_STATE_VARIANT_ID} + left = { d : v[0] for d, v in left.items() if d not in ignored } + right = { d : v[0] for d, v in right.items() if d not in ignored } return left == right def checkoutsFromState(state): @@ -1226,10 +1235,14 @@ async def _cookCheckoutStep(self, checkoutStep, depth): if self.__cleanCheckout: # check state of SCMs and invalidate if the directory is dirty + overlayState = oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {}) for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState): if scmDigest != checkoutState.get(scmDir, (None, None))[0]: continue if not os.path.exists(os.path.join(prettySrcPath, scmDir)): continue - if scmMap[scmDir].status(checkoutStep.getWorkspacePath()).dirty: + scm = scmMap[scmDir] + status = scm.statusWithOverlay(checkoutStep.getWorkspacePath(), + overlayState.get(scmDir)) + if status.dirty: # Invalidate scmDigest to forcibly move it away in the loop below. # Do not use None here to distinguish it from a non-existent directory. oldCheckoutState[scmDir] = (False, scmSpec) @@ -1244,6 +1257,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth): checkoutReason = "indeterministic" elif not compareDirectoryState(checkoutState, oldCheckoutState): checkoutReason = "recipe changed" + elif oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAY_FAILED): + checkoutReason = "failed patch overlay" elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)): checkoutReason = "dependency changed" elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \ @@ -1256,6 +1271,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth): # checkoutsFromState() to return the directories in a top-down # order. atticPaths = AtticTracker() + failedOverlays = oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAY_FAILED, []) + failedBases = oldCheckoutState.get(CHECKOUT_STATE_PATCH_BASES, {}) for (scmDir, (scmDigest, scmSpec)) in checkoutsFromState(oldCheckoutState): scmPath = os.path.normpath(os.path.join(prettySrcPath, scmDir)) if atticPaths.affected(scmPath): @@ -1267,21 +1284,41 @@ async def _cookCheckoutStep(self, checkoutStep, depth): BobState().setAtticDirectoryState(atticPath, scmSpec) del oldCheckoutState[scmDir] BobState().setDirectoryState(prettySrcPath, oldCheckoutState) - elif scmDigest != checkoutState.get(scmDir, (None, None))[0]: - canSwitch = (scmDir in scmMap) and scmDigest and \ + elif (scmDigest != checkoutState.get(scmDir, (None, None))[0] or + scmDir in failedOverlays): + retryFailedOverlay = (scmDir in failedOverlays and + scmDigest != checkoutState.get(scmDir, (None, None))[0] and + scmDir in failedBases and os.path.exists(scmPath) and + overlayFingerprint(scmPath) == failedBases[scmDir]) + canSwitch = (scmDir not in failedOverlays or retryFailedOverlay) and \ + (scmDir in scmMap) and scmDigest and \ scmSpec is not None and \ - scmMap[scmDir].canSwitch(getScm(scmSpec)) and \ + scmMap[scmDir].canSwitch(getScm({**scmSpec, + '__patchOverlay': oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {}).get(scmDir), + '__patchOverlayFailed': scmDir in failedOverlays, + '__patchOverlayBase': failedBases.get(scmDir)})) and \ os.path.exists(scmPath) didSwitch = False if canSwitch: didSwitch = await self.__runScmSwitch(checkoutStep, - scmPath, scmMap[scmDir], scmSpec) + scmPath, scmMap[scmDir], {**scmSpec, + '__patchOverlay': oldCheckoutState.get(CHECKOUT_STATE_PATCH_OVERLAYS, {}).get(scmDir), + '__patchOverlayFailed': scmDir in failedOverlays, + '__patchOverlayBase': failedBases.get(scmDir)}) if didSwitch: oldCheckoutState[scmDir] = checkoutState[scmDir] BobState().setDirectoryState(prettySrcPath, oldCheckoutState) continue + if canSwitch and scmMap[scmDir].getOverlayState() is not None: + failedOverlays = sorted(set(failedOverlays) | {scmDir}) + oldCheckoutState[CHECKOUT_STATE_PATCH_OVERLAY_FAILED] = failedOverlays + oldCheckoutState.get(CHECKOUT_STATE_PATCH_BASES, {}).pop(scmDir, None) + BobState().setDirectoryState(prettySrcPath, oldCheckoutState) + raise BuildError("Managed patch overlay switch failed; checkout left in place. " + "Run again to move it to the attic.") + if os.path.exists(scmPath): if not self.__attic: raise BuildError("SCM '{}' inline switch not possible and move to attic disabled '{}'!" @@ -1329,13 +1366,35 @@ async def _cookCheckoutStep(self, checkoutStep, depth): oldCheckoutHash = datetime.datetime.now() BobState().setResultHash(prettySrcPath, oldCheckoutHash) - with stepExec(checkoutStep, "CHECKOUT", - "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a: - await self._runShell(checkoutStep, "checkout", a, created) + try: + with stepExec(checkoutStep, "CHECKOUT", + "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a: + await self._runShell(checkoutStep, "checkout", a, created) + except BobError as e: + # Only a transactional apply failure establishes a base + # for an inline retry. Other failures require an attic + # move before trying a fresh checkout. + failed = sorted(s.getDirectory() for s in scmList + if s.getOverlayState() is not None) + if failed: + checkoutState[CHECKOUT_STATE_PATCH_OVERLAY_FAILED] = failed + if isinstance(e, PatchApplyError) and e.baseFingerprint is not None: + checkoutState[CHECKOUT_STATE_PATCH_BASES] = { + e.directory: e.baseFingerprint} + # A failed script has no successful recipe variant. + checkoutState.pop(CHECKOUT_STATE_VARIANT_ID, None) + BobState().setDirectoryState(prettySrcPath, checkoutState) + raise self.__statistic.checkouts += 1 checkoutExecuted = True currentResultHash.invalidate() # force recalculation # reflect new checkout state + overlays = {s.getDirectory(): { + 'manifest': s.getOverlayState(), + 'fingerprint': overlayFingerprint(os.path.join(prettySrcPath, s.getDirectory())) + } for s in scmList if s.getOverlayState() is not None} + if overlays: + checkoutState[CHECKOUT_STATE_PATCH_OVERLAYS] = overlays BobState().setDirectoryState(prettySrcPath, checkoutState) BobState().setInputHashes(prettySrcPath, checkoutInputHashes) BobState().setVariantId(prettySrcPath, await self.__getIncrementalVariantId(checkoutStep)) diff --git a/pym/bob/cmds/build/status.py b/pym/bob/cmds/build/status.py index e993d80e..cb1cd626 100644 --- a/pym/bob/cmds/build/status.py +++ b/pym/bob/cmds/build/status.py @@ -27,6 +27,7 @@ ScmTaint.modified : NORMAL, # modified ScmTaint.new : NORMAL, ScmTaint.overridden : DEBUG, + ScmTaint.patched : INFO, # expected managed overlay ScmTaint.switched : NORMAL, # modified ScmTaint.unknown : NORMAL, # cannot tell, could be modified ScmTaint.unpushed_main : NORMAL, # modified @@ -106,6 +107,7 @@ def __init__(self, recurse, verbose, showClean, showOverrides, showAttic): def __showCheckoutStep(self, pp, checkoutStep): workspace = checkoutStep.getWorkspacePath() oldCheckoutState = BobState().getDirectoryState(workspace, True) + overlayState = oldCheckoutState.get(2, {}) checkoutState = checkoutStep.getScmDirectories() scms = { scm.getDirectory() : scm for scm in checkoutStep.getScmList() } result = {} @@ -118,7 +120,8 @@ def __showCheckoutStep(self, pp, checkoutStep): if scmDigest == checkoutState.get(scmDir, (None, None))[0]: # The digest still matches -> use recipe values - status = scms[scmDir].status(workspace) + scm = scms[scmDir] + status = scm.statusWithOverlay(workspace, overlayState.get(scmDir)) elif scmSpec is not None: # New project that kept scm spec -> compare with that and mark # as attic @@ -292,4 +295,3 @@ def doStatus(argv, bobRoot): printer.showAllDirs(args.attic) finally: BobState().setSynchronous() - diff --git a/pym/bob/input.py b/pym/bob/input.py index 26bb00ae..bc726253 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -7,7 +7,7 @@ from .errors import ParseError, BobError from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage from .pathspec import PackageSet -from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \ +from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, PatchScm, ScmOverride, \ auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS from .state import BobState from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression, \ @@ -1491,7 +1491,15 @@ def getPreRunCmds(self): return [s.getProperties(False) for s in self.scmList] def getJenkinsPreRunCmds(self): - return [ s.getProperties(True) for s in self.scmList if not s.hasJenkinsPlugin() ] + ret = [] + for scm in self.scmList: + if not scm.hasJenkinsPlugin(): + ret.append(scm.getProperties(True)) + else: + props = scm.getJenkinsPreRunProperties() + if props is not None: + ret.append(props) + return ret def getSetupScript(self): return self.corePackage.recipe.checkoutSetupScript @@ -2411,6 +2419,9 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b for scm in self.__checkoutSCMs: scm["__source"] = sourceName scm["recipe"] = sourceFile + if scm.get("patches"): + PatchScm.trackPatchFiles(sourceFile, scm["patches"], + recipeSet.loadBinary) self.__checkoutAsserts = recipe.get("checkoutAssert", []) i = 0 for a in self.__checkoutAsserts: @@ -4861,4 +4872,3 @@ def find_class(self, module, name): return getattr(self.__plugins[module], name) else: return super().find_class(module, name) - diff --git a/pym/bob/scm/__init__.py b/pym/bob/scm/__init__.py index 99dc33aa..3bdc1c2b 100644 --- a/pym/bob/scm/__init__.py +++ b/pym/bob/scm/__init__.py @@ -10,6 +10,7 @@ from .imp import ImportScm, ImportAudit from .svn import SvnScm, SvnAudit from .url import UrlScm, UrlAudit +from .patch import PatchScm import os.path import schema @@ -58,24 +59,25 @@ def auditFromData(data): raise ParseError("Error while validating audit: {} {}".format(str(e), str(data))) def getScm(spec, overrides=[], recipeSet=None): + patches = spec.get('patches', []) scm = spec["scm"] if scm == "git": - return GitScm(spec, overrides, + ret = GitScm(spec, overrides, recipeSet and recipeSet.getPolicy('scmIgnoreUser'), recipeSet and recipeSet.getPolicy('gitCommitOnBranch')) elif scm == "import": - return ImportScm(spec, overrides, + ret = ImportScm(spec, overrides, recipeSet and recipeSet.getPolicy('pruneImportScm'), recipeSet and recipeSet.getPolicy('fixImportScmVariant'), recipeSet and recipeSet.getProjectRoot()) elif scm == "svn": - return SvnScm(spec, overrides, + ret = SvnScm(spec, overrides, recipeSet.getPreMirrors() if recipeSet else [], recipeSet.getFallbackMirrors() if recipeSet else []) elif scm == "cvs": - return CvsScm(spec, overrides) + ret = CvsScm(spec, overrides) elif scm == "url": - return UrlScm(spec, overrides, + ret = UrlScm(spec, overrides, recipeSet and recipeSet.getPolicy('scmIgnoreUser'), recipeSet.getPreMirrors() if recipeSet else [], recipeSet.getFallbackMirrors() if recipeSet else [], @@ -83,3 +85,7 @@ def getScm(spec, overrides=[], recipeSet=None): recipeSet and recipeSet.getPolicy('urlScmSeparateDownload')) else: raise ParseError("Unknown SCM '{}'".format(scm)) + return PatchScm(ret, patches, spec.get('__patchOverlay'), + spec.get('__patchOnly', False), + spec.get('__patchOverlayFailed', False), recipeSet, + spec.get('__patchOverlayBase')) if patches else ret diff --git a/pym/bob/scm/cvs.py b/pym/bob/scm/cvs.py index db4636c6..888d47e6 100644 --- a/pym/bob/scm/cvs.py +++ b/pym/bob/scm/cvs.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from .scm import Scm, ScmTaint, ScmStatus +from .patch import PatchScm import re import schema import os @@ -12,7 +13,8 @@ class CvsScm(Scm): DEFAULTS = { - schema.Optional('dir') : str + schema.Optional('dir') : str, + schema.Optional('patches') : PatchScm.PATCHES_SHEMA, } __SCHEMA = { diff --git a/pym/bob/scm/git.py b/pym/bob/scm/git.py index 94c4cfbb..57fe4a2c 100644 --- a/pym/bob/scm/git.py +++ b/pym/bob/scm/git.py @@ -8,6 +8,7 @@ from ..tty import WarnOnce, stepAction, INFO, TRACE, WARNING from ..utils import check_output, joinLines, run, removeUserFromUrl from .scm import Scm, ScmAudit, ScmStatus, ScmTaint +from .patch import PatchScm from shlex import quote from textwrap import dedent, indent from xml.etree import ElementTree @@ -91,6 +92,7 @@ class GitScm(Scm): DEFAULTS = { **__DEFAULTS, schema.Optional('dir') : str, + schema.Optional('patches') : PatchScm.PATCHES_SHEMA, } SCHEMA = schema.Schema({ @@ -1091,6 +1093,8 @@ class GitAudit(ScmAudit): 'commit' : str, 'description' : str, 'dirty' : bool, + schema.Optional('patches') : [dict], + schema.Optional('patchFingerprint') : str, schema.Optional('submodules') : schema.Or(bool, [str]), schema.Optional('recurseSubmodules') : bool, }) diff --git a/pym/bob/scm/imp.py b/pym/bob/scm/imp.py index bde5edf3..5bc903d6 100644 --- a/pym/bob/scm/imp.py +++ b/pym/bob/scm/imp.py @@ -8,6 +8,7 @@ from ..tty import stepAction, INFO, TRACE from ..utils import asHexStr, hashDirectory, emptyDirectory, tarfileOpen from .scm import Scm, ScmAudit +from .patch import PatchScm import base64 import io import os, os.path @@ -115,6 +116,7 @@ class ImportScm(Scm): DEFAULTS = { schema.Optional('dir') : str, + schema.Optional('patches') : PatchScm.PATCHES_SHEMA, schema.Optional('prune') : bool, schema.Optional('recipeRelative') : bool, } @@ -122,6 +124,8 @@ class ImportScm(Scm): __SCHEMA = { 'scm' : 'import', 'url' : str, + schema.Optional('patches') : [dict], + schema.Optional('patchFingerprint') : str, schema.Optional('if') : schema.Or(str, IfExpression), } diff --git a/pym/bob/scm/patch.py b/pym/bob/scm/patch.py new file mode 100644 index 00000000..e728d9f5 --- /dev/null +++ b/pym/bob/scm/patch.py @@ -0,0 +1,584 @@ +# Bob build tool +# SPDX-License-Identifier: GPL-3.0-or-later + +"""A transport-independent, recipe-local unified-diff overlay.""" + +from ..errors import ParseError +from .scm import Scm, ScmStatus, ScmTaint, overlayFingerprint +import base64 +import glob +import hashlib +import os +import re +import schema +import stat +import tempfile + + +# Changing this means the checkout identity changes too. Keep it separate +# from Bob's general version: it describes precisely the patch language Bob +# promises to interpret. +_PATCH_FORMAT_VERSION = 'unified-v1' +_HUNK_HEADER = re.compile(br'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(?: .*)?\n?$') +_MAX_HUNK_OFFSET = 100 + + +class _PatchFormatError(Exception): + pass + + +class PatchApplyError(ParseError): + """Application failure, optionally with a verified retained base tree.""" + + def __init__(self, message, directory, baseFingerprint=None): + super().__init__(message) + self.directory = directory + self.baseFingerprint = baseFingerprint + + +class _PatchTransaction: + """Prepare the whole overlay before writing, and journal writes for rollback.""" + + def __init__(self, root, protected=()): + self.root = os.path.realpath(root) + self.protected = {os.path.normcase(os.path.realpath(os.path.join(self.root, p))) + for p in protected} + self.originals = {} + self.results = {} + self.restored = True + self.messages = [] + + def path(self, target): + if target is None: + return None + path = os.path.join(self.root, target) + resolved = os.path.realpath(path) + if os.path.commonpath((self.root, resolved)) != self.root: + raise _PatchFormatError("target '{}' escapes checkout".format(target)) + relative = os.path.relpath(resolved, self.root) + if (os.path.normcase(resolved) in self.protected or + any(p in ('.git', '.svn', 'CVS') for p in relative.split(os.sep))): + raise _PatchFormatError("target '{}' modifies protected SCM path".format(target)) + # Reject symlinks rather than replacing their referents or leaving + # dangling links behind when a patch deletes a file. + if os.path.normcase(os.path.abspath(path)) != os.path.normcase(resolved): + raise _PatchFormatError("symlink target '{}' is not supported".format(target)) + if path not in self.originals: + try: + mode = os.lstat(path).st_mode + if not stat.S_ISREG(mode): + raise _PatchFormatError("target '{}' is not a regular file".format(target)) + with open(path, 'rb') as f: + data = f.read() + self.originals[path] = (data, stat.S_IMODE(mode)) + except FileNotFoundError: + self.originals[path] = (None, None) + except OSError as e: + raise _PatchFormatError("cannot read '{}': {}".format(target, e)) + self.results[path] = self.originals[path][0] + return path + + @staticmethod + def _write(path, data, mode): + if data is None: + os.unlink(path) + return + fd, temporary = tempfile.mkstemp(prefix='.bob-patch-', dir=os.path.dirname(path)) + try: + with os.fdopen(fd, 'wb') as f: + f.write(data) + # SCM invocations use umask 022; text additions are nonexecutable. + os.chmod(temporary, 0o644 if mode is None else mode) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + def commit(self): + written = [] + try: + for path, data in self.results.items(): + original, mode = self.originals[path] + if data == original: + continue + self._write(path, data, mode) + written.append(path) + except OSError as e: + errors = [] + for path in reversed(written): + try: + self._write(path, *self.originals[path]) + except OSError as rollback: + errors.append(str(rollback)) + self.restored = not errors + raise _PatchFormatError("cannot commit overlay: {}{}".format( + e, '; rollback failed: ' + '; '.join(errors) if errors else '')) + + +def _header_path(line): + """Return a unified-diff header path, excluding its optional timestamp.""" + path = line[4:].split(b'\t', 1)[0].rstrip(b'\r\n') + if not path: + raise _PatchFormatError("empty file name") + try: + return path.decode('utf-8') + except UnicodeDecodeError: + raise _PatchFormatError("file names must be UTF-8") + + +def _target_path(name, strip): + if name == '/dev/null': + return None + parts = name.replace('\\', '/').split('/') + target = '/'.join(parts[strip:]) + if not target or target.startswith('/') or '..' in parts[strip:]: + raise _PatchFormatError("invalid target '{}'".format(name)) + target = os.path.normpath(target) + if target == '.' or os.path.isabs(target) or os.path.splitdrive(target)[0]: + raise _PatchFormatError("invalid target '{}'".format(name)) + return target + + +def _parse_patch(data, strip): + """Parse the deliberately small, deterministic unified-diff dialect.""" + if b'\0' in data: + raise _PatchFormatError("binary patches are not supported") + lines = data.splitlines(keepends=True) + files = [] + i = 0 + # A human-readable mail-style description before the first file is common + # for recipe patches. It has no effect on application, but is retained in + # the patch digest. Do not mistake known non-unified patch formats for a + # description. + while i < len(lines) and not lines[i].startswith(b'--- '): + if lines[i].startswith((b'GIT binary patch', b'Binary files ', b'diff ', + b'Index: ', b'*** ')): + raise _PatchFormatError("unsupported patch format at line {}".format(i + 1)) + i += 1 + while i < len(lines): + if not lines[i].startswith(b'--- '): + if lines[i].startswith((b'GIT binary patch', b'Binary files ')): + raise _PatchFormatError("binary patches are not supported") + raise _PatchFormatError("expected '---' file header at line {}".format(i + 1)) + old_name = _header_path(lines[i]) + i += 1 + if i == len(lines) or not lines[i].startswith(b'+++ '): + raise _PatchFormatError("expected '+++' file header at line {}".format(i + 1)) + new_name = _header_path(lines[i]) + i += 1 + old_target = _target_path(old_name, strip) + new_target = _target_path(new_name, strip) + if old_target is None and new_target is None: + raise _PatchFormatError("both file names are /dev/null") + if old_target is not None and new_target is not None and old_target != new_target: + # diff -u foo.orig foo is a conventional way to distribute a + # modification. It is not a rename: apply against foo itself. + if old_target == new_target + '.orig': + old_target = new_target + else: + raise _PatchFormatError("renames are not supported") + hunks = [] + while i < len(lines) and lines[i].startswith(b'@@ '): + match = _HUNK_HEADER.match(lines[i]) + if match is None: + raise _PatchFormatError("malformed hunk header at line {}".format(i + 1)) + old_start, old_count, new_start, new_count = match.groups() + old_start, new_start = int(old_start), int(new_start) + old_count = int(old_count) if old_count is not None else 1 + new_count = int(new_count) if new_count is not None else 1 + if (old_count and not old_start) or (new_count and not new_start): + raise _PatchFormatError("nonempty hunk has invalid location at line {}" + .format(i + 1)) + i += 1 + records = [] + actual_old = actual_new = 0 + while i < len(lines): + line = lines[i] + if line == b'\\ No newline at end of file\n' or line == b'\\ No newline at end of file': + if not records or not records[-1][1].endswith(b'\n'): + raise _PatchFormatError("misplaced no-newline marker at line {}".format(i + 1)) + records[-1] = (records[-1][0], records[-1][1][:-1]) + i += 1 + continue + if actual_old == old_count and actual_new == new_count: + break + if not line or line[:1] not in (b' ', b'+', b'-'): + raise _PatchFormatError("invalid hunk line at line {}".format(i + 1)) + records.append((line[:1], line[1:])) + actual_old += line[:1] in (b' ', b'-') + actual_new += line[:1] in (b' ', b'+') + if actual_old > old_count or actual_new > new_count: + raise _PatchFormatError("hunk line count does not match header") + i += 1 + if actual_old != old_count or actual_new != new_count: + raise _PatchFormatError("hunk line count does not match header") + hunks.append((old_start, old_count, new_start, new_count, records)) + if not hunks: + raise _PatchFormatError("file '{}' has no hunks".format(old_name)) + files.append((old_target, new_target, hunks)) + if not files: + raise _PatchFormatError("empty patch") + return files + + +class _Patch: + """One resolved patch and the metadata needed to replay it safely.""" + + def __init__(self, path, name, strip, digest, data): + self.path = path + self.name = name + self.strip = strip + self.digest = digest + self.data = data + self.__parsed = None + + @classmethod + def fromSpec(cls, root, name, spec, loadBinary=None): + data = spec.get('__data') + if data is not None: + try: + data = base64.b64decode(data, validate=True) + except (TypeError, ValueError) as e: + raise ParseError("Invalid persisted patch '{}': {}".format(name, e)) + digest = hashlib.sha256(data).hexdigest() + if digest != spec.get('digestSHA256', digest): + raise ParseError("Persisted patch digest does not match '{}'".format(name)) + path = None + else: + path = os.path.join(root, name) + try: + if loadBinary is None: + with open(path, 'rb') as f: + data = f.read() + else: + # Besides reading the data this registers the patch with + # RecipeSet's package-cache digest. + data = loadBinary(path) + except OSError as e: + raise ParseError("Cannot read patch '{}': {}".format(name, e)) + digest = hashlib.sha256(data).hexdigest() + + strip = spec.get('strip', 1) + if not isinstance(strip, int) or isinstance(strip, bool) or strip < 0: + raise ParseError("Invalid patch strip level: '{}'".format(strip)) + return cls(path, name, strip, digest, data) + + def asProperty(self): + return {'file': self.name, 'strip': self.strip, + 'digestSHA256': self.digest, + '__data': base64.b64encode(self.data).decode('ascii')} + + def asManifestEntry(self): + return {'path': self.name, 'strip': self.strip, + 'digestSHA256': self.digest} + + def withStrip(self, strip): + """Replay these bytes using an updated target-path configuration.""" + return _Patch(self.path, self.name, strip, self.digest, self.data) + + def validateTargets(self, protected): + try: + parsed = self.__parse() + except _PatchFormatError as e: + raise ParseError("Unsupported patch '{}': {}".format(self.name, e)) + protected = {os.path.normcase(os.path.normpath(p)) for p in protected} + for old_target, new_target, hunks in parsed: + for target in (old_target, new_target): + if target is None: + continue + if (any(p in ('.git', '.svn', 'CVS') for p in target.replace('\\', '/').split('/')) or + os.path.normcase(target) in protected): + raise ParseError("Patch modifies protected SCM path: '{}'".format(target)) + + def __parse(self): + if self.__parsed is None: + self.__parsed = _parse_patch(self.data, self.strip) + return self.__parsed + + async def run(self, invoker, directory, reverse=False, transaction=None): + """Apply the stored patch directly, without relying on host tools.""" + try: + ownTransaction = transaction is None + if ownTransaction: + transaction = _PatchTransaction(invoker.joinPath(directory)) + parsed = self.__parse() + for old_target, new_target, hunks in reversed(parsed) if reverse else parsed: + self.__apply_file(transaction, old_target, new_target, hunks, reverse) + if ownTransaction: + transaction.commit() + for message in transaction.messages: + invoker.info(message) + except _PatchFormatError as e: + raise ParseError("Cannot apply patch '{}': {}".format(self.name, e)) + + @staticmethod + def __apply_file(transaction, old_target, new_target, hunks, reverse): + source, destination = (new_target, old_target) if reverse else (old_target, new_target) + source_path = transaction.path(source) + destination_path = transaction.path(destination) + if source_path is None: + if transaction.results[destination_path] is not None: + raise _PatchFormatError("creation target '{}' already exists".format(destination)) + contents = [] + else: + data = transaction.results[source_path] + if data is None: + raise _PatchFormatError("source file '{}' does not exist".format(source)) + contents = data.splitlines(keepends=True) + offset = 0 + for old_start, old_count, new_start, new_count, records in hunks: + source_start = new_start if reverse else old_start + source_count = new_count if reverse else old_count + declared_at = source_start - (1 if source_count else 0) + offset + expected, replacement = [], [] + for kind, value in records: + if reverse: + kind = {b'+': b'-', b'-': b'+'}.get(kind, kind) + if kind in (b' ', b'-'): + expected.append(value) + if kind in (b' ', b'+'): + replacement.append(value) + at = declared_at + matches_declared_location = ( + 0 <= at <= len(contents) - len(expected) and + contents[at:at + len(expected)] == expected) + if not matches_declared_location: + # An insertion has no source-side text to identify another + # location. It must remain anchored at its declared point. + if not expected: + raise _PatchFormatError( + "hunk does not match declared location in '{}' at line {}" + .format(source or destination, source_start)) + lower = max(0, declared_at - _MAX_HUNK_OFFSET) + upper = min(len(contents) - len(expected), + declared_at + _MAX_HUNK_OFFSET) + matches = [candidate for candidate in range(lower, upper + 1) + if contents[candidate:candidate + len(expected)] == expected] + if not matches: + raise _PatchFormatError( + "hunk does not match declared location in '{}' at line {}" + .format(source or destination, source_start)) + if len(matches) > 1: + raise _PatchFormatError( + "hunk matches multiple locations in '{}' near line {}" + .format(source or destination, source_start)) + at = matches[0] + transaction.messages.append("Applied patch hunk in '{}' at offset {:+d}".format( + source or destination, at - declared_at)) + contents[at:at + len(expected)] = replacement + offset += len(replacement) - len(expected) + if destination_path is None: + if contents: + raise _PatchFormatError("deletion of '{}' left content behind".format(source)) + transaction.results[source_path] = None + return + if not os.path.isdir(os.path.dirname(destination_path)): + raise _PatchFormatError("destination directory for '{}' does not exist".format(destination)) + transaction.results[destination_path] = b''.join(contents) + + +class PatchScm(Scm): + PATCHES_SHEMA = [schema.Schema({ + schema.Or('file', 'files', 'series') : str, + schema.Optional('strip') : int, + })] + + def __init__(self, scm, patches, overlay=None, patchOnly=False, overlayFailed=False, + recipeSet=None, failedBase=None): + self.__scm = scm + self.__patches = self.__resolve(patches, recipeSet) + self.__overlay = overlay + self.__patchOnly = patchOnly + self.__overlayFailed = overlayFailed + self.__failedBase = failedBase + + def __resolve(self, patches, recipeSet): + root = os.path.dirname(self.__scm._getRecipe().split('#', 1)[0]) + names = self.__resolvePaths(root, patches) + loadBinary = getattr(recipeSet, 'loadBinary', None) + return [_Patch.fromSpec(root, name, patch, + loadBinary) + for name, patch in names] + + @classmethod + def trackPatchFiles(cls, recipe, patches, loadBinary): + """Include recipe patch files in the persistent package-cache key.""" + root = os.path.dirname(recipe.split('#', 1)[0]) + for patch in patches: + if 'series' in patch: + cls.__validateRecipePath(patch['series']) + loadBinary(os.path.join(root, patch['series'])) + for name, patch in cls.__resolvePaths(root, patches): + loadBinary(os.path.join(root, name)) + + @classmethod + def __resolvePaths(cls, root, patches): + ret = [] + for patch in patches: + if 'file' in patch: + files = [patch['file']] + elif 'files' in patch: + pattern = patch['files'] + cls.__validateRecipePath(pattern) + if '**' in pattern: + raise ParseError("Recursive patch glob is not supported: '{}'".format(pattern)) + files = [os.path.relpath(path, root) for path in + sorted(glob.glob(os.path.join(root, pattern))) + if os.path.isfile(path)] + if not files: + raise ParseError("Patch glob '{}' did not match any files".format(pattern)) + elif 'series' in patch: + cls.__validateRecipePath(patch['series']) + series = os.path.join(root, patch['series']) + try: + with open(series, encoding='utf-8') as f: + files = [] + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + fields = line.split() + if len(fields) != 1: + raise ParseError("Patch series '{}' has unsupported options" + .format(patch['series'])) + files.append(fields[0]) + except OSError as e: + raise ParseError("Cannot read patch series '{}': {}".format(patch['series'], e)) + files = [os.path.join(os.path.dirname(patch['series']), f) for f in files] + else: + raise ParseError("Patch entry needs 'file', 'files', or 'series'") + for name in files: + cls.__validateRecipePath(name) + ret.append((name, patch)) + return ret + + @staticmethod + def __validateRecipePath(name): + if os.path.isabs(name) or '..' in os.path.normpath(name).split(os.sep): + raise ParseError("Patch path escapes recipe directory: '{}'".format(name)) + + def getProperties(self, isJenkins, pretty=False): + ret = self.__scm.getProperties(isJenkins, pretty) + ret['patches'] = [patch.asProperty() for patch in self.__patches] + return ret + + async def invoke(self, invoker, workspaceCreated): + if not self.__patchOnly: + await self.__scm.invoke(invoker, workspaceCreated) + await self.__apply(invoker) + + async def __apply(self, invoker): + protected = set(self.__scm.getProtectedPaths()) + directory = self.__scm.getDirectory() + transaction = _PatchTransaction(invoker.joinPath(directory), protected) + base = overlayFingerprint(transaction.root) + try: + for patch in self.__patches: + patch.validateTargets(protected) + await patch.run(invoker, directory, transaction=transaction) + transaction.commit() + for message in transaction.messages: + invoker.info(message) + except (ParseError, _PatchFormatError) as e: + raise PatchApplyError(e.slogan if isinstance(e, ParseError) else str(e), directory, + base if transaction.restored else None) from e + + async def __unapply(self, invoker, stripOverrides=None): + if self.__overlay is not None: + path = invoker.joinPath(self.__scm.getDirectory()) + fingerprint = overlayFingerprint(path) + if fingerprint != self.__overlay.get('fingerprint'): + raise ParseError("Patched checkout was modified; refusing to reverse managed overlay") + protected = set(self.__scm.getProtectedPaths()) + transaction = _PatchTransaction(invoker.joinPath(self.__scm.getDirectory()), protected) + for patch in reversed(self.__patches): + if stripOverrides is not None and patch.name in stripOverrides: + patch = patch.withStrip(stripOverrides[patch.name]) + patch.validateTargets(protected) + await patch.run(invoker, self.__scm.getDirectory(), reverse=True, + transaction=transaction) + try: + transaction.commit() + for message in transaction.messages: + invoker.info(message) + except _PatchFormatError as e: + raise ParseError("Cannot reverse overlay: {}".format(e)) from e + + def canSwitch(self, oldScm): + """A patch-only recipe change is an inline SCM transition. + + The old patch contents and expected workspace fingerprint are persisted + with the checkout state, so a changed stack can be reversed safely. + """ + return (isinstance(oldScm, PatchScm) and + (not oldScm.__overlayFailed or oldScm.__failedBase is not None) and + self.__scm.canSwitch(oldScm.__scm)) + + async def switch(self, invoker, oldScm): + if oldScm.__overlayFailed: + if (oldScm.__failedBase is None or + overlayFingerprint(invoker.joinPath(self.getDirectory())) != oldScm.__failedBase): + raise ParseError("Failed overlay has no verified unchanged base; fresh checkout required") + return await self.__scm.switch(invoker, oldScm.__scm) + if oldScm.__overlay is None: + # Legacy state without a managed-overlay fingerprint cannot + # establish that the old patch was applied. + # Leave the base checkout in place for the corrected overlay. + return await self.__scm.switch(invoker, oldScm.__scm) + # Keep old bytes for a safe reverse, but use the current path mapping. + # A corrected strip level must be able to recover an existing checkout. + strips = {patch.name: patch.strip for patch in self.__patches} + await oldScm.__unapply(invoker, strips) + # The normal checkout invocation following a successful inline switch + # refreshes the base SCM and applies this overlay exactly once. + return await self.__scm.switch(invoker, oldScm.__scm) + + def asDigestScript(self): + return self.__scm.asDigestScript() + ' patch-format {}'.format(_PATCH_FORMAT_VERSION) + ''.join( + ' patch {} p{}'.format(patch.digest, patch.strip) + for patch in self.__patches) + + def getOverlayState(self): + """Return serializable manifest data for Bob's checkout state.""" + return [patch.asManifestEntry() for patch in self.__patches] + + def getDirectory(self): return self.__scm.getDirectory() + def getSource(self): return self.__scm.getSource() + def hasJenkinsPlugin(self): + return self.__scm.hasJenkinsPlugin() + def asJenkins(self, workPath, config): + return self.__scm.asJenkins(workPath, config) + def getJenkinsPreRunProperties(self): + if self.__scm.hasJenkinsPlugin(): + props = self.getProperties(True) + props['__patchOnly'] = True + return props + return None + def isDeterministic(self): return self.__scm.isDeterministic() + def isLocal(self): return self.__scm.isLocal() + def hasLiveBuildId(self): return self.__scm.hasLiveBuildId() + async def predictLiveBuildId(self, step): return await self.__scm.predictLiveBuildId(step) + def calcLiveBuildId(self, workspacePath): return self.__scm.calcLiveBuildId(workspacePath) + def getAuditSpec(self): + spec = self.__scm.getAuditSpec() + if spec is None: + return None + typ, directory, extra = spec + extra = extra.copy() + extra['patches'] = self.getOverlayState() + extra['patchDirectory'] = self.getDirectory() + return typ, directory, extra + def status(self, workspacePath, overlay=None): + return self.statusWithOverlay(workspacePath, overlay) + + def statusWithOverlay(self, workspacePath, overlay=None): + status = self.__scm.status(workspacePath) + if overlay is not None: + path = os.path.join(workspacePath, self.__scm.getDirectory()) + fingerprint = overlayFingerprint(path) + if fingerprint == overlay.get('fingerprint'): + status.remove(ScmTaint.modified) + status.add(ScmTaint.patched, 'Declared patch overlay') + return status + def getActiveOverrides(self): return self.__scm.getActiveOverrides() + def postAttic(self, workspace): return self.__scm.postAttic(workspace) diff --git a/pym/bob/scm/scm.py b/pym/bob/scm/scm.py index 4a5b2e6f..32e18b09 100644 --- a/pym/bob/scm/scm.py +++ b/pym/bob/scm/scm.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from ..errors import ParseError -from ..utils import joinLines +from ..utils import joinLines, hashDirectory from abc import ABCMeta, abstractmethod from enum import Enum from shlex import quote @@ -13,6 +13,10 @@ SYNTHETIC_SCM_PROPS = frozenset(('__source', 'recipe', 'overridden')) +def overlayFingerprint(path): + """Fingerprint policy shared by overlay state, status and audit records.""" + return hashDirectory(path, ignoreDirs=['.git', '.svn', 'CVS']).hex() + def _freezeValue(value): """Recursively turn a (possibly nested) value into a hashable one. @@ -132,6 +136,7 @@ class ScmTaint(Enum): modified = 'M' new = 'N' overridden = 'O' + patched = 'P' switched = 'S' unknown = '?' unpushed_main = 'U' @@ -221,6 +226,9 @@ def add(self, flag, description=""): else: self.__flags[flag] = description + def remove(self, flag): + self.__flags.pop(flag, None) + def merge(self, other): for flag,description in other.__flags.items(): self.add(flag, description) @@ -316,6 +324,10 @@ def hasJenkinsPlugin(self): """Does this SCM use a Jenins plugin?""" return False + def getJenkinsPreRunProperties(self): + """Return an optional SCM command to run after a Jenkins plugin.""" + return None + @abstractmethod def getDirectory(self): """Return relative directory that this SCM owns in the workspace.""" @@ -381,6 +393,18 @@ def calcLiveBuildId(self, workspacePath): def postAttic(self, workspace): pass + def getProtectedPaths(self): + """Paths below this SCM directory that patch overlays may not modify.""" + return [] + + def getOverlayState(self): + """Return an overlay manifest, or None for an ordinary SCM.""" + return None + + def statusWithOverlay(self, workspacePath, overlay=None): + """Inspect checkout state without changing the SCM status interface.""" + return self.status(workspacePath) + class ScmAudit(metaclass=ABCMeta): @classmethod async def fromDir(cls, workspace, dir, extra): diff --git a/pym/bob/scm/svn.py b/pym/bob/scm/svn.py index 7edaa756..b3fa53d3 100644 --- a/pym/bob/scm/svn.py +++ b/pym/bob/scm/svn.py @@ -7,6 +7,7 @@ from ..stringparser import IfExpression from ..utils import joinLines, check_output from .scm import Scm, ScmAudit, ScmTaint, ScmStatus +from .patch import PatchScm from shlex import quote from textwrap import indent import os, os.path @@ -31,6 +32,7 @@ class SvnScm(Scm): DEFAULTS = { **__DEFAULTS, schema.Optional('dir') : str, + schema.Optional('patches') : PatchScm.PATCHES_SHEMA, } SCHEMA = schema.Schema({ @@ -241,7 +243,9 @@ class SvnAudit(ScmAudit): 'repository' : { 'root' : str, 'uuid' : str - } + }, + schema.Optional('patches') : [dict], + schema.Optional('patchFingerprint') : str, }) async def _scanDir(self, workspace, dir, extra): diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index d95b254f..f90796f1 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -9,6 +9,7 @@ from ..utils import asHexStr, hashFile, removeUserFromUrl, sslNoVerifyContext, \ replacePath from .scm import Scm, ScmAudit +from .patch import PatchScm from http.client import HTTPException from abc import abstractmethod import asyncio @@ -316,6 +317,7 @@ class UrlScm(Scm): DEFAULTS = { **__DEFAULTS, schema.Optional('dir') : str, + schema.Optional('patches') : PatchScm.PATCHES_SHEMA, } SCHEMA = schema.Schema({ @@ -787,6 +789,10 @@ def asDigestScript(self): def getDirectory(self): return self.__dir + def getProtectedPaths(self): + # The retained download is the verified source used for re-extraction. + return [self.__fn] + def isDeterministic(self): return (self.__digestSha1 is not None) or \ (self.__digestSha256 is not None) or \ @@ -846,6 +852,8 @@ class UrlAudit(ScmAudit): 'value' : str }, schema.Optional('url') : str, # Added in Bob 0.16 + schema.Optional('patches') : [dict], + schema.Optional('patchFingerprint') : str, }) async def _scanDir(self, workspace, dir, extra): diff --git a/pym/bob/state.py b/pym/bob/state.py index d9e36c5f..7c24d99a 100644 --- a/pym/bob/state.py +++ b/pym/bob/state.py @@ -595,7 +595,7 @@ def getDirectoryState(self, path, isSourceDir): ret = copy.deepcopy(self.__dirStates.get(path, {} if isSourceDir else None)) if isSourceDir: # convert from old format if necessary - ret = { k : v if isinstance(v, tuple) else (v, None) + ret = { k : v if isinstance(k, int) or isinstance(v, tuple) else (v, None) for k, v in ret.items() } return ret @@ -788,4 +788,3 @@ def finalize(): if _BobState.instance is not None: _BobState.instance.finalize() _BobState.instance = None - diff --git a/test/unit/test_audit.py b/test/unit/test_audit.py index 2ea01a25..8dae62ee 100644 --- a/test/unit/test_audit.py +++ b/test/unit/test_audit.py @@ -6,9 +6,15 @@ from unittest import TestCase from unittest.mock import Mock, patch from collections import namedtuple +import asyncio import datetime +import os +import tempfile +import subprocess from bob.audit import Artifact +from bob.scm import auditFromData +from bob.utils import asHexStr, hashDirectory Uname = namedtuple('Uname', ['system', 'node', 'release', 'version', 'machine', 'processor']) @@ -38,3 +44,47 @@ def testStableArtifactId(self, a): a = Artifact(fixedTime) a.setSandbox(b'\x13' * 20) self.assertEqual(a.getId(), b'\xd5\xd1d\x1c\x1d\x81\x06\xa1\xcf\xf4\x10\x1c\xce\xec\xdae)Pr\x96') + + def testPatchAuditIncludesPatchedTreeFingerprint(self): + with tempfile.TemporaryDirectory() as workspace: + with open(os.path.join(workspace, 'archive'), 'w') as f: + f.write('patched source') + artifact = Artifact(fixedTime) + asyncio.run(artifact.addScm('url', workspace, 'archive', { + 'url': 'file:///tmp/archive', 'patchDirectory': '.', + 'patches': [{'path': 'fix.patch', 'strip': 1, + 'digestSHA256': '0' * 64}], + })) + scm = artifact.dump()['scms'][0] + self.assertEqual(scm['patches'][0]['path'], 'fix.patch') + self.assertEqual(scm['patchFingerprint'], + asHexStr(hashDirectory(workspace))) + + def testPatchedGitAuditRoundTrip(self): + with tempfile.TemporaryDirectory() as workspace: + def git(*args): + subprocess.run(['git', *args], cwd=workspace, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + git('init') + git('config', 'user.name', 'Test') + git('config', 'user.email', 'test@example.invalid') + with open(os.path.join(workspace, 'message'), 'w') as f: + f.write('base\n') + git('add', 'message') + git('commit', '-m', 'base') + with open(os.path.join(workspace, 'message'), 'w') as f: + f.write('patched\n') + artifact = Artifact(fixedTime) + asyncio.run(artifact.addScm('git', workspace, '.', { + 'patchDirectory': '.', 'patches': [{ + 'path': 'fix.patch', 'strip': 1, 'digestSHA256': '0' * 64}], + })) + data = artifact.dump() + self.assertEqual(Artifact.fromData(data).dump(), data) + scm = data['scms'][0] + self.assertEqual(scm['patches'][0]['path'], 'fix.patch') + self.assertEqual(scm['patchFingerprint'], + asHexStr(hashDirectory(workspace, ignoreDirs=['.git', '.svn', 'CVS']))) + legacy = {k: v for k, v in scm.items() + if k not in ('patches', 'patchFingerprint')} + self.assertEqual(auditFromData(legacy).dump(), legacy) diff --git a/test/unit/test_scm_patch.py b/test/unit/test_scm_patch.py new file mode 100644 index 00000000..87f5e20e --- /dev/null +++ b/test/unit/test_scm_patch.py @@ -0,0 +1,527 @@ +# Bob build tool +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Regression coverage for managed, recipe-local SCM patch overlays.""" + +from unittest import TestCase, skipUnless +from unittest.mock import patch +import asyncio +import os +import subprocess +import tempfile + +from bob.errors import ParseError +from bob.scm.patch import PatchScm, PatchApplyError, _PatchTransaction +from bob.scm.scm import Scm, ScmStatus, ScmTaint, overlayFingerprint +from bob.scm.url import UrlScm +from bob.utils import hashDirectory, canSymlink + + +class DummyScm(Scm): + def __init__(self, recipe, directory='src', protected=()): + self.recipe = recipe + self.directory = directory + self.protected = protected + + def _getRecipe(self): return self.recipe + def getSource(self): return 'dummy source' + def getDirectory(self): return self.directory + def getProtectedPaths(self): return self.protected + async def invoke(self, invoker, workspaceCreated): pass + async def switch(self, invoker, oldScm): return True + def canSwitch(self, oldScm): return isinstance(oldScm, DummyScm) + def getProperties(self, isJenkins, pretty=False): + return {'scm': 'dummy', 'recipe': self.recipe, 'dir': self.directory} + def hasJenkinsPlugin(self): return False + def asDigestScript(self): return 'dummy' + def status(self, workspace): return ScmStatus(ScmTaint.modified) + def isDeterministic(self): return True + def isLocal(self): return False + def hasLiveBuildId(self): return False + async def predictLiveBuildId(self, step): return None + def calcLiveBuildId(self, workspace): return None + def getAuditSpec(self): return None + def getActiveOverrides(self): return set() + def postAttic(self, workspace): pass + + +class PluginScm(DummyScm): + def __init__(self, recipe): + super().__init__(recipe) + self.invoked = False + + async def invoke(self, invoker, workspaceCreated): self.invoked = True + def hasJenkinsPlugin(self): return True + def asJenkins(self, workPath, config): return 'plugin' + + +class Invoker: + def __init__(self, workspace): + self.workspace = workspace + self.infoMessages = [] + + def joinPath(self, *parts): return os.path.join(self.workspace, *parts) + + def info(self, *args): self.infoMessages.append(' '.join(map(str, args))) + + async def checkCommand(self, args, cwd=None): + subprocess.run(args, cwd=self.joinPath(cwd), check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + +class TestPatchScm(TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.recipe = os.path.join(self.temp.name, 'recipe.yaml') + self.workspace = os.path.join(self.temp.name, 'workspace') + os.makedirs(os.path.join(self.workspace, 'src')) + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('old\n') + + def tearDown(self): + self.temp.cleanup() + + def writePatch(self, body): + with open(os.path.join(self.temp.name, 'change.patch'), 'w') as f: + f.write(body) + + def scm(self, patches, overlay=None): + return PatchScm(DummyScm(self.recipe), patches, overlay) + + def test_changed_patch_reverses_persisted_old_contents(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + old = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(old.invoke(invoker, False)) + properties = old.getProperties(False) + + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+second\n') + new = self.scm([{'file': 'change.patch'}]) + persisted = self.scm(properties['patches'], { + 'fingerprint': hashDirectory(os.path.join(self.workspace, 'src')).hex()}) + asyncio.run(new.switch(invoker, persisted)) + asyncio.run(new.invoke(invoker, False)) + + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'second\n') + + def test_changed_strip_reverses_persisted_contents_with_new_strip(self): + os.makedirs(os.path.join(self.workspace, 'src', 'workspace')) + with open(os.path.join(self.workspace, 'src', 'setupinfo.py'), 'w') as f: + f.write('old\n') + self.writePatch('--- workspace/setupinfo.py.orig\n+++ workspace/setupinfo.py\n' + '@@ -1 +1 @@\n-old\n+new\n') + invoker = Invoker(self.workspace) + # This represents a checkout state written by an earlier overlay + # configuration. Its contents are patched, but its strip was wrong. + with open(os.path.join(self.workspace, 'src', 'setupinfo.py'), 'w') as f: + f.write('new\n') + old = self.scm([{'file': 'change.patch', 'strip': 0}], { + 'fingerprint': hashDirectory(os.path.join(self.workspace, 'src')).hex()}) + new = self.scm([{'file': 'change.patch', 'strip': 1}]) + + asyncio.run(new.switch(invoker, old)) + asyncio.run(new.invoke(invoker, False)) + + with open(os.path.join(self.workspace, 'src', 'setupinfo.py')) as f: + self.assertEqual(f.read(), 'new\n') + + def test_failed_old_overlay_is_not_reversed_before_applying_fix(self): + self.writePatch('--- a/missing\n+++ b/missing\n@@ -1 +1 @@\n-old\n+bad\n') + failed = PatchScm(DummyScm(self.recipe), [{'file': 'change.patch'}], + overlayFailed=True, + failedBase=overlayFingerprint(os.path.join(self.workspace, 'src'))) + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+fixed\n') + new = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + + asyncio.run(new.switch(invoker, failed)) + asyncio.run(new.invoke(invoker, False)) + + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'fixed\n') + + def test_unmanaged_old_overlay_is_not_reversed_before_applying_fix(self): + self.writePatch('--- a/missing\n+++ b/missing\n@@ -1 +1 @@\n-old\n+bad\n') + # Older checkout state has patch bytes but no overlay fingerprint, so + # it cannot establish that this overlay was ever applied. + old = self.scm([{'file': 'change.patch'}]) + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+fixed\n') + new = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + + asyncio.run(new.switch(invoker, old)) + asyncio.run(new.invoke(invoker, False)) + + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'fixed\n') + + def test_delegates_source_for_checkout_error_reporting(self): + self.assertEqual(self.scm([]).getSource(), 'dummy source') + + def test_legacy_failed_overlay_cannot_switch_without_verified_base(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n') + failed = PatchScm(DummyScm(self.recipe), [{'file': 'change.patch'}], + overlayFailed=True) + new = self.scm([{'file': 'change.patch'}]) + self.assertFalse(new.canSwitch(failed)) + with self.assertRaisesRegex(ParseError, 'verified unchanged base'): + asyncio.run(new.switch(Invoker(self.workspace), failed)) + + def test_modified_failed_base_is_not_reused(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n') + failed = PatchScm(DummyScm(self.recipe), [{'file': 'change.patch'}], + overlayFailed=True, + failedBase=overlayFingerprint(os.path.join(self.workspace, 'src'))) + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('user modification\n') + with self.assertRaisesRegex(ParseError, 'verified unchanged base'): + asyncio.run(self.scm([{'file': 'change.patch'}]).switch( + Invoker(self.workspace), failed)) + + def test_failure_in_later_file_leaves_complete_tree_unchanged(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n' + '--- a/missing\n+++ b/missing\n@@ -1 +1 @@\n-old\n+new\n') + before = overlayFingerprint(os.path.join(self.workspace, 'src')) + with self.assertRaises(PatchApplyError) as error: + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + self.assertEqual(error.exception.baseFingerprint, before) + self.assertEqual(overlayFingerprint(os.path.join(self.workspace, 'src')), before) + + def test_failure_in_later_patch_leaves_complete_stack_unchanged(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n') + with open(os.path.join(self.temp.name, 'bad.patch'), 'w') as f: + f.write('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-wrong\n+bad\n') + before = overlayFingerprint(os.path.join(self.workspace, 'src')) + with self.assertRaises(PatchApplyError): + asyncio.run(self.scm([{'file': 'change.patch'}, {'file': 'bad.patch'}]).invoke( + Invoker(self.workspace), False)) + self.assertEqual(overlayFingerprint(os.path.join(self.workspace, 'src')), before) + + def test_commit_failure_restores_existing_and_created_files(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n' + '--- /dev/null\n+++ b/added\n@@ -0,0 +1 @@\n+added\n' + '--- /dev/null\n+++ b/fail\n@@ -0,0 +1 @@\n+fail\n') + os.chmod(os.path.join(self.workspace, 'src', 'message'), 0o755) + before = overlayFingerprint(os.path.join(self.workspace, 'src')) + write = _PatchTransaction._write + def fail_last(path, data, mode): + if path.endswith(os.sep + 'fail'): + raise OSError('simulated write failure') + write(path, data, mode) + with patch.object(_PatchTransaction, '_write', staticmethod(fail_last)): + with self.assertRaises(PatchApplyError) as error: + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + self.assertEqual(error.exception.baseFingerprint, before) + self.assertEqual(overlayFingerprint(os.path.join(self.workspace, 'src')), before) + self.assertEqual(os.stat(os.path.join(self.workspace, 'src', 'message')).st_mode & 0o777, + 0o755) + self.assertFalse(os.path.lexists(os.path.join(self.workspace, 'src', 'added'))) + self.assertFalse(any(name.startswith('.bob-patch-') + for name in os.listdir(os.path.join(self.workspace, 'src')))) + + def test_failed_rollback_does_not_claim_verified_base(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n' + '--- /dev/null\n+++ b/fail\n@@ -0,0 +1 @@\n+fail\n') + write = _PatchTransaction._write + def fail_write_and_restore(path, data, mode): + if path.endswith(os.sep + 'fail') or data == b'old\n': + raise OSError('simulated failure') + write(path, data, mode) + with patch.object(_PatchTransaction, '_write', staticmethod(fail_write_and_restore)): + with self.assertRaises(PatchApplyError) as error: + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + self.assertIsNone(error.exception.baseFingerprint) + + def test_creation_and_reverse_creation_refuse_existing_target(self): + for body, reverse in ( + ('--- /dev/null\n+++ b/message\n@@ -0,0 +1 @@\n+new\n', False), + ('--- a/message\n+++ /dev/null\n@@ -1 +0,0 @@\n-old\n', True)): + self.writePatch(body) + scm = self.scm([{'file': 'change.patch'}]) + with self.assertRaisesRegex(ParseError, 'already exists'): + if reverse: + asyncio.run(scm._PatchScm__unapply(Invoker(self.workspace))) + else: + asyncio.run(scm.invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'old\n') + + @skipUnless(canSymlink(), 'symlinks unavailable') + def test_protected_target_normalization_and_symlink_aliases(self): + for target in ('./message', 'alias', 'nested/message'): + if target == 'alias': + os.symlink('message', os.path.join(self.workspace, 'src', 'alias')) + elif target.startswith('nested/'): + os.symlink('.', os.path.join(self.workspace, 'src', 'nested')) + self.writePatch('--- a/{0}\n+++ b/{0}\n@@ -1 +1 @@\n-old\n+new\n'.format(target)) + scm = PatchScm(DummyScm(self.recipe, protected=('message',)), + [{'file': 'change.patch'}]) + with self.assertRaisesRegex(ParseError, 'protected SCM path'): + asyncio.run(scm.invoke(Invoker(self.workspace), False)) + + @skipUnless(canSymlink(), 'symlinks unavailable') + def test_creation_refuses_dangling_symlink(self): + os.symlink('absent', os.path.join(self.workspace, 'src', 'link')) + self.writePatch('--- /dev/null\n+++ b/link\n@@ -0,0 +1 @@\n+new\n') + with self.assertRaises(ParseError): + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + self.assertTrue(os.path.islink(os.path.join(self.workspace, 'src', 'link'))) + self.assertFalse(os.path.exists(os.path.join(self.workspace, 'src', 'absent'))) + + def test_nonzero_empty_hunk_coordinates_apply_and_reverse(self): + self.writePatch('--- a/message\n+++ b/message\n' + '@@ -1,0 +2 @@\n+inserted\n') + scm = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(scm.invoke(invoker, False)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'old\ninserted\n') + asyncio.run(scm._PatchScm__unapply(invoker)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'old\n') + + def test_removed_content_may_look_like_file_header(self): + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('-- text\n') + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n--- text\n+new\n') + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'new\n') + + def test_series_content_is_registered_with_cache(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n') + with open(os.path.join(self.temp.name, 'series'), 'w') as f: + f.write('change.patch\n') + registered = [] + def load(path): + registered.append(path) + with open(path, 'rb') as f: + return f.read() + PatchScm.trackPatchFiles(self.recipe, [{'series': 'series'}], load) + self.assertEqual(registered, [os.path.join(self.temp.name, 'series'), + os.path.join(self.temp.name, 'change.patch')]) + + def test_variant_comparison_keeps_recipe_identity_and_ignores_metadata(self): + from bob.builder import compareDirectoryState + original = {None: (b'old', None), 'src': ('same', {})} + changed = {None: (b'new', None), 'src': ('same', {})} + self.assertFalse(compareDirectoryState(original, changed)) + self.assertTrue(compareDirectoryState(original, {**original, 2: {}, 3: [], 4: {}})) + + def test_series_applies_in_order_and_rejects_options(self): + for name, before, after in (('first.patch', 'old', 'first'), + ('second.patch', 'first', 'second')): + with open(os.path.join(self.temp.name, name), 'w') as f: + f.write('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-{}\n+{}\n' + .format(before, after)) + with open(os.path.join(self.temp.name, 'series'), 'w') as f: + f.write('# vendor fixes\n\nfirst.patch\nsecond.patch\n') + asyncio.run(self.scm([{'series': 'series'}]).invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'second\n') + + with open(os.path.join(self.temp.name, 'series'), 'w') as f: + f.write('first.patch -p1\n') + with self.assertRaises(ParseError): + self.scm([{'series': 'series'}]) + + def test_files_glob_applies_sorted_matches_and_rejects_empty_patterns(self): + os.makedirs(os.path.join(self.temp.name, 'patches')) + for name, before, after in (('foo-20.patch', 'first', 'second'), + ('foo-10.patch', 'old', 'first')): + with open(os.path.join(self.temp.name, 'patches', name), 'w') as f: + f.write('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-{}\n+{}\n' + .format(before, after)) + + scm = self.scm([{'files': 'patches/foo*.patch'}]) + asyncio.run(scm.invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'second\n') + self.assertEqual([patch['file'] for patch in scm.getProperties(False)['patches']], + ['patches/foo-10.patch', 'patches/foo-20.patch']) + + for pattern in ('patches/missing*.patch', '../patches/*.patch', '**/*.patch'): + with self.assertRaises(ParseError): + self.scm([{'files': pattern}]) + + def test_rejects_negative_patch_strip_level(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + with self.assertRaises(ParseError): + self.scm([{'file': 'change.patch', 'strip': -1}]) + + def test_registers_patch_contents_with_recipe_cache(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + class RecipeCache: + def __init__(self): self.paths = [] + def loadBinary(self, path): + self.paths.append(path) + with open(path, 'rb') as f: + return f.read() + cache = RecipeCache() + PatchScm(DummyScm(self.recipe), [{'file': 'change.patch'}], recipeSet=cache) + self.assertEqual(cache.paths, [os.path.join(self.temp.name, 'change.patch')]) + cache.paths.clear() + PatchScm.trackPatchFiles(self.recipe, [{'file': 'change.patch'}], cache.loadBinary) + self.assertEqual(cache.paths, [os.path.join(self.temp.name, 'change.patch')]) + + def test_accepts_execution_recipeset_without_cache_api(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + PatchScm(DummyScm(self.recipe), [{'file': 'change.patch'}], recipeSet=object()) + + def test_modified_overlay_refuses_inline_reverse(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + old = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(old.invoke(invoker, False)) + overlay = {'fingerprint': hashDirectory(os.path.join(self.workspace, 'src')).hex()} + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('user edit\n') + + with self.assertRaises(ParseError): + asyncio.run(old.switch(invoker, self.scm(old.getProperties(False)['patches'], overlay))) + + def test_rejects_scm_metadata_and_protected_url_download(self): + invoker = Invoker(self.workspace) + for target in ('.git/config', 'archive.tar'): + self.writePatch('--- /dev/null\n+++ b/{}\n@@ -0,0 +1 @@\n+x\n'.format(target)) + scm = PatchScm(DummyScm(self.recipe, protected=('archive.tar',)), + [{'file': 'change.patch'}]) + with self.assertRaises(ParseError): + asyncio.run(scm.invoke(invoker, False)) + + self.writePatch('--- a/.git/config\n+++ /dev/null\n@@ -1 +0,0 @@\n-x\n') + scm = self.scm([{'file': 'change.patch'}]) + with self.assertRaises(ParseError): + asyncio.run(scm.invoke(invoker, False)) + + def test_rejects_the_actual_url_download_path(self): + self.writePatch('--- /dev/null\n+++ b/archive.tar\n@@ -0,0 +1 @@\n+x\n') + url = UrlScm({'scm': 'url', 'url': 'file:///tmp/archive.tar', + 'recipe': self.recipe, '__source': 'test', 'dir': 'src'}) + with self.assertRaises(ParseError): + asyncio.run(PatchScm(url, [{'file': 'change.patch'}], patchOnly=True).invoke( + Invoker(self.workspace), False)) + + def test_matching_overlay_fingerprint_is_not_modified(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + scm = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(scm.invoke(invoker, False)) + status = scm.status(self.workspace, { + 'fingerprint': hashDirectory(os.path.join(self.workspace, 'src')).hex()}) + self.assertEqual(str(status), 'P') + def test_applies_multiple_hunks_and_preserves_missing_final_newline(self): + with open(os.path.join(self.workspace, 'src', 'message'), 'wb') as f: + f.write(b'one\ntwo\nthree') + self.writePatch('--- a/message\n+++ b/message\n' + '@@ -1 +1 @@\n-one\n+ONE\n' + '@@ -3 +3 @@\n-three\n\\ No newline at end of file\n' + '+THREE\n\\ No newline at end of file\n') + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'message'), 'rb') as f: + self.assertEqual(f.read(), b'ONE\ntwo\nTHREE') + + def test_finds_a_unique_nearby_hunk_location_forward_and_reverse(self): + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('intro\nold\n') + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n') + scm = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(scm.invoke(invoker, False)) + asyncio.run(scm._PatchScm__unapply(invoker)) + + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'intro\nold\n') + self.assertEqual(len(invoker.infoMessages), 2) + self.assertTrue(all("'message' at offset +1" in message + for message in invoker.infoMessages)) + + def test_nearby_hunks_keep_prior_hunk_line_delta(self): + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('old\nfiller\nfiller2\nanchor\n') + self.writePatch('--- a/message\n+++ b/message\n' + '@@ -1 +1,2 @@\n-old\n+OLD\n+inserted\n' + '@@ -3 +4 @@\n-anchor\n+ANCHOR\n') + invoker = Invoker(self.workspace) + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke(invoker, False)) + + with open(os.path.join(self.workspace, 'src', 'message')) as f: + self.assertEqual(f.read(), 'OLD\ninserted\nfiller\nfiller2\nANCHOR\n') + self.assertEqual(invoker.infoMessages, + ["Applied patch hunk in 'message' at offset +1"]) + + def test_rejects_ambiguous_nearby_hunk_location(self): + with open(os.path.join(self.workspace, 'src', 'message'), 'w') as f: + f.write('old\nmiddle\nold\n') + self.writePatch('--- a/message\n+++ b/message\n@@ -2 +2 @@\n-old\n+new\n') + with self.assertRaisesRegex(ParseError, 'multiple locations'): + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + + def test_keeps_zero_source_line_insertions_at_declared_location(self): + self.writePatch('--- a/message\n+++ b/message\n' + '@@ -1 +0,0 @@\n-old\n' + '@@ -0,0 +1 @@\n+new\n') + with self.assertRaisesRegex(ParseError, 'declared location'): + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke( + Invoker(self.workspace), False)) + + def test_adds_and_removes_file_with_reverse(self): + self.writePatch('--- /dev/null\n+++ b/added\n@@ -0,0 +1,2 @@\n+one\n+two\n') + scm = self.scm([{'file': 'change.patch'}]) + invoker = Invoker(self.workspace) + asyncio.run(scm.invoke(invoker, False)) + with open(os.path.join(self.workspace, 'src', 'added')) as f: + self.assertEqual(f.read(), 'one\ntwo\n') + asyncio.run(scm._PatchScm__unapply(invoker)) + self.assertFalse(os.path.exists(os.path.join(self.workspace, 'src', 'added'))) + + def test_deletes_file_and_honors_strip_level(self): + with open(os.path.join(self.workspace, 'src', 'remove-me'), 'w') as f: + f.write('gone\n') + self.writePatch('--- remove-me\n+++ /dev/null\n@@ -1 +0,0 @@\n-gone\n') + asyncio.run(self.scm([{'file': 'change.patch', 'strip': 0}]).invoke( + Invoker(self.workspace), False)) + self.assertFalse(os.path.exists(os.path.join(self.workspace, 'src', 'remove-me'))) + + def test_accepts_descriptive_preamble_and_orig_backup_header(self): + with open(os.path.join(self.workspace, 'src', 'setupinfo.py'), 'w') as f: + f.write('old\n') + self.writePatch('This patch explains why the change is needed.\n\n' + '--- workspace/setupinfo.py.orig\t2025-03-12 22:36:10 +0100\n' + '+++ workspace/setupinfo.py\t2025-03-12 22:36:13 +0100\n' + '@@ -1 +1 @@\n-old\n+new\n') + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke(Invoker(self.workspace), False)) + with open(os.path.join(self.workspace, 'src', 'setupinfo.py')) as f: + self.assertEqual(f.read(), 'new\n') + + def test_rejects_shifted_malformed_and_unsupported_patches(self): + invoker = Invoker(self.workspace) + for body in ( + '--- a/message\n+++ b/message\n@@ -102 +102 @@\n-old\n+new\n', + '--- a/message\n+++ b/message\n@@ -1,2 +1 @@\n-old\n+new\n', + 'diff --git a/message b/message\n--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+new\n', + 'GIT binary patch\nliteral 1\n'): + self.writePatch(body) + with self.assertRaises(ParseError): + asyncio.run(self.scm([{'file': 'change.patch'}]).invoke(invoker, False)) + def test_jenkins_uses_serialized_pre_run_overlay(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + scm = self.scm([{'file': 'change.patch'}]) + self.assertFalse(scm.hasJenkinsPlugin()) + self.assertIn('__data', scm.getProperties(True)['patches'][0]) + + def test_jenkins_plugin_checkout_is_followed_by_patch_only_command(self): + self.writePatch('--- a/message\n+++ b/message\n@@ -1 +1 @@\n-old\n+first\n') + base = PluginScm(self.recipe) + scm = PatchScm(base, [{'file': 'change.patch'}], patchOnly=True) + self.assertTrue(scm.hasJenkinsPlugin()) + self.assertEqual(scm.asJenkins('', None), 'plugin') + self.assertTrue(scm.getJenkinsPreRunProperties()['__patchOnly']) + asyncio.run(scm.invoke(Invoker(self.workspace), False)) + self.assertFalse(base.invoked) diff --git a/test/unit/test_scm_patch_checkout.py b/test/unit/test_scm_patch_checkout.py new file mode 100644 index 00000000..06787508 --- /dev/null +++ b/test/unit/test_scm_patch_checkout.py @@ -0,0 +1,136 @@ +# Bob build tool +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Exercise overlay recovery through real checkout state and recipe caching.""" + +from unittest import TestCase +from pathlib import Path +import hashlib +import os +import subprocess +import sys +import tarfile +import tempfile + +import yaml + + +class TestPatchCheckout(TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.bobRoot = Path(__file__).resolve().parents[2] + self.env = {**os.environ, 'PYTHONPATH': str(self.bobRoot / 'pym')} + (self.root / 'recipes').mkdir() + self.repo = self.root / 'repo' + self.repo.mkdir() + self.git('init') + self.git('config', 'user.name', 'Test') + self.git('config', 'user.email', 'test@example.invalid') + for name in ('message', 'second'): + (self.repo / name).write_text('old\n') + self.git('add', '.') + self.git('commit', '-m', 'base') + self.commit = self.git('rev-parse', 'HEAD').strip() + self.archive = self.root / 'source.tar' + with tarfile.open(self.archive, 'w') as archive: + for name in ('message', 'second'): + archive.add(self.repo / name, arcname=name) + + def tearDown(self): + self.temp.cleanup() + + def git(self, *args): + return subprocess.check_output(['git', *args], cwd=self.repo, + stderr=subprocess.PIPE, text=True) + + def patch(self, name, before, after, target='message'): + (self.root / 'recipes' / name).write_text( + '--- a/{0}\n+++ b/{0}\n@@ -1 +1 @@\n-{1}\n+{2}\n'.format( + target, before, after)) + + def recipe(self, kind, patches, script='true'): + if kind == 'git': + scm = {'scm': 'git', 'url': self.repo.as_uri(), 'commit': self.commit} + else: + scm = {'scm': 'url', 'url': self.archive.as_uri(), + 'digestSHA256': hashlib.sha256(self.archive.read_bytes()).hexdigest(), + 'extract': 'tar'} + scm.update(dir='src', patches=patches) + (self.root / 'recipes' / 'root.yaml').write_text(yaml.safe_dump({ + 'root': True, 'checkoutSCM': scm, 'checkoutScript': script, + 'checkoutDeterministic': True, 'buildScript': 'true', 'packageScript': 'true', + })) + + def bob(self, success=True): + result = subprocess.run([sys.executable, str(self.bobRoot / 'bob'), + '--debug=ngd', 'dev', '--checkout-only', 'root'], + cwd=self.root, env=self.env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if success: + self.assertEqual(result.returncode, 0, result.stdout) + else: + self.assertNotEqual(result.returncode, 0, result.stdout) + return result.stdout + + def workspace(self): + return next((self.root / 'dev' / 'src' / 'root').glob('*/workspace')) + + def test_git_failed_stack_retries_corrected_patch_on_retained_base(self): + self.check_failed_stack_retry('git') + + def test_url_failed_stack_retries_corrected_patch_on_retained_base(self): + self.check_failed_stack_retry('url') + + def check_failed_stack_retry(self, kind): + self.patch('first.patch', 'old', 'first') + self.patch('second.patch', 'wrong', 'second', 'second') + self.recipe(kind, [{'file': 'first.patch'}, {'file': 'second.patch'}]) + self.bob(False) + workspace = self.workspace() + self.assertEqual((workspace / 'src' / 'message').read_text(), 'old\n') + self.patch('second.patch', 'old', 'second', 'second') + self.bob() + self.assertEqual((workspace / 'src' / 'message').read_text(), 'first\n') + self.assertEqual((workspace / 'src' / 'second').read_text(), 'second\n') + self.assertFalse((workspace.parent / 'attic').exists()) + self.bob() + + def test_unverified_failed_checkout_moves_to_attic(self): + self.patch('fix.patch', 'wrong', 'patched') + self.recipe('git', [{'file': 'fix.patch'}]) + self.bob(False) + workspace = self.workspace() + (workspace / 'src' / 'message').write_text('partial legacy overlay\n') + relative = str(workspace.relative_to(self.root)) + subprocess.run([sys.executable, '-c', + 'from bob.state import BobState, finalize; ' + 'import sys; s=BobState(); d=s.getDirectoryState(sys.argv[1], True); ' + 'd.pop(4, None); s.setDirectoryState(sys.argv[1], d); finalize()', + relative], cwd=self.root, env=self.env, check=True) + self.patch('fix.patch', 'old', 'patched') + self.bob() + self.assertEqual((workspace / 'src' / 'message').read_text(), 'patched\n') + self.assertTrue((workspace.parent / 'attic').exists()) + saved = next((workspace.parent / 'attic').glob('*/message')) + self.assertEqual(saved.read_text(), 'partial legacy overlay\n') + + def test_series_reordering_invalidates_recipe_cache(self): + self.patch('first.patch', 'old', 'first') + self.patch('second.patch', 'first', 'second') + series = self.root / 'recipes' / 'series' + series.write_text('first.patch\nsecond.patch\n') + self.recipe('git', [{'series': 'series'}]) + self.bob() + series.write_text('second.patch\nfirst.patch\n') + output = self.bob(False) + self.assertIn('second.patch', output) + + def test_checkout_script_change_with_same_scm_reruns_checkout(self): + self.recipe('git', [], 'echo first > stamp') + self.bob() + workspace = self.workspace() + self.assertEqual((workspace / 'stamp').read_text(), 'first\n') + self.recipe('git', [], 'echo second > stamp') + self.bob() + self.assertEqual((workspace / 'stamp').read_text(), 'second\n')