diff --git a/doc/tutorial/compile.rst b/doc/tutorial/compile.rst index c101286c..f9e3fa77 100644 --- a/doc/tutorial/compile.rst +++ b/doc/tutorial/compile.rst @@ -69,7 +69,11 @@ directories: * ``{checkout,build,package}.sh``: A wrapper script that executes this specific step. Running the script will execute this particular step again. If you call the script with ``shell`` as first argument a new shell is spawned with - exactly the same environment as the step script would find. + exactly the same environment as the step script would find. Alternatively you + can run a single command in that same environment without entering an + interactive shell by calling the script with ``exec`` followed by the + command, e.g. ``./build.sh exec "make -j4"``. Passing several commands runs + them one after another and stops at the first one that fails. * ``script``: The actual script that was computed from the recipe and the inherited classes. This is not directly executable because it expects the right environment and arguments. @@ -162,6 +166,12 @@ Now make and save your changes. Then rebuild the kernel:: $ make -j $(nproc) bzImage $ exit +.. tip:: + For plain commands like the ``make`` call above you do not need to open an + interactive shell at all. The same could have been achieved with:: + + $ ./dev/build/linux-image/1/build.sh exec -E "make -j $(nproc) bzImage" + If you know how grab the kernel image directly out of the build tree and test it. Alternatively you can rebuild the top-level package. :: diff --git a/pym/bob/cmds/invoke.py b/pym/bob/cmds/invoke.py index 979c1ddc..7a0c8e58 100644 --- a/pym/bob/cmds/invoke.py +++ b/pym/bob/cmds/invoke.py @@ -15,8 +15,14 @@ def doInvoke(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob _invoke", description="Invoke a single step.") parser.add_argument('spec', help="The step spec file") - parser.add_argument('mode', default='run', choices=['run', 'update', 'shell', 'fingerprint'], + parser.add_argument('mode', default='run', + choices=['run', 'update', 'shell', 'fingerprint', 'exec'], nargs='?', help="Invocation mode") + parser.add_argument('command', nargs='*', + help="Command to execute in 'exec' mode. Each argument is one " + "command including its parameters. If more than one command is " + "given they are executed one after another. Execution stops at " + "the first command that fails.") group = parser.add_mutually_exclusive_group() group.add_argument('--clean', '-c', action='store_true', default=False, @@ -38,6 +44,12 @@ def doInvoke(argv, bobRoot): args = parser.parse_args(argv) verbosity = min(3, max(0, 1 + args.verbose - args.quiet)) # [0..4], default: 1 + if args.mode == 'exec': + if not args.command: + parser.error("exec mode requires at least one command") + elif args.command: + parser.error("unrecognized arguments: " + " ".join(args.command)) + try: with open(args.spec) as f: spec = StepSpec.fromFile(f) @@ -54,6 +66,11 @@ def doInvoke(argv, bobRoot): False, executor=executor) ret = loop.run_until_complete(invoker.executeStep(InvocationMode.SHELL, False, args.clean, args.keep_sandbox)) + elif args.mode == 'exec': + invoker = Invoker(spec, args.preserve_env, True, True, True, + verbosity >= 3, False, executor=executor) + ret = loop.run_until_complete(invoker.executeStep(InvocationMode.EXEC, + False, args.clean, args.keep_sandbox, commands=args.command)) elif args.mode in ('run', 'update'): invoker = Invoker(spec, args.preserve_env, args.no_logfiles, verbosity >= 2, verbosity >= 1, verbosity >= 3, False, diff --git a/pym/bob/invoker.py b/pym/bob/invoker.py index b53895bf..f03955e9 100644 --- a/pym/bob/invoker.py +++ b/pym/bob/invoker.py @@ -203,6 +203,7 @@ class InvocationMode(Enum): CALL = 'call' SHELL = 'shell' UPDATE = 'update' + EXEC = 'exec' class Invoker: def __init__(self, spec, preserveEnv, noLogFiles, showStdOut, showStdErr, @@ -385,13 +386,13 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None, return FinishedProcess(ret, stdoutBuf, stderrBuf) - def __getFatSandboxCmds(self, tmpDir): + def __getFatSandboxCmds(self, sandboxDir): if sys.platform != "linux": self.fail("Sandbox builds are only supported on Linux!") cmdArgs = [ getSandboxHelperPath() ] #FIXME: if verbosity >= 4: cmdArgs.append('-D') - cmdArgs.extend(["-S", tmpDir]) + cmdArgs.extend(["-S", sandboxDir]) cmdArgs.extend(["-H", "bob"]) sandboxRootFs = os.path.abspath(self.__spec.sandboxRootWorkspace) for f in os.listdir(sandboxRootFs): @@ -420,13 +421,13 @@ def __getFatSandboxCmds(self, tmpDir): return cmdArgs - def __getSlimSandboxCmds(self, tmpDir): + def __getSlimSandboxCmds(self, sandboxDir, includeDir): if sys.platform != "linux": self.fail("Sandbox builds are only supported on Linux!") - sandboxTmpDir = os.path.join(tmpDir, "sandbox") + sandboxTmpDir = os.path.join(sandboxDir, "sandbox") os.mkdir(sandboxTmpDir) - whiteoutTmpDir = os.path.join(tmpDir, "whiteout") + whiteoutTmpDir = os.path.join(sandboxDir, "whiteout") os.mkdir(whiteoutTmpDir) cmdArgs = [ getSandboxHelperPath() ] @@ -434,8 +435,8 @@ def __getSlimSandboxCmds(self, tmpDir): cmdArgs.append("-i") # Mount everything read-only except tmp. The caller has already created - # a "tmp" directory in tmpDir. Mount it writable at the right path. - cmdArgs.extend(["-M", os.path.join(tmpDir, "tmp"), "-w", "/tmp"]) + # a "tmp" directory in includeDir. Mount it writable at the right path. + cmdArgs.extend(["-M", os.path.join(includeDir, "tmp"), "-w", "/tmp"]) for f in os.listdir("/"): if f == "tmp": continue cmdArgs.extend(["-M", "/"+f, "-m", "/"+f]) @@ -481,7 +482,72 @@ def ensureSandboxUsable(): except OSError as e: raise BuildError("Error executing sandbox helper: " + str(e)) - async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=False): + def __wrapSandbox(self, realScriptFile, execScriptFile, callArgs, tmpDir): + # Wrap call into sandbox if requested + if self.__spec.hasSandbox: + # Every invocation gets its own, freshly created sandbox root. + # This is required because a single Invoker may run more than one + # sandboxed command (e.g. in 'exec' mode) and the sandbox helper + # does not tolerate a pre-populated root. + sandboxDir = tempfile.mkdtemp(dir=tmpDir) + if self.__spec.fatSandbox: + # The fat sandbox uses the sandbox root as its new root file + # system, i.e. "tmp" must exist as a real (writable) + # directory therein. Re-create the include files there by + # hard linking them instead of relying on a bind mount so + # that the jobserver FIFO can still be mounted underneath it. + includeTmpDir = os.path.join(tmpDir, "tmp") + sandboxTmpDir = os.path.join(sandboxDir, "tmp") + os.mkdir(sandboxTmpDir) + for name in os.listdir(includeTmpDir): + os.link(os.path.join(includeTmpDir, name), + os.path.join(sandboxTmpDir, name)) + cmdArgs = self.__getFatSandboxCmds(sandboxDir) + else: + assert self.__spec.slimSandbox + cmdArgs = self.__getSlimSandboxCmds(sandboxDir, tmpDir) + + cmdArgs.extend(["-M", os.path.abspath(realScriptFile), "-m" + , execScriptFile]) + + # Prevent network access + if not self.__spec.netAccess: cmdArgs.append('-n') + + # Create empty env file. Otheriwse bind mount fails. + if self.__spec.envFile: + with open(self.__spec.envFile, "wb"): pass + cmdArgs.extend(["-M", os.path.abspath(self.__spec.envFile), "-w", "/bob/env"]) + + # Mount workspace writable and all dependencies read-only + cmdArgs.extend(["-M", os.path.abspath(self.__spec.workspaceWorkspacePath), + "-w", os.path.abspath(self.__spec.workspaceExecPath)]) + cmdArgs.extend(["-W", os.path.abspath(self.__spec.workspaceExecPath) ]) + for argWorkspacePath, argExecPath in self.__spec.depMounts: + cmdArgs.extend(["-M", os.path.abspath(argWorkspacePath), + "-m", os.path.abspath(argExecPath)]) + + # Mount jobserver FIFO into sandbox + if self.__makeJobServer.isFifo(): + fifo = os.path.abspath(self.__makeJobServer.fifoPath()) + cmdArgs.extend(["-M", fifo, + "-w", "/tmp/bob-jobserver" if self.__spec.fatSandbox else fifo]) + + # Command follows. Stop parsing options. + cmdArgs.append("--") + + # Hard override of PATH. The sandbox helper is found by an absolute path! + if self.__spec.fatSandbox: + env = { "PATH" : ":".join(self.__spec.sandboxPaths) } + else: + env = None + else: + cmdArgs = [] + env = None + cmdArgs.extend(callArgs) + return cmdArgs, env + + async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=False, + commands=None): # make permissions predictable os.umask(0o022) @@ -510,95 +576,61 @@ async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=Fal if not os.path.isdir(self.__spec.workspaceWorkspacePath): os.makedirs(self.__spec.workspaceWorkspacePath, exist_ok=True) workspaceCreated = True - elif clean and mode != InvocationMode.SHELL: + elif clean and mode not in (InvocationMode.SHELL, InvocationMode.EXEC): emptyDirectory(self.__spec.workspaceWorkspacePath) workspaceCreated = True - # setup script and arguments - if mode == InvocationMode.SHELL: - realScriptFile, execScriptFile, callArgs = self.__spec.language.setupShell( - self.__spec, tmpDir, self.__preserveEnv) - elif mode == InvocationMode.CALL: - realScriptFile, execScriptFile, callArgs = self.__spec.language.setupCall( - self.__spec, tmpDir, self.__preserveEnv, self.__trace) - elif mode == InvocationMode.UPDATE: - realScriptFile, execScriptFile, callArgs = self.__spec.language.setupUpdate( - self.__spec, tmpDir, self.__preserveEnv, self.__trace) + if mode == InvocationMode.EXEC: + # Every command is set up, sandboxed and executed on its own. + # Execution stops at the first command that fails. + for command in commands: + realScriptFile, execScriptFile, callArgs = self.__spec.language.setupExec( + self.__spec, tmpDir, self.__preserveEnv, self.__trace, command) + cmdArgs, env = self.__wrapSandbox(realScriptFile, execScriptFile, + callArgs, tmpDir) + await self.checkCommand(cmdArgs, env=env, specEnv=False) else: - assert False, "not reached" - - # Wrap call into sandbox if requested - if self.__spec.hasSandbox: - if self.__spec.fatSandbox: - cmdArgs = self.__getFatSandboxCmds(tmpDir) + # setup script and arguments + if mode == InvocationMode.SHELL: + realScriptFile, execScriptFile, callArgs = self.__spec.language.setupShell( + self.__spec, tmpDir, self.__preserveEnv) + elif mode == InvocationMode.CALL: + realScriptFile, execScriptFile, callArgs = self.__spec.language.setupCall( + self.__spec, tmpDir, self.__preserveEnv, self.__trace) + elif mode == InvocationMode.UPDATE: + realScriptFile, execScriptFile, callArgs = self.__spec.language.setupUpdate( + self.__spec, tmpDir, self.__preserveEnv, self.__trace) else: - assert self.__spec.slimSandbox - cmdArgs = self.__getSlimSandboxCmds(tmpDir) - - cmdArgs.extend(["-M", os.path.abspath(realScriptFile), "-m" - , execScriptFile]) - - # Prevent network access - if not self.__spec.netAccess: cmdArgs.append('-n') - - # Create empty env file. Otheriwse bind mount fails. - if self.__spec.envFile: - with open(self.__spec.envFile, "wb"): pass - cmdArgs.extend(["-M", os.path.abspath(self.__spec.envFile), "-w", "/bob/env"]) - - # Mount workspace writable and all dependencies read-only - cmdArgs.extend(["-M", os.path.abspath(self.__spec.workspaceWorkspacePath), - "-w", os.path.abspath(self.__spec.workspaceExecPath)]) - cmdArgs.extend(["-W", os.path.abspath(self.__spec.workspaceExecPath) ]) - for argWorkspacePath, argExecPath in self.__spec.depMounts: - cmdArgs.extend(["-M", os.path.abspath(argWorkspacePath), - "-m", os.path.abspath(argExecPath)]) - - # Mount jobserver FIFO into sandbox - if self.__makeJobServer.isFifo(): - fifo = os.path.abspath(self.__makeJobServer.fifoPath()) - cmdArgs.extend(["-M", fifo, - "-w", "/tmp/bob-jobserver" if self.__spec.fatSandbox else fifo]) - - # Command follows. Stop parsing options. - cmdArgs.append("--") - - # Hard override of PATH. The sandbox helper is found by an absolute path! - if self.__spec.fatSandbox: - env = { "PATH" : ":".join(self.__spec.sandboxPaths) } + raise AssertionError("not reached") + + cmdArgs, env = self.__wrapSandbox(realScriptFile, execScriptFile, callArgs, tmpDir) + + if mode == InvocationMode.SHELL: + ret = await self.callCommand(cmdArgs, env=env, specEnv=False) + elif mode in (InvocationMode.CALL, InvocationMode.UPDATE): + for scm in self.__spec.preRunCmds: + scm = getScm(scm) + if mode == InvocationMode.UPDATE and not scm.isLocal(): + continue # Skip non-local SCMs on update-only + try: + await scm.invoke(self, workspaceCreated) + except CmdFailedError as e: + self.error(scm.getSource(), "failed") + self.error(e.what) + raise + except Exception: + self.error(scm.getSource(), "failed") + raise + await self.checkCommand(cmdArgs, env=env, specEnv=False) + for a in self.__spec.postRunCmds: + a = CheckoutAssert(a) + try: + await a.invoke(self) + except Exception: + self.error(a.getSource(), "failed") + raise else: - env = None - else: - cmdArgs = [] - env = None - cmdArgs.extend(callArgs) - - if mode == InvocationMode.SHELL: - ret = await self.callCommand(cmdArgs, env=env, specEnv=False) - elif mode in (InvocationMode.CALL, InvocationMode.UPDATE): - for scm in self.__spec.preRunCmds: - scm = getScm(scm) - if mode == InvocationMode.UPDATE and not scm.isLocal(): - continue # Skip non-local SCMs on update-only - try: - await scm.invoke(self, workspaceCreated) - except CmdFailedError as e: - self.error(scm.getSource(), "failed") - self.error(e.what) - raise - except Exception: - self.error(scm.getSource(), "failed") - raise - await self.checkCommand(cmdArgs, env=env, specEnv=False) - for a in self.__spec.postRunCmds: - a = CheckoutAssert(a) - try: - await a.invoke(self) - except Exception: - self.error(a.getSource(), "failed") - raise - else: - assert False, "not reached" + raise AssertionError("not reached") # everything went well ret = 0 diff --git a/pym/bob/languages.py b/pym/bob/languages.py index c866dd7d..a93add2f 100644 --- a/pym/bob/languages.py +++ b/pym/bob/languages.py @@ -400,6 +400,10 @@ def setupCall(spec, tmpDir, keepEnv, trace): def setupUpdate(spec, tmpDir, keepEnv, trace): return BashLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace) + @staticmethod + def setupExec(spec, tmpDir, keepEnv, trace, command): + return BashLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace) + @staticmethod def mangleFingerprints(scriptFragments, env): # join the script fragments first @@ -616,6 +620,10 @@ def setupCall(spec, tmpDir, keepEnv, trace): def setupUpdate(spec, tmpDir, keepEnv, trace): return PwshLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace) + @staticmethod + def setupExec(spec, tmpDir, keepEnv, trace, command): + return PwshLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace) + @staticmethod def mangleFingerprints(scriptFragments, env): # join the script fragments first @@ -773,6 +781,10 @@ def setupCall(spec, tmpDir, keepEnv, trace): def setupUpdate(spec, tmpDir, keepEnv, trace): return PythonLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace) + @staticmethod + def setupExec(spec, tmpDir, keepEnv, trace, command): + return PythonLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace) + @staticmethod def mangleFingerprints(scriptFragments, env): # join the script fragments first diff --git a/test/black-box/exec/config.yaml b/test/black-box/exec/config.yaml new file mode 100644 index 00000000..323d0919 --- /dev/null +++ b/test/black-box/exec/config.yaml @@ -0,0 +1,2 @@ +bobMinimumVersion: "1.2" +windowsPowerShellExecutable: "pwsh" diff --git a/test/black-box/exec/recipes/bash.yaml b/test/black-box/exec/recipes/bash.yaml new file mode 100644 index 00000000..dd795ec9 --- /dev/null +++ b/test/black-box/exec/recipes/bash.yaml @@ -0,0 +1,10 @@ +root: True + +depends: + - name: sandbox + use: [sandbox] + +buildScript: | + echo "build ok" +packageScript: | + echo "package ok" diff --git a/test/black-box/exec/recipes/pwsh.yaml b/test/black-box/exec/recipes/pwsh.yaml new file mode 100644 index 00000000..fa55a5be --- /dev/null +++ b/test/black-box/exec/recipes/pwsh.yaml @@ -0,0 +1,12 @@ +root: True + +depends: + - name: sandbox + use: [sandbox] + +scriptLanguage: PowerShell + +buildScript: | + Write-Host "build ok" +packageScript: | + Write-Host "package ok" diff --git a/test/black-box/exec/recipes/python.yaml b/test/black-box/exec/recipes/python.yaml new file mode 100644 index 00000000..7b80c6f9 --- /dev/null +++ b/test/black-box/exec/recipes/python.yaml @@ -0,0 +1,12 @@ +root: True + +depends: + - name: sandbox + use: [sandbox] + +scriptLanguage: python + +buildScript: | + print("build ok") +packageScript: | + print("package ok") diff --git a/test/black-box/exec/recipes/sandbox.yaml b/test/black-box/exec/recipes/sandbox.yaml new file mode 100644 index 00000000..5f660773 --- /dev/null +++ b/test/black-box/exec/recipes/sandbox.yaml @@ -0,0 +1,16 @@ +# Full host mount so that host interpreters (python3, pwsh, ...) are +# available inside the sandbox image. +provideSandbox: + paths: ["/usr/local/bin", "/usr/local/sbin", "/usr/bin", "/usr/sbin", + "/bin", "/sbin"] + mount: + - /bin + - /etc + - /lib + - /opt + - /run + - /usr + - /var + + - ["/lib32", "/lib32", [nofail]] + - ["/lib64", "/lib64", [nofail]] diff --git a/test/black-box/exec/run.sh b/test/black-box/exec/run.sh new file mode 100755 index 00000000..a6fe157d --- /dev/null +++ b/test/black-box/exec/run.sh @@ -0,0 +1,118 @@ +#!/bin/bash -e +source "$(dirname "$0")/../../test-lib.sh" "../../.." + +# Exercise the "exec" invocation mode, i.e. "build.sh exec ...". It +# must work for every supported scripting language (bash, python, PowerShell) +# and regardless of the sandboxing mode (none, slim or full/fat sandbox with +# an image). + +HAVE_SANDBOX=0 +"${BOB_ROOT}/bin/bob-namespace-sandbox" -C && HAVE_SANDBOX=1 || true + +HAVE_PWSH=0 +{ type -p pwsh >/dev/null 2>&1 || type -p powershell >/dev/null 2>&1 ; } && HAVE_PWSH=1 || true + +if is_win32 ; then + SCRIPT=build.cmd +else + SCRIPT=build.sh +fi + +# Run the "exec" checks against an already built package. Verifies that a +# single command works, that several commands run one after another and that +# execution stops at the first command that fails. The commands themselves +# are language specific and passed in via $CMD_ONE, $CMD_TWO, $CMD_FAIL and +# $CMD_THREE. +run_exec_checks() +{ + local buildsh="dev/build/$1/1/$SCRIPT" + local ws="dev/build/$1/1/workspace" + + rm -f "$ws"/*.txt + + # A single command is executed. + "$buildsh" exec "$CMD_ONE" + expect_exist "$ws/one.txt" + + # Several commands are executed one after another. + rm -f "$ws"/*.txt + "$buildsh" exec "$CMD_ONE" "$CMD_TWO" + expect_exist "$ws/one.txt" "$ws/two.txt" + + # Execution stops at the first command that fails. + rm -f "$ws"/*.txt + expect_fail "$buildsh" exec "$CMD_ONE" "$CMD_FAIL" "$CMD_THREE" + expect_exist "$ws/one.txt" + expect_not_exist "$ws/three.txt" +} + +# +# bash - the default script language. No interpreter dependency at all. +# +CMD_ONE='echo -n one > one.txt' +CMD_TWO='echo -n two > two.txt' +CMD_FAIL='exit 1' +CMD_THREE='echo -n three > three.txt' + +cleanup +run_bob dev bash +run_exec_checks bash + +if [[ $HAVE_SANDBOX == 1 ]] ; then + cleanup + run_bob dev bash --slim-sandbox + run_exec_checks bash + + cleanup + run_bob dev bash --strict-sandbox + run_exec_checks bash +fi + +# +# python - a "python3" binary must be available in $PATH. Bob would use its +# own, bundled interpreter outside of a sandbox, but we require "python3" +# unconditionally here to keep this test's requirements simple and uniform. +# +CMD_ONE="open('one.txt', 'w').write('one')" +CMD_TWO="open('two.txt', 'w').write('two')" +CMD_FAIL='exit(1)' +CMD_THREE="open('three.txt', 'w').write('three')" + +cleanup +run_bob dev python +run_exec_checks python + +if [[ $HAVE_SANDBOX == 1 ]] ; then + cleanup + run_bob dev python --slim-sandbox + run_exec_checks python + + cleanup + run_bob dev python --strict-sandbox + run_exec_checks python +fi + +# +# PowerShell - always needs a real "pwsh"/"powershell" interpreter, with or +# without a sandbox. +# +if [[ $HAVE_PWSH == 1 ]] ; then + CMD_ONE='Set-Content -NoNewline -Path one.txt -Value "one"' + CMD_TWO='Set-Content -NoNewline -Path two.txt -Value "two"' + CMD_FAIL='exit 1' + CMD_THREE='Set-Content -NoNewline -Path three.txt -Value "three"' + + cleanup + run_bob dev pwsh + run_exec_checks pwsh + + if [[ $HAVE_SANDBOX == 1 ]] ; then + cleanup + run_bob dev pwsh --slim-sandbox + run_exec_checks pwsh + + cleanup + run_bob dev pwsh --strict-sandbox + run_exec_checks pwsh + fi +fi