Exec mode for build.sh

doc/tutorial/compile.rst

@@ -69,7 +69,11 @@ directories:
 * ``{checkout,build,package}.sh``: A wrapper script that executes this specific
   step. Running the script will execute this particular step again. If you call
   the script with ``shell`` as first argument a new shell is spawned with
-  exactly the same environment as the step script would find.
+  exactly the same environment as the step script would find. Alternatively you
+  can run a single command in that same environment without entering an
+  interactive shell by calling the script with ``exec`` followed by the
+  command, e.g. ``./build.sh exec "make -j4"``. Passing several commands runs
+  them one after another and stops at the first one that fails.
 * ``script``: The actual script that was computed from the recipe and the
   inherited classes. This is not directly executable because it expects the
   right environment and arguments.
@@ -162,6 +166,12 @@ Now make and save your changes. Then rebuild the kernel::
     $ make -j $(nproc) bzImage
     $ exit
 
+.. tip::
+   For plain commands like the ``make`` call above you do not need to open an
+   interactive shell at all. The same could have been achieved with::
+
+       $ ./dev/build/linux-image/1/build.sh exec -E "make -j $(nproc) bzImage"
+
 If you know how grab the kernel image directly out of the build tree and test
 it. Alternatively you can rebuild the top-level package. ::
 

pym/bob/cmds/invoke.py

@@ -15,8 +15,14 @@ def doInvoke(argv, bobRoot):
     parser = argparse.ArgumentParser(prog="bob _invoke",
         description="Invoke a single step.")
     parser.add_argument('spec', help="The step spec file")
-    parser.add_argument('mode', default='run', choices=['run', 'update', 'shell', 'fingerprint'],
+    parser.add_argument('mode', default='run',
+        choices=['run', 'update', 'shell', 'fingerprint', 'exec'],
         nargs='?', help="Invocation mode")
+    parser.add_argument('command', nargs='*',
+        help="Command to execute in 'exec' mode. Each argument is one "
+             "command including its parameters. If more than one command is "
+             "given they are executed one after another. Execution stops at "
+             "the first command that fails.")
 
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--clean', '-c', action='store_true', default=False,
@@ -38,6 +44,12 @@ def doInvoke(argv, bobRoot):
     args = parser.parse_args(argv)
     verbosity = min(3, max(0, 1 + args.verbose - args.quiet)) # [0..4], default: 1
 
+    if args.mode == 'exec':
+        if not args.command:
+            parser.error("exec mode requires at least one command")
+    elif args.command:
+        parser.error("unrecognized arguments: " + " ".join(args.command))
+
     try:
         with open(args.spec) as f:
             spec = StepSpec.fromFile(f)
@@ -54,6 +66,11 @@ def doInvoke(argv, bobRoot):
                               False, executor=executor)
             ret = loop.run_until_complete(invoker.executeStep(InvocationMode.SHELL,
                 False, args.clean, args.keep_sandbox))
+        elif args.mode == 'exec':
+            invoker = Invoker(spec, args.preserve_env, True, True, True,
+                              verbosity >= 3, False, executor=executor)
+            ret = loop.run_until_complete(invoker.executeStep(InvocationMode.EXEC,
+                False, args.clean, args.keep_sandbox, commands=args.command))
         elif args.mode in ('run', 'update'):
             invoker = Invoker(spec, args.preserve_env, args.no_logfiles,
                 verbosity >= 2, verbosity >= 1, verbosity >= 3, False,

pym/bob/invoker.py

@@ -203,6 +203,7 @@ class InvocationMode(Enum):
     CALL = 'call'
     SHELL = 'shell'
     UPDATE = 'update'
+    EXEC = 'exec'
 
 class Invoker:
     def __init__(self, spec, preserveEnv, noLogFiles, showStdOut, showStdErr,
@@ -385,13 +386,13 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None,
 
         return FinishedProcess(ret, stdoutBuf, stderrBuf)
 
-    def __getFatSandboxCmds(self, tmpDir):
+    def __getFatSandboxCmds(self, sandboxDir):
         if sys.platform != "linux":
             self.fail("Sandbox builds are only supported on Linux!")
 
         cmdArgs = [ getSandboxHelperPath() ]
         #FIXME: if verbosity >= 4: cmdArgs.append('-D')
-        cmdArgs.extend(["-S", tmpDir])
+        cmdArgs.extend(["-S", sandboxDir])
         cmdArgs.extend(["-H", "bob"])
         sandboxRootFs = os.path.abspath(self.__spec.sandboxRootWorkspace)
         for f in os.listdir(sandboxRootFs):
@@ -420,22 +421,22 @@ def __getFatSandboxCmds(self, tmpDir):
 
         return cmdArgs
 
-    def __getSlimSandboxCmds(self, tmpDir):
+    def __getSlimSandboxCmds(self, sandboxDir, includeDir):
         if sys.platform != "linux":
             self.fail("Sandbox builds are only supported on Linux!")
 
-        sandboxTmpDir = os.path.join(tmpDir, "sandbox")
+        sandboxTmpDir = os.path.join(sandboxDir, "sandbox")
         os.mkdir(sandboxTmpDir)
-        whiteoutTmpDir = os.path.join(tmpDir, "whiteout")
+        whiteoutTmpDir = os.path.join(sandboxDir, "whiteout")
         os.mkdir(whiteoutTmpDir)
 
         cmdArgs = [ getSandboxHelperPath() ]
         cmdArgs.extend(["-S", sandboxTmpDir])
         cmdArgs.append("-i")
 
         # Mount everything read-only except tmp. The caller has already created
-        # a "tmp" directory in tmpDir.  Mount it writable at the right path.
-        cmdArgs.extend(["-M", os.path.join(tmpDir, "tmp"), "-w", "/tmp"])
+        # a "tmp" directory in includeDir. Mount it writable at the right path.
+        cmdArgs.extend(["-M", os.path.join(includeDir, "tmp"), "-w", "/tmp"])
         for f in os.listdir("/"):
             if f == "tmp": continue
             cmdArgs.extend(["-M", "/"+f, "-m", "/"+f])
@@ -481,7 +482,72 @@ def ensureSandboxUsable():
         except OSError as e:
             raise BuildError("Error executing sandbox helper: " + str(e))
 
-    async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=False):
+    def __wrapSandbox(self, realScriptFile, execScriptFile, callArgs, tmpDir):
+        # Wrap call into sandbox if requested
+        if self.__spec.hasSandbox:
+            # Every invocation gets its own, freshly created sandbox root.
+            # This is required because a single Invoker may run more than one
+            # sandboxed command (e.g. in 'exec' mode) and the sandbox helper
+            # does not tolerate a pre-populated root.
+            sandboxDir = tempfile.mkdtemp(dir=tmpDir)
+            if self.__spec.fatSandbox:
+                # The fat sandbox uses the sandbox root as its new root file
+                # system, i.e. "tmp" must exist as a real (writable)
+                # directory therein. Re-create the include files there by
+                # hard linking them instead of relying on a bind mount so
+                # that the jobserver FIFO can still be mounted underneath it.
+                includeTmpDir = os.path.join(tmpDir, "tmp")
+                sandboxTmpDir = os.path.join(sandboxDir, "tmp")
+                os.mkdir(sandboxTmpDir)
+                for name in os.listdir(includeTmpDir):
+                    os.link(os.path.join(includeTmpDir, name),
+                             os.path.join(sandboxTmpDir, name))
+                cmdArgs = self.__getFatSandboxCmds(sandboxDir)
+            else:
+                assert self.__spec.slimSandbox
+                cmdArgs = self.__getSlimSandboxCmds(sandboxDir, tmpDir)
+
+            cmdArgs.extend(["-M", os.path.abspath(realScriptFile), "-m"
+                , execScriptFile])
+
+            # Prevent network access
+            if not self.__spec.netAccess: cmdArgs.append('-n')
+
+            # Create empty env file. Otheriwse bind mount fails.
+            if self.__spec.envFile:
+                with open(self.__spec.envFile, "wb"): pass
+                cmdArgs.extend(["-M", os.path.abspath(self.__spec.envFile), "-w", "/bob/env"])
+
+            # Mount workspace writable and all dependencies read-only
+            cmdArgs.extend(["-M", os.path.abspath(self.__spec.workspaceWorkspacePath),
+                            "-w", os.path.abspath(self.__spec.workspaceExecPath)])
+            cmdArgs.extend(["-W", os.path.abspath(self.__spec.workspaceExecPath) ])
+            for argWorkspacePath, argExecPath in self.__spec.depMounts:
+                cmdArgs.extend(["-M", os.path.abspath(argWorkspacePath),
+                                "-m", os.path.abspath(argExecPath)])
+
+            # Mount jobserver FIFO into sandbox
+            if self.__makeJobServer.isFifo():
+                fifo = os.path.abspath(self.__makeJobServer.fifoPath())
+                cmdArgs.extend(["-M", fifo,
+                                "-w", "/tmp/bob-jobserver" if self.__spec.fatSandbox else fifo])
+
+            # Command follows. Stop parsing options.
+            cmdArgs.append("--")
+
+            # Hard override of PATH. The sandbox helper is found by an absolute path!
+            if self.__spec.fatSandbox:
+                env = { "PATH" : ":".join(self.__spec.sandboxPaths) }
+            else:
+                env = None
+        else:
+            cmdArgs = []
+            env = None
+        cmdArgs.extend(callArgs)
+        return cmdArgs, env
+
+    async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=False,
+                           commands=None):
         # make permissions predictable
         os.umask(0o022)
 
@@ -510,95 +576,61 @@ async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=Fal
             if not os.path.isdir(self.__spec.workspaceWorkspacePath):
                 os.makedirs(self.__spec.workspaceWorkspacePath, exist_ok=True)
                 workspaceCreated = True
-            elif clean and mode != InvocationMode.SHELL:
+            elif clean and mode not in (InvocationMode.SHELL, InvocationMode.EXEC):
                 emptyDirectory(self.__spec.workspaceWorkspacePath)
                 workspaceCreated = True
 
-            # setup script and arguments
-            if mode == InvocationMode.SHELL:
-                realScriptFile, execScriptFile, callArgs = self.__spec.language.setupShell(
-                    self.__spec, tmpDir, self.__preserveEnv)
-            elif mode == InvocationMode.CALL:
-                realScriptFile, execScriptFile, callArgs = self.__spec.language.setupCall(
-                    self.__spec, tmpDir, self.__preserveEnv, self.__trace)
-            elif mode == InvocationMode.UPDATE:
-                realScriptFile, execScriptFile, callArgs = self.__spec.language.setupUpdate(
-                    self.__spec, tmpDir, self.__preserveEnv, self.__trace)
+            if mode == InvocationMode.EXEC:
+                # Every command is set up, sandboxed and executed on its own.
+                # Execution stops at the first command that fails.
+                for command in commands:
+                    realScriptFile, execScriptFile, callArgs = self.__spec.language.setupExec(
+                        self.__spec, tmpDir, self.__preserveEnv, self.__trace, command)
+                    cmdArgs, env = self.__wrapSandbox(realScriptFile, execScriptFile,
+                                                       callArgs, tmpDir)
+                    await self.checkCommand(cmdArgs, env=env, specEnv=False)
             else:
-                assert False, "not reached"
-
-            # Wrap call into sandbox if requested
-            if self.__spec.hasSandbox:
-                if self.__spec.fatSandbox:
-                    cmdArgs = self.__getFatSandboxCmds(tmpDir)
+                # setup script and arguments
+                if mode == InvocationMode.SHELL:
+                    realScriptFile, execScriptFile, callArgs = self.__spec.language.setupShell(
+                        self.__spec, tmpDir, self.__preserveEnv)
+                elif mode == InvocationMode.CALL:
+                    realScriptFile, execScriptFile, callArgs = self.__spec.language.setupCall(
+                        self.__spec, tmpDir, self.__preserveEnv, self.__trace)
+                elif mode == InvocationMode.UPDATE:
+                    realScriptFile, execScriptFile, callArgs = self.__spec.language.setupUpdate(
+                        self.__spec, tmpDir, self.__preserveEnv, self.__trace)
                 else:
-                    assert self.__spec.slimSandbox
-                    cmdArgs = self.__getSlimSandboxCmds(tmpDir)
-
-                cmdArgs.extend(["-M", os.path.abspath(realScriptFile), "-m"
-                    , execScriptFile])
-
-                # Prevent network access
-                if not self.__spec.netAccess: cmdArgs.append('-n')
-
-                # Create empty env file. Otheriwse bind mount fails.
-                if self.__spec.envFile:
-                    with open(self.__spec.envFile, "wb"): pass
-                    cmdArgs.extend(["-M", os.path.abspath(self.__spec.envFile), "-w", "/bob/env"])
-
-                # Mount workspace writable and all dependencies read-only
-                cmdArgs.extend(["-M", os.path.abspath(self.__spec.workspaceWorkspacePath),
-                                "-w", os.path.abspath(self.__spec.workspaceExecPath)])
-                cmdArgs.extend(["-W", os.path.abspath(self.__spec.workspaceExecPath) ])
-                for argWorkspacePath, argExecPath in self.__spec.depMounts:
-                    cmdArgs.extend(["-M", os.path.abspath(argWorkspacePath),
-                                    "-m", os.path.abspath(argExecPath)])
-
-                # Mount jobserver FIFO into sandbox
-                if self.__makeJobServer.isFifo():
-                    fifo = os.path.abspath(self.__makeJobServer.fifoPath())
-                    cmdArgs.extend(["-M", fifo,
-                                    "-w", "/tmp/bob-jobserver" if self.__spec.fatSandbox else fifo])
-
-                # Command follows. Stop parsing options.
-                cmdArgs.append("--")
-
-                # Hard override of PATH. The sandbox helper is found by an absolute path!
-                if self.__spec.fatSandbox:
-                    env = { "PATH" : ":".join(self.__spec.sandboxPaths) }
+                    raise AssertionError("not reached")
+
+                cmdArgs, env = self.__wrapSandbox(realScriptFile, execScriptFile, callArgs, tmpDir)
+
+                if mode == InvocationMode.SHELL:
+                    ret = await self.callCommand(cmdArgs, env=env, specEnv=False)
+                elif mode in (InvocationMode.CALL, InvocationMode.UPDATE):
+                    for scm in self.__spec.preRunCmds:
+                        scm = getScm(scm)
+                        if mode == InvocationMode.UPDATE and not scm.isLocal():
+                            continue # Skip non-local SCMs on update-only
+                        try:
+                            await scm.invoke(self, workspaceCreated)
+                        except CmdFailedError as e:
+                            self.error(scm.getSource(), "failed")
+                            self.error(e.what)
+                            raise
+                        except Exception:
+                            self.error(scm.getSource(), "failed")
+                            raise
+                    await self.checkCommand(cmdArgs, env=env, specEnv=False)
+                    for a in self.__spec.postRunCmds:
+                        a = CheckoutAssert(a)
+                        try:
+                            await a.invoke(self)
+                        except Exception:
+                            self.error(a.getSource(), "failed")
+                            raise
                 else:
-                    env = None
-            else:
-                cmdArgs = []
-                env = None
-            cmdArgs.extend(callArgs)
-
-            if mode == InvocationMode.SHELL:
-                ret = await self.callCommand(cmdArgs, env=env, specEnv=False)
-            elif mode in (InvocationMode.CALL, InvocationMode.UPDATE):
-                for scm in self.__spec.preRunCmds:
-                    scm = getScm(scm)
-                    if mode == InvocationMode.UPDATE and not scm.isLocal():
-                        continue # Skip non-local SCMs on update-only
-                    try:
-                        await scm.invoke(self, workspaceCreated)
-                    except CmdFailedError as e:
-                        self.error(scm.getSource(), "failed")
-                        self.error(e.what)
-                        raise
-                    except Exception:
-                        self.error(scm.getSource(), "failed")
-                        raise
-                await self.checkCommand(cmdArgs, env=env, specEnv=False)
-                for a in self.__spec.postRunCmds:
-                    a = CheckoutAssert(a)
-                    try:
-                        await a.invoke(self)
-                    except Exception:
-                        self.error(a.getSource(), "failed")
-                        raise
-            else:
-                assert False, "not reached"
+                    raise AssertionError("not reached")
 
             # everything went well
             ret = 0

pym/bob/languages.py

@@ -400,6 +400,10 @@ def setupCall(spec, tmpDir, keepEnv, trace):
     def setupUpdate(spec, tmpDir, keepEnv, trace):
         return BashLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace)
 
+    @staticmethod
+    def setupExec(spec, tmpDir, keepEnv, trace, command):
+        return BashLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace)
+
     @staticmethod
     def mangleFingerprints(scriptFragments, env):
         # join the script fragments first
@@ -616,6 +620,10 @@ def setupCall(spec, tmpDir, keepEnv, trace):
     def setupUpdate(spec, tmpDir, keepEnv, trace):
         return PwshLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace)
 
+    @staticmethod
+    def setupExec(spec, tmpDir, keepEnv, trace, command):
+        return PwshLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace)
+
     @staticmethod
     def mangleFingerprints(scriptFragments, env):
         # join the script fragments first
@@ -773,6 +781,10 @@ def setupCall(spec, tmpDir, keepEnv, trace):
     def setupUpdate(spec, tmpDir, keepEnv, trace):
         return PythonLanguage.__setupExec(spec, spec.updateScript, tmpDir, keepEnv, trace)
 
+    @staticmethod
+    def setupExec(spec, tmpDir, keepEnv, trace, command):
+        return PythonLanguage.__setupExec(spec, command, tmpDir, keepEnv, trace)
+
     @staticmethod
     def mangleFingerprints(scriptFragments, env):
         # join the script fragments first

test/black-box/exec/config.yaml

@@ -0,0 +1,2 @@
+bobMinimumVersion: "1.2"
+windowsPowerShellExecutable: "pwsh"

test/black-box/exec/recipes/bash.yaml

@@ -0,0 +1,10 @@
+root: True
+
+depends:
+    - name: sandbox
+      use: [sandbox]
+
+buildScript: |
+    echo "build ok"
+packageScript: |
+    echo "package ok"

test/black-box/exec/recipes/pwsh.yaml

@@ -0,0 +1,12 @@
+root: True
+
+depends:
+    - name: sandbox
+      use: [sandbox]
+
+scriptLanguage: PowerShell
+
+buildScript: |
+    Write-Host "build ok"
+packageScript: |
+    Write-Host "package ok"