Svn mirrors

doc/manual/configuration.rst

@@ -3033,9 +3033,16 @@ these suffixes instead of the bare ``preMirror``/``fallbackMirror`` to enable
 interoperability between projects, layers and the local user configuration.
 
 Each mirror entry specifies the SCM type (``scm``), a regular expression to
-match the URL (``url``) and a replacement URL (``mirror``). Optionally, it is
-possible to upload to the mirror (``upload``). Currently only the URL SCM is
-supported for mirrors.
+match the URL (``url``) and a replacement URL (``mirror``). Optionally, for
+the URL SCM it is possible to upload to the mirror (``upload``). Currently
+the URL and SVN SCMs are supported for mirrors. For the SVN SCM, mirrors are
+only tried during the initial checkout of a working copy. Uploading to a SVN
+mirror is not supported.
+
+.. attention::
+   Bob assumes that SVN mirrors hold the same content as the primary URL. This
+   implies that changing the mirror URLs won't lead to new checkout executions.
+   Only the primary URL of the recipe is indicative for the content.
 
 Examples::
 
@@ -3046,6 +3053,9 @@ Examples::
         - scm: url
           url: "https?://ftp.gnu.org/(pub/)?gnu/(.*)"
           mirror: "http://www.mirrorservice.org/sites/ftp.gnu.org/gnu/\\2"
+        - scm: svn
+          url: "https://company.local/svn/repo/(.*)"
+          mirror: "file:///Z:/attic/svn/repo/\\1"
 
 A typical mirror configuration for the global user configuration could look
 like the following. It mirrors all remote URLs to a local directory::

pym/bob/input.py

@@ -3652,6 +3652,7 @@ class RecipeSet:
     }
 
     MIRRORS_SCHEMA = ScmValidator({
+        'svn' : SvnScm.MIRRORS_SCHEMA,
         'url' : UrlScm.MIRRORS_SCHEMA,
     })
 

pym/bob/scm/__init__.py

@@ -69,7 +69,9 @@ def getScm(spec, overrides=[], recipeSet=None):
             recipeSet and recipeSet.getPolicy('fixImportScmVariant'),
             recipeSet and recipeSet.getProjectRoot())
     elif scm == "svn":
-        return SvnScm(spec, overrides)
+        return SvnScm(spec, overrides,
+            recipeSet.getPreMirrors() if recipeSet else [],
+            recipeSet.getFallbackMirrors() if recipeSet else [])
     elif scm == "cvs":
         return CvsScm(spec, overrides)
     elif scm == "url":

pym/bob/scm/svn.py

@@ -10,7 +10,9 @@
 from shlex import quote
 from textwrap import indent
 import os, os.path
+import re
 import schema
+import shutil
 import subprocess
 from xml.etree import ElementTree
 
@@ -43,17 +45,27 @@ class SvnScm(Scm):
         **__DEFAULTS,
     })
 
+    MIRRORS_SCHEMA = schema.Schema({
+        'scm' : 'svn',
+        'url' : re.compile,
+        'mirror' : str,
+    })
+
     DEFAULT_VALUES = {
         "dir" : ".",
         "sslVerify" : True,
     }
 
-    def __init__(self, spec, overrides=[]):
+    def __init__(self, spec, overrides=[], preMirrors=[], fallbackMirrors=[]):
         super().__init__(spec, overrides)
         self.__url = spec["url"]
         self.__dir = spec.get("dir", ".")
         self.__revision = spec.get("revision")
         self.__sslVerify = spec.get('sslVerify', True)
+        self.__preMirrors = preMirrors
+        self.__fallbackMirrors = fallbackMirrors
+        self.__preMirrorsUrls = spec.get("preMirrors")
+        self.__fallbackMirrorsUrls = spec.get("fallbackMirrors")
 
     def getProperties(self, isJenkins, pretty=False):
         ret = super().getProperties(isJenkins, pretty)
@@ -62,6 +74,8 @@ def getProperties(self, isJenkins, pretty=False):
             "url" : self.__url,
             "dir" : self.__dir,
             'sslVerify' : self.__sslVerify,
+            'preMirrors' : self.__getPreMirrors(),
+            'fallbackMirrors' : self.__getFallbackMirrors(),
         })
         if self.__revision:
             ret["revision"] = self.__revision
@@ -72,6 +86,26 @@ def getProperties(self, isJenkins, pretty=False):
 
         return ret
 
+    def __applyMirrors(self, mirrors):
+        return [ re.sub(m['url'], m['mirror'], self.__url)
+                 for m in mirrors
+                 if m['scm'] == 'svn' and re.match(m['url'], self.__url) ]
+
+    def __getPreMirrors(self):
+        ret = self.__preMirrorsUrls
+        if ret is None:
+            ret = self.__preMirrorsUrls = self.__applyMirrors(self.__preMirrors)
+        return ret
+
+    def __getFallbackMirrors(self):
+        ret = self.__fallbackMirrorsUrls
+        if ret is None:
+            ret = self.__fallbackMirrorsUrls = self.__applyMirrors(self.__fallbackMirrors)
+        return ret
+
+    def _getCandidateUrls(self):
+        return self.__getPreMirrors() + [self.__url] + self.__getFallbackMirrors()
+
     async def invoke(self, invoker, workspaceCreated):
         options = [ "--non-interactive" ]
         if not self.__sslVerify:
@@ -83,7 +117,19 @@ async def invoke(self, invoker, workspaceCreated):
             if "/tags/" not in self.__url:
                 await invoker.checkCommand(["svn", "up"] + options, cwd=self.__dir)
         else:
-            await invoker.checkCommand(["svn", "co"] + options + [self.__url, self.__dir])
+            err = None
+            for url in self._getCandidateUrls():
+                if err:
+                    invoker.warn(f"Checkout from '{err}' failed! Trying '{url}' next...")
+                ret = await invoker.callCommand(["svn", "co"] + options + [url, self.__dir])
+                if ret == 0:
+                    break
+                svnDir = invoker.joinPath(self.__dir, ".svn")
+                if os.path.isdir(svnDir):
+                    shutil.rmtree(svnDir)
+                err = url
+            else:
+                invoker.fail("Checkout of '{}' failed!".format(err))
 
     def asDigestScript(self):
         """Return forward compatible stable string describing this svn module.
@@ -134,6 +180,11 @@ def isDeterministic(self):
         return str(self.__revision).isnumeric()
 
     def hasJenkinsPlugin(self):
+        # The Jenkins plugin does not support mirrors. Fall back to direct
+        # execution in this case.
+        if self.__getPreMirrors() or self.__getFallbackMirrors():
+            return False
+
         return True
 
     def callSubversion(self, workspacePath, *args):

test/unit/test_input_svnscm.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from unittest import TestCase, skipIf
+from unittest.mock import MagicMock
+import os
+import subprocess
+import sys
+import tempfile
+
+from bob.input import SvnScm
+from bob.invoker import Invoker, InvocationError
+from bob.utils import isMsys, runInEventLoop
+
+if sys.platform == "win32":
+    def makeUrl(path):
+        return 'file:///' + path.replace("\\", "/")
+else:
+    def makeUrl(path):
+        return 'file://' + path
+
+def createSvnRepo(root, name, content):
+    repodir = os.path.join(root, name)
+    subprocess.check_call(['svnadmin', 'create', name], cwd=root)
+    with tempfile.TemporaryDirectory() as tmp:
+        with open(os.path.join(tmp, "test.txt"), "w") as f:
+            f.write(content)
+        subprocess.check_call(['svn', 'import', tmp, makeUrl(repodir) + '/trunk',
+            '-m', "Initial Import"], cwd=tempfile.gettempdir())
+    return makeUrl(repodir) + '/trunk'
+
+
+@skipIf(isMsys(), "svnadmin fails on MSYS")
+class TestSvnScmMirrors(TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.__repodir_root = tempfile.TemporaryDirectory()
+        root = cls.__repodir_root.name
+        cls.mainUrl = createSvnRepo(root, "main", "main")
+        cls.mirrorUrl = createSvnRepo(root, "mirror", "mirror")
+        cls.invalidUrl = makeUrl(os.path.join(root, "does-not-exist")) + '/trunk'
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.__repodir_root.cleanup()
+
+    def createSvnScm(self, spec={}, preMirrors=[], fallbackMirrors=[]):
+        s = {
+            'scm' : 'svn',
+            'url' : self.mainUrl,
+            'revision' : 1,
+            'recipe' : "foo.yaml#0",
+            '__source' : "Recipe foo",
+        }
+        s.update(spec)
+        return SvnScm(s, preMirrors=preMirrors, fallbackMirrors=fallbackMirrors)
+
+    def invokeScm(self, workspace, scm):
+        spec = MagicMock(workspaceWorkspacePath=workspace, envWhiteList=set())
+        invoker = Invoker(spec, True, True, True, True, True, False)
+        runInEventLoop(scm.invoke(invoker, False))
+
+    def assertContent(self, workspace, content):
+        with open(os.path.join(workspace, "test.txt")) as f:
+            self.assertEqual(f.read(), content)
+
+    def testPlainCheckout(self):
+        """Sanity check: a plain checkout still works"""
+        scm = self.createSvnScm()
+        with tempfile.TemporaryDirectory() as workspace:
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "main")
+
+    def testPreMirrorUsed(self):
+        """A pre-mirror is used before the primary URL"""
+        scm = self.createSvnScm(
+            { 'url' : self.invalidUrl },
+            preMirrors=[{ 'scm' : 'svn', 'url' : r".+", 'mirror' : self.mirrorUrl }])
+        with tempfile.TemporaryDirectory() as workspace:
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "mirror")
+
+    def testFallbackMirrorUsed(self):
+        """A fallback mirror is used if the primary URL fails"""
+        scm = self.createSvnScm(
+            { 'url' : self.invalidUrl },
+            fallbackMirrors=[{ 'scm' : 'svn', 'url' : r".+", 'mirror' : self.mirrorUrl }])
+        with tempfile.TemporaryDirectory() as workspace:
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "mirror")
+
+    def testGracefulMirrorFallback(self):
+        """A failing mirror is ignored and the next candidate is used"""
+        scm = self.createSvnScm(
+            { 'url' : self.invalidUrl },
+            preMirrors=[
+                { 'scm' : 'svn', 'url' : r".+", 'mirror' : self.invalidUrl },
+                { 'scm' : 'svn', 'url' : r".+", 'mirror' : self.mirrorUrl },
+            ])
+        with tempfile.TemporaryDirectory() as workspace:
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "mirror")
+
+    def testAllCandidatesFail(self):
+        """If no candidate works the checkout fails"""
+        scm = self.createSvnScm(
+            { 'url' : self.invalidUrl },
+            fallbackMirrors=[{ 'scm' : 'svn', 'url' : r".+", 'mirror' : self.invalidUrl }])
+        with tempfile.TemporaryDirectory() as workspace:
+            with self.assertRaises(InvocationError):
+                self.invokeScm(workspace, scm)
+
+    def testOtherScmMirrorIgnored(self):
+        """Mirrors of other SCM types are not applied to svn checkouts"""
+        scm = self.createSvnScm(
+            { 'url' : self.invalidUrl },
+            preMirrors=[{ 'scm' : 'url', 'url' : r".+", 'mirror' : self.mirrorUrl }])
+        with tempfile.TemporaryDirectory() as workspace:
+            with self.assertRaises(InvocationError):
+                self.invokeScm(workspace, scm)
+
+    def testMirrorKeptOnUpdate(self):
+        """A chosen mirror is kept across invocations"""
+        scm = self.createSvnScm(
+            preMirrors=[{ 'scm' : 'svn', 'url' : r".+", 'mirror' : self.mirrorUrl }])
+        with tempfile.TemporaryDirectory() as workspace:
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "mirror")
+
+            # Update without mirror. The existing checkout must stay on the
+            # mirror even though the primary URL is available.
+            scm = self.createSvnScm()
+            self.invokeScm(workspace, scm)
+            self.assertContent(workspace, "mirror")
+            info = subprocess.check_output(['svn', 'info', '--show-item', 'url'],
+                cwd=workspace, universal_newlines=True)
+            self.assertEqual(info.strip(), self.mirrorUrl)