From feacf59a533e9a80e00a7efda34603bd796d4b4b Mon Sep 17 00:00:00 2001 From: ehennestad Date: Mon, 21 Sep 2026 20:13:12 +0200 Subject: [PATCH 1/3] feat: add a task that copies a package from a GitHub repository matbox.tasks.vendorPackage downloads a GitHub repository at a branch, tag or commit and copies one of its folders into a project, replacing the previous copy. A toolbox uses it to ship another package inside its own namespace instead of requiring a separate installation. When the source and target are namespace folders with different names, the references to the source namespace in the copied .m files are renamed, for example "webprogress.upload" to "mytoolbox.external.webprogress.upload", in code, strings and comments. Error identifiers keep the source name. Files are rewritten as bytes, so everything outside the renamed references is unchanged whatever the encoding of the file. The license file of the repository is copied along, and vendorinfo.json in the copy records the repository, the reference, the commit, the two namespace names and the excluded files. The reference is resolved to a commit before the download, so the recorded commit is the one the files come from. The copy is prepared in a temporary folder and replaces the target as the last step, so an error leaves an existing copy unchanged. A target folder without a vendorinfo.json file is never replaced, so a wrong target path cannot delete other source code. The copy itself is in matbox.tasks.internal.vendorPackageFromFolder, which needs no network access. VendorPackageTest tests it against a repository created on disk; one test downloads the v0.11.0 tag of MatBox. Co-Authored-By: Claude Fable 5.1 --- .../+tasks/+internal/getNamespaceName.m | 42 ++ .../+tasks/+internal/renameNamespaceInText.m | 47 ++ .../+internal/vendorPackageFromFolder.m | 172 ++++++++ code/+matbox/+tasks/vendorPackage.m | 115 +++++ .../+unittest/VendorPackageTest.m | 412 ++++++++++++++++++ 5 files changed, 788 insertions(+) create mode 100644 code/+matbox/+tasks/+internal/getNamespaceName.m create mode 100644 code/+matbox/+tasks/+internal/renameNamespaceInText.m create mode 100644 code/+matbox/+tasks/+internal/vendorPackageFromFolder.m create mode 100644 code/+matbox/+tasks/vendorPackage.m create mode 100644 tools/tests/+matboxtools/+unittest/VendorPackageTest.m diff --git a/code/+matbox/+tasks/+internal/getNamespaceName.m b/code/+matbox/+tasks/+internal/getNamespaceName.m new file mode 100644 index 0000000..2fc23f5 --- /dev/null +++ b/code/+matbox/+tasks/+internal/getNamespaceName.m @@ -0,0 +1,42 @@ +function namespaceName = getNamespaceName(folderPath) +%getNamespaceName - Get the name of the namespace that a folder defines +% namespaceName = matbox.tasks.internal.getNamespaceName(folderPath) +% returns the namespace name given by the namespace folders at the end +% of folderPath, joined with ".". Both "/" and "\" are accepted as +% separators. The result is "" if the last folder is not a namespace +% folder. +% +% Example: Namespace of a nested namespace folder +% folderPath = "src/mytoolbox/+mytoolbox/+external/+webprogress"; +% name = matbox.tasks.internal.getNamespaceName(folderPath) +% % name = "mytoolbox.external.webprogress" +% +% See also matbox.tasks.vendorPackage + + arguments + folderPath (1,1) string + end + + NAMESPACE_PREFIX = "+"; + + folderNames = split(replace(folderPath, "\", "/"), "/"); + folderNames(folderNames == "") = []; + + % Only the unbroken run of namespace folders at the end of the path + % counts, because a namespace folder below an ordinary folder starts a + % new namespace. + isNamespaceFolder = startsWith(folderNames, NAMESPACE_PREFIX); + lastOrdinaryFolder = find(~isNamespaceFolder, 1, "last"); + if isempty(lastOrdinaryFolder) + lastOrdinaryFolder = 0; + end + namespaceFolders = folderNames(lastOrdinaryFolder+1:end); + + % join returns for an empty array, so that case is set apart. + isInNamespace = ~isempty(namespaceFolders); + if isInNamespace + namespaceName = join(extractAfter(namespaceFolders, NAMESPACE_PREFIX), "."); + else + namespaceName = ""; + end +end diff --git a/code/+matbox/+tasks/+internal/renameNamespaceInText.m b/code/+matbox/+tasks/+internal/renameNamespaceInText.m new file mode 100644 index 0000000..407bccd --- /dev/null +++ b/code/+matbox/+tasks/+internal/renameNamespaceInText.m @@ -0,0 +1,47 @@ +function text = renameNamespaceInText(text, sourceNamespace, targetNamespace) +%renameNamespaceInText - Rename the references to a namespace in code +% text = matbox.tasks.internal.renameNamespaceInText(text,sourceNamespace,targetNamespace) +% replaces each reference to sourceNamespace in text, such as +% "webprogress.upload", with the same reference to targetNamespace, such +% as "mytoolbox.external.webprogress.upload". Code, strings and comments +% are all renamed. The class of text is kept. +% +% A reference is the namespace name followed by "." and a letter or "*" +% (as in "import webprogress.*"). The name is not a reference when a +% letter, digit, "_", ".", "/" or "\" comes before it, which leaves +% longer names ("mywebprogress.x"), property access ("obj.webprogress.x") +% and paths ("src/webprogress.zip") unchanged. Error identifiers +% ("webprogress:upload:Failed") have no "." after the name and are also +% unchanged. A variable with the same name as the namespace is +% indistinguishable from the namespace and is renamed. +% +% See also matbox.tasks.vendorPackage, regexprep + + arguments + text {mustBeTextScalar} + sourceNamespace (1,1) string {mustBeNamespaceName} + targetNamespace (1,1) string {mustBeNamespaceName} + end + + % The character classes are spelled out because \w may also match + % non-ASCII letters, and the text can be the raw bytes of a file. + notInsideLongerName = "(? 0 && ~isfolder(parentFolder) + mkdir(parentFolder) + end + movefile(stagingFolder, targetFolder) + + if nargout < 1 + clear vendorInfo + end +end + +function removeExcludedFiles(folderPath, excludeFiles) +%removeExcludedFiles - Delete the excluded files from the copied folder + excludedPaths = fullfile(folderPath, excludeFiles); + + % A name that matches nothing is an error, because it no longer leaves + % out the file it was meant for: the source has renamed or removed it. + isMissing = ~isfile(excludedPaths); + if any(isMissing) + error("MatBox:VendorPackage:ExcludedFileNotFound", ... + "The source folder has no file ""%s"". Update the ExcludeFiles " + ... + "option to match the files of the source folder.", ... + strjoin(excludeFiles(isMissing), """, """)) + end + + for i = 1:numel(excludedPaths) + delete(excludedPaths(i)) + end +end + +function renameNamespaceInFolder(folderPath, sourceNamespace, targetNamespace) +%renameNamespaceInFolder - Rename a namespace in all .m files below a folder + fileListing = dir(fullfile(folderPath, "**", "*.m")); + filePaths = string(fullfile({fileListing.folder}, {fileListing.name})); + + for i = 1:numel(filePaths) + % The file is handled as bytes, one character per byte, so that + % everything but the renamed references is written back unchanged + % whatever the encoding of the file is. The namespace names are + % ASCII, which every common encoding stores as single bytes. + originalText = char(readBytes(filePaths(i))); + renamedText = matbox.tasks.internal.renameNamespaceInText( ... + originalText, sourceNamespace, targetNamespace); + + if ~strcmp(renamedText, originalText) + writeBytes(filePaths(i), uint8(renamedText)) + end + end +end + +function bytes = readBytes(filePath) +%readBytes - Read a file as a row vector of bytes + fileId = fopen(filePath, "r"); + fileCleanup = onCleanup(@() fclose(fileId)); + bytes = fread(fileId, [1, Inf], "*uint8"); +end + +function writeBytes(filePath, bytes) +%writeBytes - Replace the content of a file with a vector of bytes + fileId = fopen(filePath, "w"); + fileCleanup = onCleanup(@() fclose(fileId)); + fwrite(fileId, bytes, "uint8"); +end + +function writeVendorInfo(filePath, vendorInfo) +%writeVendorInfo - Save the vendor information as a JSON file + % A cell array is encoded as a JSON array whatever its length, while a + % string array with one element would be encoded as a JSON string. + vendorInfo.ExcludedFiles = cellstr(vendorInfo.ExcludedFiles); + writeBytes(filePath, unicode2native(jsonencode(vendorInfo, PrettyPrint=true), "UTF-8")) +end + +function removeFolderIfPresent(folderPath) +%removeFolderIfPresent - Remove a folder and its content if the folder exists + if isfolder(folderPath) + rmdir(folderPath, "s") + end +end diff --git a/code/+matbox/+tasks/vendorPackage.m b/code/+matbox/+tasks/vendorPackage.m new file mode 100644 index 0000000..333d86b --- /dev/null +++ b/code/+matbox/+tasks/vendorPackage.m @@ -0,0 +1,115 @@ +function vendorInfo = vendorPackage(projectRootDirectory, sourceUri, sourceFolder, targetFolder, options) +%vendorPackage - Copy a folder from a GitHub repository into a project +% matbox.tasks.vendorPackage(projectRootDirectory,sourceUri,sourceFolder,targetFolder) +% downloads the GitHub repository sourceUri and copies its folder +% sourceFolder to the folder targetFolder of the project, which is +% replaced if it exists. sourceUri has the form +% "https://github.com//[@ref]", where ref is a branch name, +% tag name or commit SHA, and the default branch if left out. +% sourceFolder is relative to the root of the repository and +% targetFolder is relative to projectRootDirectory. +% +% If the two folders are namespace folders with different namespace +% names, the references to the source namespace in the copied .m files +% are renamed. For example, copying "src/webprogress/+webprogress" to +% "src/mytoolbox/+mytoolbox/+external/+webprogress" turns +% "webprogress.upload" into "mytoolbox.external.webprogress.upload" in +% code, strings and comments. Error identifiers keep the source name. +% +% The license file of the repository is copied into targetFolder, and +% the file vendorinfo.json there records the repository, the reference, +% the commit that was copied, the two namespace names and the excluded +% files. Changes made to the copied files are lost on the next call, +% because the folder is replaced as a whole. A folder without a +% vendorinfo.json file is never replaced. +% +% vendorInfo = matbox.tasks.vendorPackage(...) returns the information +% written to vendorinfo.json as a struct. +% +% [...] = matbox.tasks.vendorPackage(...,ExcludeFiles=FILES) leaves out +% the files FILES, given relative to sourceFolder with "/" as separator, +% for example ["toolboxdir.m","private/helper.m"]. A name that matches +% no file is an error. +% +% [...] = matbox.tasks.vendorPackage(...,LicenseFile=FILE) copies FILE, +% given relative to the root of the repository, as the license file. The +% default is "LICENSE". +% +% [...] = matbox.tasks.vendorPackage(...,Verbose=TF) prints a summary of +% the copy if TF is true, the default. +% +% Example: Copy a package into the namespace of a toolbox +% sourceUri = "https://github.com/ehennestad/http-progressbar-matlab@main"; +% sourceFolder = "src/webprogress/+webprogress"; +% targetFolder = "src/mytoolbox/external/+mytoolbox/+external/+webprogress"; +% matbox.tasks.vendorPackage(pwd, sourceUri, sourceFolder, ... +% targetFolder, ExcludeFiles=["toolboxdir.m","toolboxversion.m"]) +% +% See also matbox.installRequirements, matbox.tasks.packageToolbox + + arguments + projectRootDirectory (1,1) string {mustBeFolder} + sourceUri (1,1) string + sourceFolder (1,1) string + targetFolder (1,1) string {mustBeNonzeroLengthText, mustStayBelowRoot} + options.ExcludeFiles (1,:) string = string.empty + options.LicenseFile (1,1) string = "LICENSE" + options.Verbose (1,1) logical = true + end + + [ownerName, repositoryName, gitRef] = ... + matbox.setup.internal.github.parseRepositoryURL(sourceUri); + repositoryUrl = "https://github.com/" + ownerName + "/" + repositoryName; + + if ismissing(gitRef) + gitRef = matbox.setup.internal.github.api.getDefaultBranch( ... + ownerName, repositoryName); + end + + % The reference is resolved to a commit first and the archive of that + % commit is downloaded, so that the recorded commit is the one the files + % come from even if a branch moves between the two requests. + commitId = string(matbox.setup.internal.github.api.getCurrentCommitID( ... + repositoryName, Owner=ownerName, BranchName=gitRef)); + + downloadFolder = string(tempname()); + mkdir(downloadFolder) + downloadFolderCleanup = onCleanup(@() rmdir(downloadFolder, "s")); + + % The URL is a character vector because downloadZippedGithubRepo builds + % a file name from it by concatenation. + archiveUrl = sprintf('%s/archive/%s.zip', repositoryUrl, commitId); + isUpdate = false; + throwErrorIfFails = true; + repositoryFolder = matbox.setup.internal.downloadZippedGithubRepo( ... + archiveUrl, downloadFolder, isUpdate, throwErrorIfFails); + + vendorInfo = matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, sourceFolder, fullfile(projectRootDirectory, targetFolder), ... + ExcludeFiles=options.ExcludeFiles, ... + LicenseFile=options.LicenseFile, ... + RepositoryUrl=repositoryUrl, ... + Reference=gitRef, ... + CommitID=commitId); + + if options.Verbose + SHORT_COMMIT_LENGTH = 7; + fprintf("Copied ""%s"" of %s at commit %s to ""%s"".\n", ... + sourceFolder, repositoryUrl, ... + extractBefore(commitId, SHORT_COMMIT_LENGTH+1), targetFolder) + end + + if nargout < 1 + clear vendorInfo + end +end + +function mustStayBelowRoot(folderPath) +%mustStayBelowRoot - Validate that a relative path has no ".." folder + folderNames = split(replace(folderPath, "\", "/"), "/"); + if any(folderNames == "..") + error("MatBox:VendorPackage:InvalidTargetFolder", ... + "The target folder ""%s"" must be below the project root " + ... + "directory. Give a path without "".."".", folderPath) + end +end diff --git a/tools/tests/+matboxtools/+unittest/VendorPackageTest.m b/tools/tests/+matboxtools/+unittest/VendorPackageTest.m new file mode 100644 index 0000000..6efebbd --- /dev/null +++ b/tools/tests/+matboxtools/+unittest/VendorPackageTest.m @@ -0,0 +1,412 @@ +classdef VendorPackageTest < matlab.unittest.TestCase +% VendorPackageTest - Tests for copying a package of another repository into a project +% +% matbox.tasks.vendorPackage downloads a repository and hands the copy +% to matbox.tasks.internal.vendorPackageFromFolder. Most tests call the +% latter with a repository that the test creates on disk, and need no +% network access. testVendorFromGitHub downloads a tagged release of +% MatBox itself. + + properties (Constant) + SourceFolder = "src/webprogress/+webprogress" + SourceNamespace = "webprogress" + TargetNamespace = "mytoolbox.external.webprogress" + ExcludedFile = "toolboxdir.m" + LicenseText = "License of the source repository" + % A tagged release of MatBox and the commit the tag points to + GitHubSourceUri = "https://github.com/ehennestad/matbox@v0.11.0" + GitHubCommitId = "a805e35ea3905d914b41e4001e53a943519eb4d7" + end + + properties (TestParameter) + % Text with a reference to SourceNamespace, followed by the text + % expected after renaming the namespace to TargetNamespace + referenceCase = struct( ... + "functionCall", ["x = webprogress.upload(a);", ... + "x = mytoolbox.external.webprogress.upload(a);"], ... + "functionHandle", ["f = @(o) webprogress.Monitor(o{:});", ... + "f = @(o) mytoolbox.external.webprogress.Monitor(o{:});"], ... + "staticMethod", ["tf = ~webprogress.Monitor.isWebFigure(fig);", ... + "tf = ~mytoolbox.external.webprogress.Monitor.isWebFigure(fig);"], ... + "importStatement", ["import webprogress.*", ... + "import mytoolbox.external.webprogress.*"], ... + "helpText", ["% webprogress.upload(FILENAME,URL) uploads", ... + "% mytoolbox.external.webprogress.upload(FILENAME,URL) uploads"], ... + "metaClassQuery", ["?webprogress.Monitor", ... + "?mytoolbox.external.webprogress.Monitor"]) + + % Text in which the name "webprogress" is not a namespace reference + unchangedText = struct( ... + "errorIdentifier", "error(""webprogress:upload:Failed"", ""Failed."")", ... + "longerNamespaceName", "webprogresstools.projectdir()", ... + "nameWithPrefix", "mywebprogress.upload(a)", ... + "propertyAccess", "obj.webprogress.upload", ... + "folderPath", "% see src/webprogress/+webprogress", ... + "fileName", "% see https://github.com/owner/webprogress.git", ... + "endOfSentence", "% is in the namespace webprogress. It has", ... + "alreadyRenamed", "mytoolbox.external.webprogress.upload(a)") + + % A folder path followed by the name of the namespace it defines + namespaceCase = struct( ... + "singleFolder", ["src/webprogress/+webprogress", "webprogress"], ... + "nestedFolders", ["src/tb/+tb/+external/+webprogress", "tb.external.webprogress"], ... + "backslashSeparators", ["src\tb\+tb\+external", "tb.external"], ... + "trailingSeparator", ["src/+tb/", "tb"], ... + "onlyNamespaceFolders", ["+tb/+external", "tb.external"], ... + "restartedBelowOrdinaryFolder", ["+tb/external/+tb/+external", "tb.external"], ... + "ordinaryFolder", ["src/webprogress", ""], ... + "ordinaryFolderInNamespace", ["src/+tb/private", ""]) + end + + methods (TestMethodSetup) + function setupMethod(testCase) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture) + end + end + + methods (Test) % renameNamespaceInText + function testReferenceIsRenamed(testCase, referenceCase) + renamedText = matbox.tasks.internal.renameNamespaceInText( ... + referenceCase(1), testCase.SourceNamespace, testCase.TargetNamespace); + + testCase.verifyEqual(renamedText, referenceCase(2)) + end + + function testOtherTextIsUnchanged(testCase, unchangedText) + renamedText = matbox.tasks.internal.renameNamespaceInText( ... + unchangedText, testCase.SourceNamespace, testCase.TargetNamespace); + + testCase.verifyEqual(renamedText, unchangedText) + end + + function testDotsInNamespaceNameMatchLiterally(testCase) + % An unescaped "." matches any character, so "axb.run" would be + % taken for a reference to the namespace "a.b". + renamedText = matbox.tasks.internal.renameNamespaceInText( ... + "axb.run() + a.b.run()", "a.b", "c.d"); + + testCase.verifyEqual(renamedText, "axb.run() + c.d.run()") + end + + function testClassOfTextIsKept(testCase) + renamedText = matbox.tasks.internal.renameNamespaceInText( ... + 'webprogress.upload(a)', testCase.SourceNamespace, testCase.TargetNamespace); + + testCase.verifyEqual(renamedText, 'mytoolbox.external.webprogress.upload(a)') + end + + function testInvalidNamespaceNameErrors(testCase) + testCase.verifyError( ... + @() matbox.tasks.internal.renameNamespaceInText("x", "web progress", "a"), ... + "MatBox:VendorPackage:InvalidNamespaceName") + end + end + + methods (Test) % getNamespaceName + function testNamespaceNameOfFolder(testCase, namespaceCase) + namespaceName = matbox.tasks.internal.getNamespaceName(namespaceCase(1)); + + testCase.verifyEqual(namespaceName, namespaceCase(2)) + end + end + + methods (Test) % vendorPackageFromFolder + function testFilesAreCopiedAndRenamed(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + uploadCode = fileread(fullfile(targetFolder, "upload.m")); + testCase.verifySubstring(uploadCode, ... + "@(o) mytoolbox.external.webprogress.Monitor(o{:})") + testCase.verifySubstring(uploadCode, ... + "% mytoolbox.external.webprogress.upload(FILENAME,URL) uploads") + testCase.verifySubstring(uploadCode, """webprogress:upload:Failed""") + testCase.verifyTrue(isfile(fullfile(targetFolder, "private", "mustBeValidUrl.m"))) + end + + function testExcludedFilesAreNotCopied(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + ExcludeFiles=[testCase.ExcludedFile, "private/mustBeValidUrl.m"]) + + testCase.verifyFalse(isfile(fullfile(targetFolder, testCase.ExcludedFile))) + testCase.verifyFalse(isfile(fullfile(targetFolder, "private", "mustBeValidUrl.m"))) + testCase.verifyTrue(isfile(fullfile(targetFolder, "upload.m"))) + end + + function testExcludedFileMissingFromSourceErrors(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + ExcludeFiles="renamedUpstream.m"), ... + "MatBox:VendorPackage:ExcludedFileNotFound") + testCase.verifyFalse(isfolder(targetFolder)) + end + + function testLicenseFileIsCopied(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + testCase.verifyEqual( ... + string(fileread(fullfile(targetFolder, "LICENSE"))), testCase.LicenseText) + end + + function testLicenseFileInSubfolderIsCopiedByName(testCase) + repositoryFolder = testCase.createRepository(); + writeText(fullfile(repositoryFolder, "licenses", "MIT.txt"), "MIT license text") + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + LicenseFile="licenses/MIT.txt") + + testCase.verifyEqual( ... + string(fileread(fullfile(targetFolder, "MIT.txt"))), "MIT license text") + end + + function testVendorInfoIsWritten(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + returnedInfo = matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + ExcludeFiles=testCase.ExcludedFile, ... + RepositoryUrl="https://github.com/owner/webprogress", ... + Reference="main", ... + CommitID="0123abc"); + + expectedInfo = struct( ... + "Source", 'https://github.com/owner/webprogress', ... + "Reference", 'main', ... + "CommitID", '0123abc', ... + "SourceFolder", char(testCase.SourceFolder), ... + "SourceNamespace", char(testCase.SourceNamespace), ... + "TargetNamespace", char(testCase.TargetNamespace), ... + "ExcludedFiles", {{char(testCase.ExcludedFile)}}); + savedInfo = jsondecode(fileread(fullfile(targetFolder, "vendorinfo.json"))); + testCase.verifyEqual(savedInfo, expectedInfo) + testCase.verifyEqual(returnedInfo.CommitID, "0123abc") + testCase.verifyEqual(returnedInfo.ExcludedFiles, testCase.ExcludedFile) + end + + function testSecondCallGivesIdenticalFolder(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + [firstPaths, firstContents] = readFolder(targetFolder); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + [secondPaths, secondContents] = readFolder(targetFolder); + + testCase.verifyEqual(secondPaths, firstPaths) + testCase.verifyEqual(secondContents, firstContents) + testCase.verifyFalse(contains( ... + fileread(fullfile(targetFolder, "upload.m")), "external.mytoolbox")) + end + + function testFileRemovedFromSourceIsRemovedFromTarget(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + delete(fullfile(repositoryFolder, testCase.SourceFolder, "toolboxdir.m")) + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + testCase.verifyFalse(isfile(fullfile(targetFolder, "toolboxdir.m"))) + testCase.verifyTrue(isfile(fullfile(targetFolder, "upload.m"))) + end + + function testFolderWithoutVendorInfoIsNotReplaced(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + ownFile = fullfile(targetFolder, "ownCode.m"); + writeText(ownFile, "x = 1;") + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder), ... + "MatBox:VendorPackage:TargetFolderNotVendored") + testCase.verifyTrue(isfile(ownFile)) + end + + function testErrorLeavesExistingCopyUnchanged(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + [pathsBefore, contentsBefore] = readFolder(targetFolder); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + ExcludeFiles="renamedUpstream.m"), ... + "MatBox:VendorPackage:ExcludedFileNotFound") + + [pathsAfter, contentsAfter] = readFolder(targetFolder); + testCase.verifyEqual(pathsAfter, pathsBefore) + testCase.verifyEqual(contentsAfter, contentsBefore) + end + + function testSameNamespaceCopiesFilesUnchanged(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = fullfile(pwd, "project", "external", "+webprogress"); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + sourceFile = fullfile(repositoryFolder, testCase.SourceFolder, "upload.m"); + testCase.verifyEqual(fileread(fullfile(targetFolder, "upload.m")), ... + fileread(sourceFile)) + end + + function testOrdinaryTargetFolderErrors(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = fullfile(pwd, "project", "external", "webprogress"); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder), ... + "MatBox:VendorPackage:NamespaceMismatch") + end + + function testMissingSourceFolderErrors(testCase) + repositoryFolder = testCase.createRepository(); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, "src/+otherName", testCase.getTargetFolder()), ... + "MatBox:VendorPackage:SourceFolderNotFound") + end + + function testMissingLicenseFileErrors(testCase) + repositoryFolder = testCase.createRepository(); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, testCase.getTargetFolder(), ... + LicenseFile="license.txt"), ... + "MatBox:VendorPackage:LicenseFileNotFound") + end + + function testBytesOutsideReferencesArePreserved(testCase) + % The comment holds multi-byte UTF-8 characters and the file has + % Windows line endings; both must come through byte for byte. + repositoryFolder = testCase.createRepository(); + sourceText = "% µm – ok" + char([13 10]) + "webprogress.upload(a)" + char([13 10]); + writeText(fullfile(repositoryFolder, testCase.SourceFolder, "scale.m"), sourceText) + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + expectedText = replace(sourceText, "webprogress.", testCase.TargetNamespace + "."); + testCase.verifyEqual(readBytes(fullfile(targetFolder, "scale.m")), ... + unicode2native(expectedText, "UTF-8")) + end + end + + methods (Test) % vendorPackage + function testTargetFolderOutsideProjectErrors(testCase) + testCase.verifyError( ... + @() matbox.tasks.vendorPackage(pwd, testCase.GitHubSourceUri, ... + "code/+matbox/+utility", "../+mytoolbox/+utility"), ... + "MatBox:VendorPackage:InvalidTargetFolder") + end + + function testVendorFromGitHub(testCase) + % Requires network access. + targetFolder = "src/+mytoolbox/+external/+utility"; + + vendorInfo = matbox.tasks.vendorPackage(pwd, testCase.GitHubSourceUri, ... + "code/+matbox/+utility", targetFolder, ... + ExcludeFiles="filewrite.m", Verbose=false); + + testCase.verifyEqual(vendorInfo.Source, "https://github.com/ehennestad/matbox") + testCase.verifyEqual(vendorInfo.Reference, "v0.11.0") + testCase.verifyEqual(vendorInfo.CommitID, testCase.GitHubCommitId) + testCase.verifyTrue(isfile(fullfile(pwd, targetFolder, "LICENSE"))) + testCase.verifyTrue(isfile(fullfile(pwd, targetFolder, "vendorinfo.json"))) + testCase.verifyFalse(isfile(fullfile(pwd, targetFolder, "filewrite.m"))) + testCase.verifySubstring( ... + fileread(fullfile(pwd, targetFolder, "getLatestMatlabReleaseForGitHubActions.m")), ... + "mytoolbox.external.utility.getLatestMatlabReleaseForGitHubActions()") + end + end + + methods (Access = private) + function repositoryFolder = createRepository(testCase) + % Create the files of a repository whose package is in the + % namespace SourceNamespace, as an unzipped download would be. + repositoryFolder = string(fullfile(pwd, "repository")); + packageFolder = fullfile(repositoryFolder, testCase.SourceFolder); + + uploadCode = join([ + "function upload(filePath, url)" + "%upload - Upload a file" + "% webprogress.upload(FILENAME,URL) uploads the file." + " monitorFcn = @(o) webprogress.Monitor(o{:});" + " error(""webprogress:upload:Failed"", ""Upload failed."")" + "end" + ], newline); + + writeText(fullfile(repositoryFolder, "LICENSE"), testCase.LicenseText) + writeText(fullfile(packageFolder, "upload.m"), uploadCode) + writeText(fullfile(packageFolder, testCase.ExcludedFile), "function toolboxdir(), end") + writeText(fullfile(packageFolder, "private", "mustBeValidUrl.m"), ... + "function mustBeValidUrl(~), end") + end + + function targetFolder = getTargetFolder(~) + % Full path of a folder that defines the namespace TargetNamespace + targetFolder = string(fullfile(pwd, "project", "src", ... + "+mytoolbox", "+external", "+webprogress")); + end + end +end + +function writeText(filePath, text) +% writeText - Write text to a UTF-8 file, creating its folder if needed + folderPath = fileparts(filePath); + if ~isfolder(folderPath) + mkdir(folderPath) + end + fileId = fopen(filePath, "w"); + fileCleanup = onCleanup(@() fclose(fileId)); + fwrite(fileId, unicode2native(text, "UTF-8"), "uint8"); +end + +function bytes = readBytes(filePath) +% readBytes - Read a file as a row vector of bytes + fileId = fopen(filePath, "r"); + fileCleanup = onCleanup(@() fclose(fileId)); + bytes = fread(fileId, [1, Inf], "*uint8"); +end + +function [relativePaths, contents] = readFolder(folderPath) +% readFolder - Get the relative paths and the bytes of all files below a folder + fileListing = dir(fullfile(folderPath, "**", "*")); + fileListing([fileListing.isdir]) = []; + filePaths = sort(string(fullfile({fileListing.folder}, {fileListing.name}))); + + relativePaths = erase(filePaths, string(folderPath) + filesep); + contents = cell(size(filePaths)); + for i = 1:numel(filePaths) + contents{i} = readBytes(filePaths(i)); + end +end From c0bf11e4db48b5c96981848a17fb855630818bc3 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 15 Oct 2026 22:10:05 +0200 Subject: [PATCH 2/3] fix: keep the vendored copy on a failed replace and check its source vendorPackageFromFolder deleted the target folder before moving the new copy in, and its cleanup then removed the staged copy, so an error in either step left no copy at all. The new copy is now first moved next to the target, which is the step that can copy between volumes. The old folder is then renamed aside, the new one renamed into place, and the old one deleted. If the second rename fails, the old folder is moved back. The guard against replacing other code only checked that the target had a vendorinfo.json file. It now also requires that file to record the same repository and source folder, so a wrong target path cannot replace a different vendored package. Co-Authored-By: Claude Opus 5.5 --- .../+internal/vendorPackageFromFolder.m | 72 ++++++++++++++--- code/+matbox/+tasks/vendorPackage.m | 3 +- .../+unittest/VendorPackageTest.m | 81 +++++++++++++++++++ 3 files changed, 146 insertions(+), 10 deletions(-) diff --git a/code/+matbox/+tasks/+internal/vendorPackageFromFolder.m b/code/+matbox/+tasks/+internal/vendorPackageFromFolder.m index 8c2efc6..a9d09e0 100644 --- a/code/+matbox/+tasks/+internal/vendorPackageFromFolder.m +++ b/code/+matbox/+tasks/+internal/vendorPackageFromFolder.m @@ -62,6 +62,24 @@ targetFolder, VENDOR_INFO_FILE_NAME) end + % A vendored folder is only replaced by a copy of the same source, so + % that a wrong target path cannot replace another vendored package. + recordedSourceFolder = replace(sourceFolder, "\", "/"); + if isfolder(targetFolder) + previousInfo = jsondecode(native2unicode( ... + readBytes(fullfile(targetFolder, VENDOR_INFO_FILE_NAME)), "UTF-8")); + isSameSource = string(previousInfo.Source) == options.RepositoryUrl ... + && string(previousInfo.SourceFolder) == recordedSourceFolder; + if ~isSameSource + error("MatBox:VendorPackage:TargetFolderHasOtherSource", ... + "The folder ""%s"" holds a copy of ""%s"" from ""%s"", so it " + ... + "is not replaced by a copy of ""%s"" from ""%s"". Choose " + ... + "another target folder, or remove the folder first to switch " + ... + "it to this source.", targetFolder, previousInfo.SourceFolder, ... + previousInfo.Source, recordedSourceFolder, options.RepositoryUrl) + end + end + % The copy is prepared in a temporary folder and replaces the target % folder as the last step, so that an error leaves the target unchanged. stagingFolder = string(tempname()); @@ -81,7 +99,7 @@ "Source", options.RepositoryUrl, ... "Reference", options.Reference, ... "CommitID", options.CommitID, ... - "SourceFolder", replace(sourceFolder, "\", "/"), ... + "SourceFolder", recordedSourceFolder, ... "SourceNamespace", sourceNamespace, ... "TargetNamespace", targetNamespace, ... "ExcludedFiles", options.ExcludeFiles); @@ -89,14 +107,7 @@ % Replacing the whole folder also removes the files that the source no % longer has, which copying over the existing folder would leave behind. - if isfolder(targetFolder) - rmdir(targetFolder, "s") - end - parentFolder = fileparts(targetFolder); - if strlength(parentFolder) > 0 && ~isfolder(parentFolder) - mkdir(parentFolder) - end - movefile(stagingFolder, targetFolder) + replaceFolder(targetFolder, stagingFolder) if nargout < 1 clear vendorInfo @@ -164,6 +175,49 @@ function writeVendorInfo(filePath, vendorInfo) writeBytes(filePath, unicode2native(jsonencode(vendorInfo, PrettyPrint=true), "UTF-8")) end +function replaceFolder(targetFolder, newFolder) +%replaceFolder - Replace a folder with another folder, keeping one complete copy at every step + parentFolder = fileparts(targetFolder); + if strlength(parentFolder) > 0 && ~isfolder(parentFolder) + mkdir(parentFolder) + end + + % The new folder is first moved next to the target. That move can copy + % between volumes and fail partway, but it does not touch the target. + % The moves after it stay within one folder, where a move is a rename. + [~, uniqueName] = fileparts(tempname()); + incomingFolder = fullfile(parentFolder, uniqueName + "_incoming"); + movefile(newFolder, incomingFolder) + incomingFolderCleanup = onCleanup(@() removeFolderIfPresent(incomingFolder)); + + if isfolder(targetFolder) + previousFolder = fullfile(parentFolder, uniqueName + "_previous"); + movefile(targetFolder, previousFolder) + try + movefile(incomingFolder, targetFolder) + catch ME + movefile(previousFolder, targetFolder) + rethrow(ME) + end + removePreviousCopy(previousFolder) + else + movefile(incomingFolder, targetFolder) + end +end + +function removePreviousCopy(previousFolder) +%removePreviousCopy - Delete the replaced copy, or warn if that fails + % The new copy is in place at this point, so a failure to delete the + % old one does not fail the call. + try + rmdir(previousFolder, "s") + catch ME + warning("MatBox:VendorPackage:PreviousCopyNotRemoved", ... + "The previous copy in ""%s"" could not be removed: %s " + ... + "Delete that folder.", previousFolder, ME.message) + end +end + function removeFolderIfPresent(folderPath) %removeFolderIfPresent - Remove a folder and its content if the folder exists if isfolder(folderPath) diff --git a/code/+matbox/+tasks/vendorPackage.m b/code/+matbox/+tasks/vendorPackage.m index 333d86b..38d161f 100644 --- a/code/+matbox/+tasks/vendorPackage.m +++ b/code/+matbox/+tasks/vendorPackage.m @@ -21,7 +21,8 @@ % the commit that was copied, the two namespace names and the excluded % files. Changes made to the copied files are lost on the next call, % because the folder is replaced as a whole. A folder without a -% vendorinfo.json file is never replaced. +% vendorinfo.json file is never replaced, and neither is a folder whose +% vendorinfo.json file records another repository or source folder. % % vendorInfo = matbox.tasks.vendorPackage(...) returns the information % written to vendorinfo.json as a struct. diff --git a/tools/tests/+matboxtools/+unittest/VendorPackageTest.m b/tools/tests/+matboxtools/+unittest/VendorPackageTest.m index 6efebbd..607825f 100644 --- a/tools/tests/+matboxtools/+unittest/VendorPackageTest.m +++ b/tools/tests/+matboxtools/+unittest/VendorPackageTest.m @@ -263,6 +263,87 @@ function testErrorLeavesExistingCopyUnchanged(testCase) testCase.verifyEqual(contentsAfter, contentsBefore) end + function testFailedMoveLeavesExistingCopyUnchanged(testCase) + % A parent folder without write permission makes the move of the + % new copy into it fail, after the copy has been prepared. + testCase.assumeFalse(ispc, ... + "A read-only folder on Windows still accepts new files.") + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + [pathsBefore, contentsBefore] = readFolder(targetFolder); + + parentFolder = fileparts(targetFolder); + fileattrib(parentFolder, "-w") + testCase.addTeardown(@() fileattrib(parentFolder, "+w")) + testCase.assumeFalse(mkdir(fullfile(parentFolder, "writeProbe")), ... + "The test user can write to folders without write permission.") + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder), ... + ?MException) + + [pathsAfter, contentsAfter] = readFolder(targetFolder); + testCase.verifyEqual(pathsAfter, pathsBefore) + testCase.verifyEqual(contentsAfter, contentsBefore) + end + + function testReplacementLeavesNoTemporaryFolders(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + + [parentFolder, targetName] = fileparts(targetFolder); + folderListing = dir(parentFolder); + folderNames = setdiff(string({folderListing.name}), [".", ".."]); + testCase.verifyEqual(folderNames, targetName) + end + + function testFolderFromOtherRepositoryIsNotReplaced(testCase) + repositoryFolder = testCase.createRepository(); + targetFolder = testCase.getTargetFolder(); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + RepositoryUrl="https://github.com/owner/webprogress") + [pathsBefore, contentsBefore] = readFolder(targetFolder); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder, ... + RepositoryUrl="https://github.com/owner/otherpackage"), ... + "MatBox:VendorPackage:TargetFolderHasOtherSource") + + [pathsAfter, contentsAfter] = readFolder(targetFolder); + testCase.verifyEqual(pathsAfter, pathsBefore) + testCase.verifyEqual(contentsAfter, contentsBefore) + end + + function testFolderFromOtherSourceFolderIsNotReplaced(testCase) + repositoryFolder = testCase.createRepository(); + otherSourceFolder = "legacy/+webprogress"; + copyfile(fullfile(repositoryFolder, testCase.SourceFolder), ... + fullfile(repositoryFolder, otherSourceFolder)) + targetFolder = testCase.getTargetFolder(); + matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, testCase.SourceFolder, targetFolder) + [pathsBefore, contentsBefore] = readFolder(targetFolder); + + testCase.verifyError( ... + @() matbox.tasks.internal.vendorPackageFromFolder( ... + repositoryFolder, otherSourceFolder, targetFolder), ... + "MatBox:VendorPackage:TargetFolderHasOtherSource") + + [pathsAfter, contentsAfter] = readFolder(targetFolder); + testCase.verifyEqual(pathsAfter, pathsBefore) + testCase.verifyEqual(contentsAfter, contentsBefore) + end + function testSameNamespaceCopiesFilesUnchanged(testCase) repositoryFolder = testCase.createRepository(); targetFolder = fullfile(pwd, "project", "external", "+webprogress"); From f6e2b7e008de3b8b0afcd551cb82f5deb397f4f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:17:54 +0000 Subject: [PATCH 3/3] Update GitHub badges [skip ci] --- .github/badges/tests.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/badges/tests.svg b/.github/badges/tests.svg index 7199f22..828f2d7 100644 --- a/.github/badges/tests.svg +++ b/.github/badges/tests.svg @@ -1 +1 @@ -tests: 37 passedtests37 passed +tests: 83 passedtests83 passed