feat: warn about namespace references that vendorPackage cannot rename

code/+matbox/+tasks/+internal/findNamespaceReferences.m

@@ -0,0 +1,119 @@
+function locations = findNamespaceReferences(folderPath, namespaceName)
+%findNamespaceReferences - Find the places where the files below a folder name a namespace
+%   locations = matbox.tasks.internal.findNamespaceReferences(folderPath,namespaceName)
+%   searches the files below folderPath for namespaceName as a whole word
+%   and returns a string column vector with one element per place found.
+%   For a text file, an element is the path of the file relative to
+%   folderPath, with "/" as separator, followed by ":" and the line
+%   number, such as "upload.m:12". For a binary file, it is the path
+%   alone, such as "examples/demo.mlx".
+%
+%   The name is not counted when a letter, digit, "_", ".", "/" or "\"
+%   comes before it, as in matbox.tasks.internal.renameNamespaceInText, or
+%   when a letter, digit, "_" or ":" comes after it. This leaves out longer
+%   names, renamed references ("mytoolbox.external.webprogress.upload"),
+%   paths and error identifiers ("webprogress:upload:Failed"). Lines of .m
+%   files that are comments are not searched.
+%
+%   .mlx and .mlapp files are zip archives and are searched in the files
+%   they hold. A file with a NUL byte is binary and is searched as it is
+%   stored, so a name in compressed data, such as a MAT-file saved in the
+%   default format, is not found.
+%
+%   See also matbox.tasks.vendorPackage, matbox.tasks.internal.renameNamespaceInText
+
+    arguments
+        folderPath (1,1) string {mustBeFolder}
+        namespaceName (1,1) string {mustBeNonzeroLengthText}
+    end
+
+    ARCHIVE_EXTENSIONS = [".mlx", ".mlapp"];
+    CODE_EXTENSION = ".m";
+
+    % The character classes are spelled out because \w may also match
+    % non-ASCII letters, and the text is the raw bytes of a file.
+    notInsideLongerName = "(?<![A-Za-z0-9_.\\/])";
+    notFollowedByNameOrColon = "(?![A-Za-z0-9_:])";
+    pattern = notInsideLongerName ...
+        + regexptranslate("escape", namespaceName) + notFollowedByNameOrColon;
+
+    fileListing = dir(fullfile(folderPath, "**", "*"));
+    fileListing([fileListing.isdir]) = [];
+    filePaths = sort(string(fullfile({fileListing.folder}, {fileListing.name})));
+
+    % dir gives full paths with symbolic links resolved, so the root folder
+    % is taken from dir as well for the file paths to start with it.
+    rootListing = dir(folderPath);
+    rootFolder = string(rootListing(1).folder);
+    relativePaths = replace(extractAfter(filePaths, strlength(rootFolder) + 1), "\", "/");
+
+    locationsPerFile = cell(numel(filePaths), 1);
+    for i = 1:numel(filePaths)
+        relativePath = relativePaths(i);
+        [~, ~, fileExtension] = fileparts(filePaths(i));
+
+        if any(strcmpi(fileExtension, ARCHIVE_EXTENSIONS))
+            isNamed = isNameInArchive(filePaths(i), pattern);
+            locationsPerFile{i} = relativePath(isNamed);
+        else
+            % The file is handled as bytes, one character per byte, because
+            % its encoding is unknown. The namespace name is ASCII, which
+            % every common encoding stores as single bytes.
+            bytes = readBytes(filePaths(i));
+            if any(bytes == 0)
+                isNamed = ~isempty(regexp(char(bytes), pattern, "once"));
+                locationsPerFile{i} = relativePath(isNamed);
+            else
+                isCodeFile = strcmpi(fileExtension, CODE_EXTENSION);
+                lineNumbers = findLinesWithName(char(bytes), pattern, isCodeFile);
+                locationsPerFile{i} = relativePath + ":" + lineNumbers;
+            end
+        end
+    end
+
+    locations = vertcat(strings(0, 1), locationsPerFile{:});
+end
+
+function lineNumbers = findLinesWithName(text, pattern, isCodeFile)
+%findLinesWithName - Get the numbers of the lines of a text that match a pattern
+    % Splitting at the line feed alone also handles Windows line endings,
+    % because the carriage return left at the end of a line cannot be part
+    % of a match.
+    textLines = split(string(text), newline);
+    isNamed = contains(textLines, regexpPattern(pattern));
+
+    % A comment line cannot refer to a namespace at run time, and help text
+    % often names the package it belongs to.
+    if isCodeFile
+        isCommentLine = startsWith(strip(textLines, "left"), "%");
+        isNamed = isNamed & ~isCommentLine;
+    end
+
+    lineNumbers = find(isNamed);
+end
+
+function isNamed = isNameInArchive(filePath, pattern)
+%isNameInArchive - Check whether a file in a zip archive matches a pattern
+    extractionFolder = string(tempname());
+    mkdir(extractionFolder)
+    extractionFolderCleanup = onCleanup(@() rmdir(extractionFolder, "s"));
+
+    extractedPaths = string(unzip(filePath, extractionFolder));
+    extractedPaths = extractedPaths(isfile(extractedPaths));
+
+    isNamed = false;
+    for i = 1:numel(extractedPaths)
+        bytes = readBytes(extractedPaths(i));
+        if ~isempty(regexp(char(bytes), pattern, "once"))
+            isNamed = true;
+            break
+        end
+    end
+end
+
+function bytes = readBytes(filePath)
+%readBytes - Read a file as a row vector of bytes
+    fileId = fopen(filePath, "r");
+    fileCleanup = onCleanup(@() fclose(fileId));
+    bytes = fread(fileId, [1, Inf], "*uint8");
+end

code/+matbox/+tasks/+internal/vendorPackageFromFolder.m

@@ -88,8 +88,14 @@
 
     removeExcludedFiles(stagingFolder, options.ExcludeFiles)
 
+    % The search runs before the license file and vendorinfo.json are added,
+    % because both can name the source namespace without referring to it.
     if isRenameNeeded
         renameNamespaceInFolder(stagingFolder, sourceNamespace, targetNamespace)
+        unrenamedLocations = matbox.tasks.internal.findNamespaceReferences( ...
+            stagingFolder, sourceNamespace);
+    else
+        unrenamedLocations = strings(0, 1);
     end
 
     [~, licenseName, licenseExtension] = fileparts(options.LicenseFile);
@@ -109,6 +115,17 @@
     % longer has, which copying over the existing folder would leave behind.
     replaceFolder(targetFolder, stagingFolder)
 
+    % The warning comes after the replacement, so that it is only given for
+    % a copy that is in place.
+    if ~isempty(unrenamedLocations)
+        warning("MatBox:VendorPackage:UnrenamedReferences", ...
+            "References to the namespace ""%s"" in ""%s"" were not renamed " + ...
+            "to ""%s"", so they still resolve to ""%s"". Change them in the " + ...
+            "source repository, or check that they are not namespace " + ...
+            "references:\n%s", sourceNamespace, targetFolder, targetNamespace, ...
+            sourceNamespace, strjoin("    " + unrenamedLocations, newline))
+    end
+
     if nargout < 1
         clear vendorInfo
     end

code/+matbox/+tasks/vendorPackage.m

@@ -16,6 +16,14 @@
 %   "webprogress.upload" into "mytoolbox.external.webprogress.upload" in
 %   code, strings and comments. Error identifiers keep the source name.
 %
+%   A namespace name that code builds at run time, such as
+%   "webprogress." + name, is not renamed, and neither is a reference in a
+%   file other than a .m file, such as an .mlx file or
+%   functionSignatures.json. These references still resolve to the source
+%   namespace. The warning MatBox:VendorPackage:UnrenamedReferences lists
+%   the places in the copied files that still name the source namespace,
+%   leaving out comment lines of .m files.
+%
 %   The license file of the repository is copied into targetFolder, and
 %   the file vendorinfo.json there records the repository, the reference,
 %   the commit that was copied, the two namespace names and the excluded

tools/tests/+matboxtools/+unittest/VendorPackageTest.m

@@ -46,6 +46,23 @@
             "endOfSentence", "% is in the namespace webprogress. It has", ...
             "alreadyRenamed", "mytoolbox.external.webprogress.upload(a)")
 
+        % A line of code that refers to SourceNamespace in a way that
+        % renameNamespaceInText cannot rename
+        unrenamedReferenceCase = struct( ...
+            "stringConcatenation", "className = ""webprogress."" + name;", ...
+            "fevalCall", "feval(""webprogress."" + functionName, a)", ...
+            "namespaceQuery", "namespaceInfo = meta.package.fromName(""webprogress"");", ...
+            "classNameCheck", "tf = startsWith(class(obj), ""webprogress."");")
+
+        % A line of code that names SourceNamespace in a place that is not
+        % an unrenamed reference
+        ignoredLineCase = struct( ...
+            "renamedReference", "x = mytoolbox.external.webprogress.upload(a);", ...
+            "errorIdentifier", "error(""webprogress:upload:Failed"", ""Failed."")", ...
+            "commentLine", "    % Part of the webprogress toolbox", ...
+            "longerNamespaceName", "webprogresstools.projectdir()", ...
+            "folderPath", "folderPath = fullfile(root, ""src/webprogress"");")
+
         % A folder path followed by the name of the namespace it defines
         namespaceCase = struct( ...
             "singleFolder", ["src/webprogress/+webprogress", "webprogress"], ...
@@ -110,6 +127,82 @@ function testNamespaceNameOfFolder(testCase, namespaceCase)
         end
     end
 
+    methods (Test) % findNamespaceReferences
+        function testUnrenamedReferenceIsFound(testCase, unrenamedReferenceCase)
+            folderPath = fullfile(pwd, "package");
+            writeText(fullfile(folderPath, "code.m"), ...
+                join(["function code(name)", unrenamedReferenceCase, "end"], newline))
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, "code.m:2")
+        end
+
+        function testIgnoredLineIsNotFound(testCase, ignoredLineCase)
+            folderPath = fullfile(pwd, "package");
+            writeText(fullfile(folderPath, "code.m"), ...
+                join(["function code(name)", ignoredLineCase, "end"], newline))
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, strings(0, 1))
+        end
+
+        function testNameInJsonFileIsFoundByLine(testCase)
+            folderPath = fullfile(pwd, "package");
+            writeText(fullfile(folderPath, "resources", "functionSignatures.json"), ...
+                testCase.createFunctionSignatures())
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, "resources/functionSignatures.json:3")
+        end
+
+        function testPlacesInSeveralFilesAreListedInOrder(testCase)
+            folderPath = fullfile(pwd, "package");
+            writeText(fullfile(folderPath, "b.m"), ...
+                join(["x = 1;", "f = ""webprogress."" + a;", "g = ""webprogress."" + b;"], newline))
+            writeText(fullfile(folderPath, "a.m"), "x = 1;")
+            writeText(fullfile(folderPath, "sub", "c.json"), """webprogress.upload""")
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, ["b.m:2"; "b.m:3"; "sub/c.json:1"])
+        end
+
+        function testNameInLiveScriptIsFoundByFileName(testCase)
+            % A live script is a zip archive that holds its code in the file
+            % matlab/document.xml.
+            contentFolder = fullfile(pwd, "liveScriptContent");
+            writeText(fullfile(contentFolder, "matlab", "document.xml"), ...
+                "<w:t>x = webprogress.upload(a);</w:t>")
+            zip(fullfile(pwd, "demo.zip"), "matlab", contentFolder)
+            folderPath = fullfile(pwd, "package");
+            mkdir(fullfile(folderPath, "examples"))
+            movefile(fullfile(pwd, "demo.zip"), fullfile(folderPath, "examples", "demo.mlx"))
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, "examples/demo.mlx")
+        end
+
+        function testNameInBinaryFileIsFoundByFileName(testCase)
+            folderPath = fullfile(pwd, "package");
+            writeText(fullfile(folderPath, "data.bin"), ...
+                char(0) + "webprogress.upload" + char(0))
+
+            locations = matbox.tasks.internal.findNamespaceReferences( ...
+                folderPath, testCase.SourceNamespace);
+
+            testCase.verifyEqual(locations, "data.bin")
+        end
+    end
+
     methods (Test) % vendorPackageFromFolder
         function testFilesAreCopiedAndRenamed(testCase)
             repositoryFolder = testCase.createRepository();
@@ -400,6 +493,52 @@ function testBytesOutsideReferencesArePreserved(testCase)
             testCase.verifyEqual(readBytes(fullfile(targetFolder, "scale.m")), ...
                 unicode2native(expectedText, "UTF-8"))
         end
+
+        function testStringBuiltReferenceWarns(testCase)
+            repositoryFolder = testCase.createRepository();
+            writeText(fullfile(repositoryFolder, testCase.SourceFolder, "createMonitor.m"), ...
+                join([
+                    "function monitor = createMonitor(name)"
+                    "    monitor = feval(""webprogress."" + name);"
+                    "end"
+                    ], newline))
+            targetFolder = testCase.getTargetFolder();
+
+            testCase.verifyWarning( ...
+                @() matbox.tasks.internal.vendorPackageFromFolder( ...
+                    repositoryFolder, testCase.SourceFolder, targetFolder), ...
+                "MatBox:VendorPackage:UnrenamedReferences")
+
+            testCase.verifySubstring(lastwarn(), "createMonitor.m:2")
+            testCase.verifyTrue(isfile(fullfile(targetFolder, "createMonitor.m")))
+        end
+
+        function testFunctionSignaturesReferenceWarns(testCase)
+            repositoryFolder = testCase.createRepository();
+            writeText(fullfile(repositoryFolder, testCase.SourceFolder, ...
+                "resources", "functionSignatures.json"), testCase.createFunctionSignatures())
+            targetFolder = testCase.getTargetFolder();
+
+            testCase.verifyWarning( ...
+                @() matbox.tasks.internal.vendorPackageFromFolder( ...
+                    repositoryFolder, testCase.SourceFolder, targetFolder), ...
+                "MatBox:VendorPackage:UnrenamedReferences")
+
+            testCase.verifySubstring(lastwarn(), "resources/functionSignatures.json:3")
+        end
+
+        function testCopyWithoutUnrenamedReferencesDoesNotWarn(testCase)
+            % The license file and vendorinfo.json name the source namespace
+            % without referring to it, and are not searched.
+            repositoryFolder = testCase.createRepository();
+            writeText(fullfile(repositoryFolder, "LICENSE"), ...
+                "Copyright (c) the authors of webprogress")
+            targetFolder = testCase.getTargetFolder();
+
+            testCase.verifyWarningFree( ...
+                @() matbox.tasks.internal.vendorPackageFromFolder( ...
+                    repositoryFolder, testCase.SourceFolder, targetFolder))
+        end
     end
 
     methods (Test) % vendorPackage
@@ -453,6 +592,19 @@ function testVendorFromGitHub(testCase)
                 "function mustBeValidUrl(~), end")
         end
 
+        function signaturesText = createFunctionSignatures(~)
+            % Text of a functionSignatures.json file that names a function
+            % of SourceNamespace on line 3
+            signaturesText = join([
+                "{"
+                "  ""_schemaVersion"": ""1.0.0"","
+                "  ""webprogress.upload"": {"
+                "    ""inputs"": []"
+                "  }"
+                "}"
+                ], newline);
+        end
+
         function targetFolder = getTargetFolder(~)
             % Full path of a folder that defines the namespace TargetNamespace
             targetFolder = string(fullfile(pwd, "project", "src", ...