Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,10 @@ jobs:
if: needs.build-impact.outputs.frontend_required != 'false'
run: pnpm --dir src/mobile-web run type-check

- name: Validate mobile-web build cache contract
if: needs.build-impact.outputs.frontend_required != 'false'
run: node --test scripts/mobile-web-build.test.mjs

- name: Test mobile web account login and reload
if: needs.build-impact.outputs.frontend_required != 'false'
run: pnpm --dir src/mobile-web run test:account-login
Expand Down
125 changes: 118 additions & 7 deletions scripts/check-build-prereqs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
*
* - Root node_modules missing → pnpm scripts fail with "node_modules missing,
* did you mean to install?"
* - A package directory inside node_modules/.pnpm unexpectedly empty (or a
* dangling link) → the matching node_modules/.bin shim resolves to a path
* without content, and nested tools fail with a confusing "Cannot find
* module .../bin/<tool>" error, for example
* ".../design-system/packages/ui/node_modules/typescript/bin/tsc"
* - src/mobile-web/dist missing → cargo check -p openbitfun-desktop and
* cargo check --workspace fail with "resource path '../../mobile-web/dist'
* doesn't exist" in the openbitfun-desktop build script
Expand All @@ -24,8 +29,8 @@
*/

import { execFileSync } from 'node:child_process';
import { existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { existsSync, readdirSync, rmSync } from 'node:fs';
import { join, dirname, relative, sep } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
Expand All @@ -36,6 +41,69 @@ const FIX = process.argv.includes('--fix');

// --- Check logic (extracted for re-use and testing) ---

const VIRTUAL_STORE_SAMPLE_LIMIT = 5;

function readdirWithTypes(dir) {
try {
return readdirSync(dir, { withFileTypes: true });
} catch {
return [];
}
}

/**
* Package directories inside node_modules/.pnpm that pnpm recorded but whose
* files are gone: an existing but empty package directory, or a link whose
* target disappeared. Either one makes the workspace bin shims resolve to a
* path without content.
*/
function findBrokenVirtualStorePackages(rootDir) {
const virtualStoreDir = join(rootDir, 'node_modules', '.pnpm');
if (!existsSync(virtualStoreDir)) {
return [];
}

const isBroken = (packageDir, entry) => {
// readdir reports the name of a dangling link, but the path cannot be opened.
if (!existsSync(packageDir)) {
return true;
}
// Links to peer packages hold no files themselves; their target is checked
// through its own entry in the virtual store.
if (entry.isSymbolicLink()) {
return false;
}
return readdirWithTypes(packageDir).length === 0;
};

const broken = [];
for (const storeEntry of readdirWithTypes(virtualStoreDir)) {
if (!storeEntry.isDirectory()) {
continue;
}
const storeNodeModules = join(virtualStoreDir, storeEntry.name, 'node_modules');

for (const entry of readdirWithTypes(storeNodeModules)) {
if (entry.name.startsWith('@')) {
const scopeDir = join(storeNodeModules, entry.name);
for (const scopedEntry of readdirWithTypes(scopeDir)) {
const packageDir = join(scopeDir, scopedEntry.name);
if (isBroken(packageDir, scopedEntry)) {
broken.push(packageDir);
}
}
continue;
}
const packageDir = join(storeNodeModules, entry.name);
if (isBroken(packageDir, entry)) {
broken.push(packageDir);
}
}
}

return broken;
}

function runChecks(rootDir) {
const errors = [];
const warnings = [];
Expand All @@ -49,7 +117,30 @@ function runChecks(rootDir) {
});
}

// --- Check 2: mobile-web dist (required by openbitfun-desktop build script) ---
// --- Check 2: pnpm virtual store integrity ---
const brokenPackages = findBrokenVirtualStorePackages(rootDir);
if (brokenPackages.length > 0) {
const samples = brokenPackages
.slice(0, VIRTUAL_STORE_SAMPLE_LIMIT)
.map((packageDir) => relative(rootDir, packageDir));
const remaining = brokenPackages.length - samples.length;
const sampleText = remaining > 0 ? `${samples.join(', ')}, ...(+${remaining} more)` : samples.join(', ');

errors.push({
name: 'pnpm virtual store',
message:
`${brokenPackages.length} installed package(s) are missing their files: ${sampleText}. ` +
'The matching node_modules/.bin shims point at a path without content, so nested tools fail with ' +
'a confusing "Cannot find module .../bin/<tool>" error. pnpm install alone does not repair this: ' +
'pnpm leaves an existing empty package directory untouched, so the broken directories must be removed first.',
fix: ['pnpm', 'install'],
cleanPaths: brokenPackages,
fixNote:
'--fix removes the broken package directories first, because pnpm install would otherwise leave them empty. To repair by hand, delete the paths above and run pnpm install.',
});
}

// --- Check 3: mobile-web dist (required by openbitfun-desktop build script) ---
if (!existsSync(join(rootDir, 'src', 'mobile-web', 'dist', 'index.html'))) {
errors.push({
name: 'mobile-web dist',
Expand All @@ -59,7 +150,7 @@ function runChecks(rootDir) {
});
}

// --- Check 3: OpenCode extension Host dist (product runtime resource) ---
// --- Check 4: OpenCode extension Host dist (product runtime resource) ---
const pluginHostDist = join(
rootDir,
'src',
Expand All @@ -77,7 +168,7 @@ function runChecks(rootDir) {
});
}

// --- Check 4: sherpa-onnx prebuilt libs ---
// --- Check 5: sherpa-onnx prebuilt libs ---
// sherpa-onnx-sys build.rs auto-detects target/sherpa-onnx-prebuilt/<version>/lib/
// and returns immediately without downloading. Only warn for the first-build
// scenario where no prebuilt cache exists yet.
Expand Down Expand Up @@ -106,7 +197,11 @@ function runChecks(rootDir) {
function collectPendingFixes(errors) {
return errors
.filter((e) => e.fix)
.map((e) => ({ name: e.name, fix: e.fix }));
.map((e) => ({
name: e.name,
fix: e.fix,
cleanPaths: e.cleanPaths ?? [],
}));
}

function reportResults({ errors, warnings }) {
Expand All @@ -117,6 +212,9 @@ function reportResults({ errors, warnings }) {
if (e.fix) {
console.error(` Fix: ${e.fix.join(' ')}`);
}
if (e.fixNote) {
console.error(` Note: ${e.fixNote}`);
}
}
console.error();
}
Expand All @@ -132,7 +230,20 @@ function reportResults({ errors, warnings }) {

function runFixes(pendingFixes, rootDir) {
let allSucceeded = true;
for (const { name, fix } of pendingFixes) {
for (const { fix, cleanPaths } of pendingFixes) {
for (const target of cleanPaths) {
// Only ever delete inside the virtual store: a broken path must not be
// allowed to escape into source or user data.
const virtualStorePrefix = `${join(rootDir, 'node_modules', '.pnpm')}${sep}`;
if (!target.startsWith(virtualStorePrefix)) {
console.error(`Refusing to remove path outside the pnpm virtual store: ${target}\n`);
allSucceeded = false;
continue;
}
console.log(`Removing broken package directory ${relative(rootDir, target)}`);
rmSync(target, { recursive: true, force: true });
}

const [cmd, ...args] = fix;
console.log(`$ ${fix.join(' ')}`);
try {
Expand Down
118 changes: 115 additions & 3 deletions scripts/check-build-prereqs.test.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
import assert from 'node:assert/strict';
import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import {
chmodSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
Expand All @@ -17,13 +24,26 @@ function createTestRoot({
mobileWebDist = false,
pluginHostDist = false,
sherpaOnnx = null,
virtualStoreEntries = null,
} = {}) {
const root = mkdtempSync(path.join(tmpdir(), 'openbitfun-build-prereqs-'));

if (nodeModules) {
mkdirSync(path.join(root, 'node_modules'), { recursive: true });
}

// Maps a virtual store package path, relative to the root, to the files it
// should contain. An empty file list models a package whose files are gone.
if (virtualStoreEntries) {
for (const [relativeDir, files] of Object.entries(virtualStoreEntries)) {
const packageDir = path.join(root, relativeDir);
mkdirSync(packageDir, { recursive: true });
for (const file of files) {
writeFileSync(path.join(packageDir, file), '{}');
}
}
}

if (mobileWebDist) {
const distDir = path.join(root, 'src', 'mobile-web', 'dist');
mkdirSync(distDir, { recursive: true });
Expand Down Expand Up @@ -65,10 +85,24 @@ function createFakePnpm() {
writeFileSync(
fakePnpmPath,
`
const { mkdirSync, writeFileSync } = require('fs');
const { existsSync, mkdirSync, writeFileSync } = require('fs');
const path = require('path');
const args = process.argv.slice(2);
if (args[0] === 'install') {
mkdirSync('node_modules', { recursive: true });
const restorePath = process.env.FAKE_PNPM_RESTORE_PATH;
if (restorePath) {
// Records whether the broken package was still present when install ran,
// which is what pnpm itself would trip over.
if (process.env.FAKE_PNPM_REPORT_PATH) {
writeFileSync(
process.env.FAKE_PNPM_REPORT_PATH,
JSON.stringify({ dirExistedWhenInstallRan: existsSync(restorePath) }),
);
}
mkdirSync(restorePath, { recursive: true });
writeFileSync(path.join(restorePath, 'package.json'), '{}');
}
} else if (args[0] === 'run' && args[1] === 'prepare:mobile-web') {
mkdirSync('src/mobile-web/dist', { recursive: true });
writeFileSync('src/mobile-web/dist/index.html', '<html></html>');
Expand All @@ -94,7 +128,10 @@ if (args[0] === 'install') {
return binDir;
}

function runCheck(root, { fix = false, extraPath = null, sherpaEnv = null } = {}) {
function runCheck(
root,
{ fix = false, extraPath = null, sherpaEnv = null, extraEnv = null } = {},
) {
const env = {
...process.env,
OPENBITFUN_BUILD_PREREQS_TEST_ROOT: root,
Expand All @@ -109,6 +146,9 @@ function runCheck(root, { fix = false, extraPath = null, sherpaEnv = null } = {}
env.SHERPA_ONNX_LIB_DIR = sherpaEnv;
}
}
if (extraEnv) {
Object.assign(env, extraEnv);
}

const args = fix ? [scriptPath, '--fix'] : [scriptPath];

Expand All @@ -124,6 +164,12 @@ test('passes when all prerequisites are present (including sherpa-onnx prebuilt)
mobileWebDist: true,
pluginHostDist: true,
sherpaOnnx: ['sherpa-onnx-v1.13.4-osx-arm64-static-lib'],
virtualStoreEntries: {
'node_modules/.pnpm/typescript@5.8.3/node_modules/typescript': ['package.json'],
'node_modules/.pnpm/@openbitfun+ui@0.1.0/node_modules/@openbitfun/ui': [
'package.json',
],
},
});
t.after(() => rmSync(root, { recursive: true, force: true }));

Expand All @@ -134,6 +180,72 @@ test('passes when all prerequisites are present (including sherpa-onnx prebuilt)
assert.doesNotMatch(result.stderr, /\[WARN\]/);
});

test('fails when a pnpm virtual store package has no files', (t) => {
const root = createTestRoot({
nodeModules: true,
mobileWebDist: true,
pluginHostDist: true,
sherpaOnnx: ['sherpa-onnx-v1.13.4-osx-arm64-static-lib'],
virtualStoreEntries: {
'node_modules/.pnpm/typescript@5.8.3/node_modules/typescript': [],
},
});
t.after(() => rmSync(root, { recursive: true, force: true }));

const result = runCheck(root, { sherpaEnv: '' });

assert.notEqual(result.status, 0);
assert.match(result.stderr, /\[FAIL\] pnpm virtual store/);
assert.match(
result.stderr,
/node_modules[\\/]\.pnpm[\\/]typescript@5\.8\.3[\\/]node_modules[\\/]typescript/,
);
assert.match(result.stderr, /Fix: pnpm install/);
assert.match(result.stderr, /does not repair this/);
});

test('--fix removes broken virtual store packages before running pnpm install', (t) => {
const binDir = createFakePnpm();
const brokenRelativeDir =
'node_modules/.pnpm/typescript@5.8.3/node_modules/typescript';
const root = createTestRoot({
nodeModules: true,
mobileWebDist: true,
pluginHostDist: true,
sherpaOnnx: ['sherpa-onnx-v1.13.4-osx-arm64-static-lib'],
virtualStoreEntries: { [brokenRelativeDir]: [] },
});
const reportPath = path.join(root, 'fake-pnpm-report.json');
t.after(() => {
rmSync(root, { recursive: true, force: true });
rmSync(binDir, { recursive: true, force: true });
});

const result = runCheck(root, {
fix: true,
extraPath: binDir,
sherpaEnv: '',
extraEnv: {
FAKE_PNPM_RESTORE_PATH: path.join(root, brokenRelativeDir),
FAKE_PNPM_REPORT_PATH: reportPath,
},
});

assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
assert.match(
result.stdout,
/Removing broken package directory node_modules[\\/]\.pnpm[\\/]typescript@5\.8\.3[\\/]node_modules[\\/]typescript/,
);
assert.match(result.stdout, /\$ pnpm install/);
assert.match(result.stdout, /All errors resolved/);
// The empty directory has to be gone before install runs; pnpm does not
// repair a package directory that still exists with no files.
assert.equal(
JSON.parse(readFileSync(reportPath, 'utf8')).dirExistedWhenInstallRan,
false,
);
});

test('fails when root node_modules is missing', (t) => {
const root = createTestRoot({
mobileWebDist: true,
Expand Down
Loading
Loading