diff --git a/src/build/delta.cc b/src/build/delta.cc index f12053595..070998f4b 100644 --- a/src/build/delta.cc +++ b/src/build/delta.cc @@ -7,6 +7,7 @@ #include // std::span #include // std::string #include // std::string_view +#include // std::error_code #include // std::unordered_map #include // std::unordered_set #include // std::vector @@ -532,8 +533,12 @@ auto delta_engine(const BuildPhase phase, const BuildPlan::Type build_type, continue; } + // Probed without throwing, since a global this build cannot examine, + // whatever the reason, is one to produce again rather than trust + std::error_code existence_error; const auto path{output / rule.filename}; - if (entries.contains(path.native()) && !std::filesystem::exists(path)) { + if (entries.contains(path.native()) && + !std::filesystem::exists(path, existence_error)) { missing_globals[index] = true; has_missing_globals = true; } diff --git a/src/index/index.cc b/src/index/index.cc index b959dbd33..e2a8b7714 100644 --- a/src/index/index.cc +++ b/src/index/index.cc @@ -245,6 +245,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com )EOF"}; @@ -829,6 +837,13 @@ auto main(int argc, char *argv[]) noexcept -> int { std::print(stdout, "error: {}\n at path {}\n with count {}\n", error.what(), error.path().string(), error.count()); return EXIT_FAILURE; + } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) { + std::print(stdout, + "error: {}\n at path {}\n\n" + "Something other than the indexer modified the output " + "directory, so delete it and index again from scratch\n", + error.what(), error.path().string()); + return EXIT_FAILURE; } catch (const sourcemeta::one::ResolverNotASchemaError &error) { std::print(stdout, "error: {}\n at path {}\n", error.what(), error.path().string()); diff --git a/src/resolver/include/sourcemeta/one/resolver_error.h b/src/resolver/include/sourcemeta/one/resolver_error.h index f8644f4a9..b0edac929 100644 --- a/src/resolver/include/sourcemeta/one/resolver_error.h +++ b/src/resolver/include/sourcemeta/one/resolver_error.h @@ -26,6 +26,27 @@ class ResolverNotASchemaError : public std::exception { std::filesystem::path path_; }; +// The output directory belongs to the indexer, so a recorded artifact that is +// gone from disk means something else modified the directory, which is the one +// assumption the build cannot recover from on its own +class ResolverMissingCachedArtifactError : public std::exception { +public: + ResolverMissingCachedArtifactError(std::filesystem::path path) + : path_{std::move(path)} {} + + [[nodiscard]] auto what() const noexcept -> const char * override { + return "The build state references an artifact that no longer exists on " + "disk"; + } + + [[nodiscard]] auto path() const noexcept -> const std::filesystem::path & { + return this->path_; + } + +private: + std::filesystem::path path_; +}; + class ResolverOutsideBaseError : public std::exception { public: ResolverOutsideBaseError(std::filesystem::path path, diff --git a/src/resolver/resolver.cc b/src/resolver/resolver.cc index 1a981ab42..a6d02dc5e 100644 --- a/src/resolver/resolver.cc +++ b/src/resolver/resolver.cc @@ -17,6 +17,7 @@ #include // std::shared_lock #include // std::ostringstream #include // std::string +#include // std::error_code #include // std::unordered_set static auto @@ -532,10 +533,18 @@ auto Resolver::add(const std::filesystem::path &collection_relative_path, auto Resolver::emplace(std::string new_identifier, Entry entry) -> void { assert(std::filesystem::exists(entry.path)); - // As the materialised path must exist if we are emplacing - // given a cache hit assert(entry.cache_path.has_value()); - assert(std::filesystem::exists(entry.cache_path.value())); + // A cache hit means a finished build wrote this artifact, so it being gone + // means the output directory was modified from outside. That violates the + // one assumption the cache rests on, and it has to fail the same way in + // every build type rather than trip an assertion only where those exist. + // The check must not itself throw, since a path the build cannot examine, + // whatever the reason, is equally a record it cannot honour + std::error_code existence_error; + if (!std::filesystem::exists(entry.cache_path.value(), existence_error)) { + throw ResolverMissingCachedArtifactError{entry.cache_path.value()}; + } + assert(entry.collection); const auto path{entry.path}; auto result{this->views.emplace(std::move(new_identifier), std::move(entry))}; diff --git a/test/cli/CMakeLists.txt b/test/cli/CMakeLists.txt index e172978cd..1784115dd 100644 --- a/test/cli/CMakeLists.txt +++ b/test/cli/CMakeLists.txt @@ -59,6 +59,7 @@ if(ONE_INDEX) sourcemeta_one_test_cli(common index fail-option-overflow-value) sourcemeta_one_test_cli(common index fail-output-non-directory) sourcemeta_one_test_cli(common index fail-page-level-extends) + sourcemeta_one_test_cli(common index fail-removed-schema-artifact) sourcemeta_one_test_cli(common index fail-resolve-ref-target-foreign-authority) sourcemeta_one_test_cli(common index fail-resolve-schema-invalid-uri) sourcemeta_one_test_cli(common index fail-resolve-target-chain) diff --git a/test/cli/index/common/fail-no-arguments.sh b/test/cli/index/common/fail-no-arguments.sh index 57b89229a..716f0d0b1 100755 --- a/test/cli/index/common/fail-no-arguments.sh +++ b/test/cli/index/common/fail-no-arguments.sh @@ -35,6 +35,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/common/fail-removed-schema-artifact.sh b/test/cli/index/common/fail-removed-schema-artifact.sh new file mode 100755 index 000000000..8ee37cd7f --- /dev/null +++ b/test/cli/index/common/fail-removed-schema-artifact.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# The build state records that a build materialised a schema, and an +# incremental build trusts it. The output directory belongs to the indexer, so +# a recorded artifact that is gone from disk means something else modified the +# directory, and the build must refuse loudly instead of continuing over a +# record it cannot honour + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/one.json" +{ + "url": "https://example.com/", + "contents": { + "schemas": { "baseUri": "https://example.com/", "path": "./schemas" } + } +} +EOF + +mkdir "$TMP/schemas" + +cat << 'EOF' > "$TMP/schemas/a.json" +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/a" +} +EOF + +"$1" --skip-banner "$TMP/one.json" "$TMP/output" > /dev/null 2>&1 + +rm "$TMP/output/schemas/schemas/a/%/schema.metapack" + +"$1" --skip-banner "$TMP/one.json" "$TMP/output" > "$TMP/output.txt" 2> /dev/null \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +error: The build state references an artifact that no longer exists on disk + at path $(realpath "$TMP")/output/schemas/schemas/a/%/schema.metapack + +Something other than the indexer modified the output directory, so delete it and index again from scratch +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/cli/index/common/output-help-skip-banner.sh b/test/cli/index/common/output-help-skip-banner.sh index 3df41ceee..6423dee87 100755 --- a/test/cli/index/common/output-help-skip-banner.sh +++ b/test/cli/index/common/output-help-skip-banner.sh @@ -32,6 +32,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/common/output-help.sh b/test/cli/index/common/output-help.sh index 4c3a2d050..f8b5b2fcb 100755 --- a/test/cli/index/common/output-help.sh +++ b/test/cli/index/common/output-help.sh @@ -34,6 +34,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/community/fail-no-options.sh b/test/cli/index/community/fail-no-options.sh index 57b89229a..716f0d0b1 100755 --- a/test/cli/index/community/fail-no-options.sh +++ b/test/cli/index/community/fail-no-options.sh @@ -35,6 +35,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/community/fail-no-output.sh b/test/cli/index/community/fail-no-output.sh index 6e0211837..b045fb8c2 100755 --- a/test/cli/index/community/fail-no-output.sh +++ b/test/cli/index/community/fail-no-output.sh @@ -51,6 +51,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/enterprise/fail-no-options.sh b/test/cli/index/enterprise/fail-no-options.sh index 57b89229a..716f0d0b1 100755 --- a/test/cli/index/enterprise/fail-no-options.sh +++ b/test/cli/index/enterprise/fail-no-options.sh @@ -35,6 +35,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/cli/index/enterprise/fail-no-output.sh b/test/cli/index/enterprise/fail-no-output.sh index 6e0211837..b045fb8c2 100755 --- a/test/cli/index/enterprise/fail-no-output.sh +++ b/test/cli/index/enterprise/fail-no-output.sh @@ -51,6 +51,14 @@ Advanced Options: Set the maximum number of direct entries in a directory listing +Output Directory: + + The output directory is owned by the indexer. Do NOT: + + - Manually modify, add, or delete any file inside it + - Sync content into it or partially restore it from a backup + - Run concurrent indexer invocations over the same output + For more documentation, visit https://one.sourcemeta.com EOF diff --git a/test/unit/resolver/resolver_test.cc b/test/unit/resolver/resolver_test.cc index fa41f7b9c..dd90b0534 100644 --- a/test/unit/resolver/resolver_test.cc +++ b/test/unit/resolver/resolver_test.cc @@ -694,3 +694,79 @@ TEST(path_url_absolute_ref) { "$ref": "2020-12-id" })JSON"); } + +TEST(emplace_accepts_a_cache_hit_whose_artifact_is_present) { + sourcemeta::one::Resolver resolver{shared_configuration().url}; + const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" / + "2020-12-with-id.json"}; + resolver.emplace( + "http://localhost:8000/example/2020-12-with-id", + {.path = source_path, + .relative_path = "example/2020-12-with-id", + .mtime = std::filesystem::last_write_time(source_path), + .evaluate = true, + .cache_path = source_path, + .dialect = "https://json-schema.org/draft/2020-12/schema", + .original_identifier = "https://example.com/schemas/2020-12-with-id", + .collection = &std::get( + shared_configuration().entries.at("example"))}); + EXPECT_EQ(resolver.entry("http://localhost:8000/example/2020-12-with-id") + .cache_path.value(), + source_path); +} + +TEST(emplace_refuses_a_cache_hit_whose_artifact_cannot_be_examined) { + sourcemeta::one::Resolver resolver{shared_configuration().url}; + const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" / + "2020-12-with-id.json"}; + // A component longer than any filesystem accepts makes examining the path + // an error rather than a clean answer, in every environment including + // running as root, where permission denials cannot be provoked + const auto unexaminable_path{std::filesystem::path{SCHEMAS_PATH} / + std::string(300, 'x') / "schema.metapack"}; + try { + resolver.emplace( + "http://localhost:8000/example/2020-12-with-id", + {.path = source_path, + .relative_path = "example/2020-12-with-id", + .mtime = std::filesystem::last_write_time(source_path), + .evaluate = true, + .cache_path = unexaminable_path, + .dialect = "https://json-schema.org/draft/2020-12/schema", + .original_identifier = "https://example.com/schemas/2020-12-with-id", + .collection = &std::get( + shared_configuration().entries.at("example"))}); + FAIL(); + } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) { + EXPECT_STREQ( + error.what(), + "The build state references an artifact that no longer exists on disk"); + EXPECT_EQ(error.path(), unexaminable_path); + } +} + +TEST(emplace_refuses_a_cache_hit_whose_artifact_is_gone) { + sourcemeta::one::Resolver resolver{shared_configuration().url}; + const auto source_path{std::filesystem::path{SCHEMAS_PATH} / "example" / + "2020-12-with-id.json"}; + const std::filesystem::path missing_path{"/no/such/cache/schema.metapack"}; + try { + resolver.emplace( + "http://localhost:8000/example/2020-12-with-id", + {.path = source_path, + .relative_path = "example/2020-12-with-id", + .mtime = std::filesystem::last_write_time(source_path), + .evaluate = true, + .cache_path = missing_path, + .dialect = "https://json-schema.org/draft/2020-12/schema", + .original_identifier = "https://example.com/schemas/2020-12-with-id", + .collection = &std::get( + shared_configuration().entries.at("example"))}); + FAIL(); + } catch (const sourcemeta::one::ResolverMissingCachedArtifactError &error) { + EXPECT_STREQ( + error.what(), + "The build state references an artifact that no longer exists on disk"); + EXPECT_EQ(error.path(), missing_path); + } +}