diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f633a9bb..4c531872 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,7 +110,7 @@ If Docker tests skip with "no free 10.88-119.0.0/16 range", earlier runs leaked | --- | --- | | `HC_TEST_PYTHON` | Python interpreter with boto3 (default `python3`); e.g. `HC_TEST_PYTHON=$PWD/.venv/bin/python` | | `HC_TEST_AWS` | path to the `aws` CLI (default: `aws` on `PATH`) | -| `HC_TEST_TERRAFORM` | set to run the Terraform/OpenTofu tests (they download the AWS provider; set `TF_PLUGIN_CACHE_DIR` to reuse it) | +| `HC_TEST_TERRAFORM` | set to run the Terraform/OpenTofu tests: `1` uses `terraform`, then `tofu`, from `PATH`; a path (e.g. `/opt/tofu/tofu`) uses that binary. They download the AWS provider; set `TF_PLUGIN_CACHE_DIR` to reuse it | | `HC_TEST_NODE_PATH` | a `node_modules` directory with `amazon-cognito-identity-js`, for the Cognito SRP test in Node.js | **End to end.** With a server running, `./scripts/smoke.sh` exercises every service through the CLI and diff --git a/cli/internal/awsapi/awstest/awstest.go b/cli/internal/awsapi/awstest/awstest.go index fc66ccd0..47123415 100644 --- a/cli/internal/awsapi/awstest/awstest.go +++ b/cli/internal/awsapi/awstest/awstest.go @@ -11,6 +11,10 @@ // // The AWS CLI is found on PATH (or HC_TEST_AWS); Python with boto3 is // HC_TEST_PYTHON (default python3). Tests skip when a tool is missing. +// +// The Terraform/OpenTofu tests are opt-in because they download the AWS +// provider: set HC_TEST_TERRAFORM to run them, either to "1" (use terraform, +// then tofu, from PATH) or to the path of the binary to use. See Terraform. package awstest import ( @@ -179,6 +183,33 @@ func (h *Harness) Signed(t *testing.T, service, method, path string, header http return resp.StatusCode, resp.Header, out } +// Terraform returns the Terraform or OpenTofu binary for a Terraform test, or +// skips the test. The tests are opt-in: they run only when HC_TEST_TERRAFORM +// is set, and not in -short mode. A value that names an executable file (for +// example /opt/tofu/tofu) is used as the binary; any other value (such as 1) +// looks up terraform, then tofu, on PATH. Set TF_PLUGIN_CACHE_DIR to reuse the +// provider download across runs. +func Terraform(t *testing.T) string { + t.Helper() + v := os.Getenv("HC_TEST_TERRAFORM") + if v == "" { + t.Skip("set HC_TEST_TERRAFORM=1 (or the path to terraform/tofu) to run") + } + if testing.Short() { + t.Skip("terraform test is slow") + } + if fi, err := os.Stat(v); err == nil && fi.Mode().IsRegular() && fi.Mode().Perm()&0o111 != 0 { + return v + } + for _, name := range []string{"terraform", "tofu"} { + if p, err := exec.LookPath(name); err == nil { + return p + } + } + t.Skipf("HC_TEST_TERRAFORM=%q is not an executable, and neither terraform nor tofu is on PATH", v) + return "" +} + func awsBin(t *testing.T) string { if p := os.Getenv("HC_TEST_AWS"); p != "" { return p diff --git a/cli/internal/svc/cfn/terraform_test.go b/cli/internal/svc/cfn/terraform_test.go index ca4047b3..9ee3acb2 100644 --- a/cli/internal/svc/cfn/terraform_test.go +++ b/cli/internal/svc/cfn/terraform_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" ) // TestTerraformCloudFormationStack applies, re-plans, updates and destroys an @@ -13,15 +15,7 @@ import ( // (HC_TEST_TERRAFORM=1) since it downloads the provider; set TF_PLUGIN_CACHE_DIR // to reuse it. func TestTerraformCloudFormationStack(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) e := newEnv(t, false) dir := t.TempDir() cfg := func(retention string) string { diff --git a/cli/internal/svc/iam/terraform_test.go b/cli/internal/svc/iam/terraform_test.go index 05a8f8ae..55a8978d 100644 --- a/cli/internal/svc/iam/terraform_test.go +++ b/cli/internal/svc/iam/terraform_test.go @@ -12,20 +12,10 @@ import ( ) // TestTerraform applies and destroys a Terraform configuration with the AWS -// provider against HomeCloud's IAM and STS. It needs terraform (HC_TEST_TERRAFORM -// or PATH) and network access to download the provider on first use; set -// TF_PLUGIN_CACHE_DIR to reuse the download. +// provider against HomeCloud's IAM and STS. Opt-in (see awstest.Terraform) since it +// downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse the download. func TestTerraform(t *testing.T) { - bin := os.Getenv("HC_TEST_TERRAFORM") - if bin == "" { - var err error - if bin, err = exec.LookPath("terraform"); err != nil { - t.Skip("terraform not installed (set HC_TEST_TERRAFORM)") - } - } - if testing.Short() { - t.Skip("terraform test is slow") - } + bin := awstest.Terraform(t) h := awstest.New(t) dir := t.TempDir() write := func(policyAction string) { diff --git a/cli/internal/svc/route53/terraform_test.go b/cli/internal/svc/route53/terraform_test.go index a4eb15e8..eb5e5821 100644 --- a/cli/internal/svc/route53/terraform_test.go +++ b/cli/internal/svc/route53/terraform_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" "github.com/homecloudhq/homecloud/cli/internal/svc/acm" ) @@ -14,15 +15,7 @@ import ( // and the hashicorp/aws provider. Opt-in (HC_TEST_TERRAFORM=1) since it // downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse it. func TestTerraform(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) h, _ := setup(t) a := acm.New(h.Env, h.Secrets) a.RegisterAWS() diff --git a/cli/internal/svc/s3/terraform_test.go b/cli/internal/svc/s3/terraform_test.go index f65b039e..3864ba6b 100644 --- a/cli/internal/svc/s3/terraform_test.go +++ b/cli/internal/svc/s3/terraform_test.go @@ -6,21 +6,15 @@ import ( "path/filepath" "strings" "testing" + + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" ) // TestTerraform applies, re-plans and destroys S3 resources with Terraform or -// OpenTofu and the hashicorp/aws provider. Opt-in (HC_TEST_TERRAFORM=1) since +// OpenTofu and the hashicorp/aws provider. Opt-in (see awstest.Terraform) since // it downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse it. func TestTerraform(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) h, _ := newHarness(t) b := bucketName() dir := t.TempDir() diff --git a/console/app/(console)/page.tsx b/console/app/(console)/page.tsx index 1b4570ad..48b0d13c 100644 --- a/console/app/(console)/page.tsx +++ b/console/app/(console)/page.tsx @@ -1,5 +1,7 @@ import { ConsoleHome } from "@/components/home/console-home" +export const metadata = { title: "Home" } + export default function HomePage() { return } diff --git a/console/app/layout.tsx b/console/app/layout.tsx index ca4f4298..55e5f158 100644 --- a/console/app/layout.tsx +++ b/console/app/layout.tsx @@ -13,7 +13,7 @@ export const metadata: Metadata = { default: "HomeCloud Console", template: "%s | HomeCloud Console", }, - description: "Manage your self-hosted HomeCloud: EC2, S3, VPC, IAM, Secrets Manager, CloudWatch and CloudTrail.", + description: "The web console for HomeCloud, a self-hosted AWS-compatible cloud.", applicationName: "HomeCloud Console", icons: { icon: `${process.env.NEXT_PUBLIC_BASE_PATH ?? ""}/favicon.svg` }, } diff --git a/console/app/login/layout.tsx b/console/app/login/layout.tsx new file mode 100644 index 00000000..fcc24a61 --- /dev/null +++ b/console/app/login/layout.tsx @@ -0,0 +1,6 @@ +// The sign-in page is a client component, so its tab title is set here. +export const metadata = { title: "Sign in" } + +export default function LoginLayout({ children }: Readonly<{ children: React.ReactNode }>) { + return children +} diff --git a/docs/integrations.md b/docs/integrations.md index 25f60cca..9154a749 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -162,7 +162,9 @@ with types. See [integrations/testcontainers-node/README.md](../integrations/tes All four modules start the image (default `ghcr.io/solinode/homecloud:latest`, configurable) with the Docker socket mounted and **host networking**, and the API on -`127.0.0.1:18080` of the Docker host (configurable; not 8080, which is often taken). +`127.0.0.1:18080` of the Docker host (configurable; not 8080, which is often taken). The +endpoint they hand out names it `http://localhost:18080`, for the reason in +[SDK notes](#sdk-notes). HomeCloud then reaches the containers it starts on the host's loopback ports, and they call it back through `host.docker.internal`, exactly as when it runs on the host. This works on Linux (GitHub Actions included), OrbStack, and Docker Desktop with host diff --git a/integrations/github-action/README.md b/integrations/github-action/README.md index 48a1b542..854b300d 100644 --- a/integrations/github-action/README.md +++ b/integrations/github-action/README.md @@ -13,7 +13,7 @@ an AWS account. The action: -1. downloads the HomeCloud release you ask for and verifies it against the release's `checksums.txt`, +1. downloads the HomeCloud release you ask for, verifies the Sigstore signature on its `checksums.txt` (releases from v0.5.0 on are signed) and checks the download against it, 2. starts `homecloud serve` in the background (data in `$RUNNER_TEMP/homecloud-data`, log in `$RUNNER_TEMP/homecloud.log`), 3. waits for `/api/v1/health` and for any services listed in `services-wait`, 4. exports `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (masked), `AWS_REGION`, `AWS_DEFAULT_REGION` and `HOMECLOUD_DATA_DIR` for the following steps, and puts `homecloud` on `PATH`, @@ -44,6 +44,7 @@ jobs: | `port` | `8080` | Port the API listens on, on `127.0.0.1`. | | `services-wait` | (none) | Comma or newline separated services to wait for. `s3`, `ecr` and `route53` start containers in the background; in-process services (`sqs`, `dynamodb`, `lambda`, ...) are ready with the API; an unknown name fails the step. | | `wait-timeout` | `180` | Seconds to wait for the API and the services. | +| `verify-signature` | `true` | Verify the release signature with cosign before trusting `checksums.txt` (see [Release verification](#release-verification)). `false` checks the SHA-256 checksum only. | ### Outputs @@ -126,6 +127,30 @@ provider "aws" { - run: ./run-tests.sh --endpoint "${{ steps.homecloud.outputs.endpoint }}" ``` +## Release verification + +HomeCloud releases from v0.5.0 on publish `checksums.txt.sigstore.json`, a Sigstore bundle +over `checksums.txt` made by keyless signing in HomeCloud's release workflow. When the bundle +exists, the action runs + +```sh +cosign verify-blob checksums.txt --bundle checksums.txt.sigstore.json \ + --certificate-identity https://github.com/solinode/homecloud/.github/workflows/release.yml@refs/tags/ \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com +``` + +and fails the step if the signature does not verify, or if a release from v0.5.0 on has no +bundle. Only then does it trust the SHA-256 in `checksums.txt` for the archive. `latest` is +resolved to its tag first, so the signer identity names the exact release. + +- cosign comes from the runner's `PATH` if it is there (for example installed with + `sigstore/cosign-installer`; cosign 2.4 or newer), otherwise the action downloads cosign + v3.1.3 from its GitHub release and checks it against a SHA-256 pinned in `setup.sh`. +- Verification fetches Sigstore's trust root over the network. On a runner that cannot + reach Sigstore, set `verify-signature: false` to check the checksum only. +- Releases before v0.5.0 are unsigned: they are checked against `checksums.txt` only, as + before. + ## Notes - One HomeCloud per Docker host: it names its helper containers (`homecloud-s3`, ...) and diff --git a/integrations/github-action/action.yml b/integrations/github-action/action.yml index 6000f3e7..5bcc6cca 100644 --- a/integrations/github-action/action.yml +++ b/integrations/github-action/action.yml @@ -25,6 +25,13 @@ inputs: description: Seconds to wait for the API and every service in services-wait. required: false default: "180" + verify-signature: + description: >- + Verify the release's Sigstore signature (checksums.txt.sigstore.json, published from + v0.5.0 on) with cosign before trusting checksums.txt: "true" or "false". Releases + before v0.5.0 are unsigned and get the checksum check only. + required: false + default: "true" outputs: endpoint: diff --git a/integrations/github-action/main.js b/integrations/github-action/main.js index 9fa77135..85f1246d 100644 --- a/integrations/github-action/main.js +++ b/integrations/github-action/main.js @@ -10,6 +10,7 @@ const env = { HC_PORT: input("port", "8080"), HC_SERVICES_WAIT: input("services-wait", ""), HC_WAIT_TIMEOUT: input("wait-timeout", "180"), + HC_VERIFY_SIGNATURE: input("verify-signature", "true"), }; const r = spawnSync("bash", [path.join(__dirname, "setup.sh")], { stdio: "inherit", env }); if (r.error) { diff --git a/integrations/github-action/setup.sh b/integrations/github-action/setup.sh index 1bd79fe7..85387c6d 100755 --- a/integrations/github-action/setup.sh +++ b/integrations/github-action/setup.sh @@ -25,6 +25,11 @@ port=$((10#$port)) case "$timeout" in '' | *[!0-9]*) fail "wait-timeout must be a whole number of seconds, got '$timeout'" ;; esac timeout=$((10#$timeout)) [ "$timeout" -ge 1 ] || fail "wait-timeout must be at least 1 second" +case "$(echo "${HC_VERIFY_SIGNATURE:-true}" | tr '[:upper:]' '[:lower:]')" in + true) verify_sig=1 ;; + false) verify_sig= ;; + *) fail "verify-signature must be true or false, got '$HC_VERIFY_SIGNATURE'" ;; +esac # Services that start containers in the background log a line when they are ready; # the others run inside the HomeCloud process and are ready with the API. @@ -36,7 +41,7 @@ for svc in $(echo "${HC_SERVICES_WAIT:-}" | tr ',\n' ' ' | tr '[:upper:]' '[:lo *) case "$in_process" in *" $svc "*) ;; *) fail "services-wait: unknown service '$svc' (background: s3, ecr, route53; in-process:$in_process)" ;; esac ;; esac done -[ "${HC_VALIDATE_ONLY:-}" = 1 ] && { echo "inputs OK: port=$port wait-timeout=$timeout waits=${waits[*]:-}"; exit 0; } +[ "${HC_VALIDATE_ONLY:-}" = 1 ] && { echo "inputs OK: port=$port wait-timeout=$timeout waits=${waits[*]:-} verify-signature=${verify_sig:-0}"; exit 0; } command -v docker >/dev/null 2>&1 || fail "HomeCloud needs Docker on the runner (ubuntu-* runners have it)" docker info >/dev/null 2>&1 || fail "Docker is installed but not running" @@ -49,24 +54,91 @@ case "$(uname -m)" in esac name="homecloud-$os-$arch" if [ "$version" = latest ]; then - base="https://github.com/$REPO/releases/latest/download" -else - case "$version" in v*) ;; *) version="v$version" ;; esac - base="https://github.com/$REPO/releases/download/$version" + # Resolve "latest" to its tag: the signature names the tag it was made for, and + # every file then comes from the same release. + version=$(curl -fsSL --retry 3 -o /dev/null -w '%{url_effective}' "https://github.com/$REPO/releases/latest" | sed -n 's#.*/releases/tag/##p') || true + [ -n "$version" ] || fail "could not determine the latest HomeCloud release" fi +case "$version" in v*) ;; *) version="v$version" ;; esac +base="https://github.com/$REPO/releases/download/$version" + +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +# signed_release VERSION: true for v0.5.0 and later, which are all signed. +signed_release() { + local v=${1#v} major rest minor + v=${v%%-*} + major=${v%%.*} + rest=${v#*.} + minor=${rest%%.*} + case "$major$minor" in *[!0-9]* | "") return 1 ;; esac + [ "$major" -gt 0 ] || [ "$minor" -ge 5 ] +} + +# cosign for the signature check: the runner's own (for example from +# sigstore/cosign-installer), else this pinned release, checked against its SHA-256. +COSIGN_VERSION=v3.1.3 +cosign_sha256() { + case "$1" in + linux-amd64) echo 4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 ;; + linux-arm64) echo c5d324e091826b0d7a78eb16fef316450b4eb9aaec045611c08ba06f5e73220a ;; + darwin-amd64) echo 2347488e5d5b25336644024dfeca5601b190e91197a71a917bda44744aff106c ;; + darwin-arm64) echo 5cf948c2f4dfe59687bdd0b8523709067383e03982cc543475c8a7dc70e92a76 ;; + esac +} +find_cosign() { + if cosign=$(command -v cosign); then + return + fi + local dir="$tmp/homecloud-cosign" want got + want=$(cosign_sha256 "$os-$arch") + [ -n "$want" ] || fail "no pinned cosign for $os/$arch: install cosign first, or set verify-signature: false" + mkdir -p "$dir" + curl -fsSL --retry 3 "https://github.com/sigstore/cosign/releases/download/$COSIGN_VERSION/cosign-$os-$arch" -o "$dir/cosign" || + fail "download of cosign $COSIGN_VERSION failed" + got=$(sha256 "$dir/cosign") + [ "$got" = "$want" ] || fail "checksum mismatch for cosign $COSIGN_VERSION (expected $want, got $got)" + chmod +x "$dir/cosign" + cosign="$dir/cosign" +} echo "::group::Install HomeCloud ($version, $os/$arch)" dl="$tmp/homecloud-dl" rm -rf "$dl" && mkdir -p "$dl" "$bin_dir" curl -fsSL --retry 3 "$base/$name.tar.gz" -o "$dl/$name.tar.gz" || fail "download $base/$name.tar.gz failed" curl -fsSL --retry 3 "$base/checksums.txt" -o "$dl/checksums.txt" || fail "download $base/checksums.txt failed" -expected=$(awk -v f="$name.tar.gz" '$2 == f || $2 == "*" f {print $1}' "$dl/checksums.txt") -[ -n "$expected" ] || fail "checksums.txt has no entry for $name.tar.gz" -if command -v sha256sum >/dev/null 2>&1; then - actual=$(sha256sum "$dl/$name.tar.gz" | cut -d' ' -f1) + +# Releases from v0.5.0 on sign checksums.txt with Sigstore keyless signing in the +# release workflow (checksums.txt.sigstore.json). Verify it when it exists, so the +# checksum below is trusted; older releases are unsigned and get the checksum only. +if [ -z "$verify_sig" ]; then + echo "verify-signature is false: checking the SHA-256 checksum only" +elif curl -fsSL --retry 3 "$base/checksums.txt.sigstore.json" -o "$dl/checksums.txt.sigstore.json" 2>/dev/null; then + find_cosign + echo "verifying the release signature with $cosign" + identity="https://github.com/$REPO/.github/workflows/release.yml@refs/tags/$version" + if ! "$cosign" verify-blob "$dl/checksums.txt" --bundle "$dl/checksums.txt.sigstore.json" \ + --certificate-identity "$identity" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com >"$dl/cosign.log" 2>&1; then + cat "$dl/cosign.log" + fail "signature verification of checksums.txt failed for $version (expected signer $identity)" + fi + echo "signature OK (checksums.txt signed by $identity)" +elif signed_release "$version"; then + fail "$version should be signed, but $base/checksums.txt.sigstore.json could not be downloaded (set verify-signature: false to skip the check)" else - actual=$(shasum -a 256 "$dl/$name.tar.gz" | cut -d' ' -f1) + echo "$version is not signed (releases before v0.5.0 were not): checking the SHA-256 checksum only" fi + +expected=$(awk -v f="$name.tar.gz" '$2 == f || $2 == "*" f {print $1}' "$dl/checksums.txt") +[ -n "$expected" ] || fail "checksums.txt has no entry for $name.tar.gz" +actual=$(sha256 "$dl/$name.tar.gz") [ "$expected" = "$actual" ] || fail "checksum mismatch for $name.tar.gz (expected $expected, got $actual)" echo "checksum OK ($actual)" tar -xzf "$dl/$name.tar.gz" -C "$dl" @@ -75,6 +147,8 @@ rm -rf "$dl" [ -n "${GITHUB_PATH:-}" ] && echo "$bin_dir" >> "$GITHUB_PATH" "$bin_dir/homecloud" version echo "::endgroup::" +# Tests of the download and verification stop here (test.yml). +[ "${HC_INSTALL_ONLY:-}" = 1 ] && exit 0 echo "Starting homecloud serve on 127.0.0.1:$port (data: $data_dir, log: $log)" mkdir -p "$data_dir" diff --git a/integrations/github-action/test/inputs.sh b/integrations/github-action/test/inputs.sh index e2386bc6..c0bbc0ed 100755 --- a/integrations/github-action/test/inputs.sh +++ b/integrations/github-action/test/inputs.sh @@ -5,16 +5,16 @@ set -u setup="$(cd "$(dirname "$0")/.." && pwd)/setup.sh" fails=0 -# expect ok|fail PORT TIMEOUT SERVICES +# expect ok|fail PORT TIMEOUT SERVICES [VERIFY_SIGNATURE] expect() { local want=$1 out got=ok - out=$(HC_VALIDATE_ONLY=1 HC_PORT="$2" HC_WAIT_TIMEOUT="$3" HC_SERVICES_WAIT="$4" \ + out=$(HC_VALIDATE_ONLY=1 HC_PORT="$2" HC_WAIT_TIMEOUT="$3" HC_SERVICES_WAIT="$4" HC_VERIFY_SIGNATURE="${5-true}" \ GITHUB_OUTPUT=/dev/null GITHUB_ENV=/dev/null GITHUB_STATE= RUNNER_TEMP="${TMPDIR:-/tmp}" bash "$setup" 2>&1) || got=fail if [ "$got" != "$want" ]; then - echo "FAIL: port='$2' timeout='$3' services='$4': want $want, got $got: $out" + echo "FAIL: port='$2' timeout='$3' services='$4' verify='${5-true}': want $want, got $got: $out" fails=$((fails + 1)) else - echo "ok: port='$2' timeout='$3' services='$4' -> $want" + echo "ok: port='$2' timeout='$3' services='$4' verify='${5-true}' -> $want" fi } @@ -30,5 +30,10 @@ expect fail http 180 "" expect fail 70000 180 "" expect fail 8080 180 "s33" expect fail 8080 180 "s3,nosuchservice" +expect ok 8080 180 "" false +expect ok 8080 180 "" FALSE +expect ok 8080 180 "" "" # empty means the default (true) +expect fail 8080 180 "" no +expect fail 8080 180 "" 1 [ "$fails" -eq 0 ] && echo "all input checks passed" || { echo "$fails input checks failed"; exit 1; } diff --git a/integrations/testcontainers-go/README.md b/integrations/testcontainers-go/README.md index 81e58307..0f875ae8 100644 --- a/integrations/testcontainers-go/README.md +++ b/integrations/testcontainers-go/README.md @@ -32,7 +32,7 @@ func TestUpload(t *testing.T) { | | | | --- | --- | | `Run(ctx, image, opts...)` | starts HomeCloud and waits for the API (and S3, by default) | -| `hc.EndpointURL()` | `http://127.0.0.1:18080` | +| `hc.EndpointURL()` | `http://localhost:18080` | | `hc.Endpoint(ctx, "http")` | same, through the `testcontainers.Container` interface | | `hc.Credentials()` | root `AccessKeyID`, `SecretAccessKey`, `Region` | | `hc.AWSConfig()` | `aws.Config` for any `NewFromConfig` | @@ -55,6 +55,12 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1`. S3 clients should use +path-style addressing (`UsePathStyle`), so bucket names need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel, nor next to a HomeCloud you run yourself on the same Docker host diff --git a/integrations/testcontainers-go/go.mod b/integrations/testcontainers-go/go.mod index 489ec01d..52d935df 100644 --- a/integrations/testcontainers-go/go.mod +++ b/integrations/testcontainers-go/go.mod @@ -3,11 +3,12 @@ module github.com/solinode/homecloud/integrations/testcontainers-go go 1.25.0 require ( - github.com/aws/aws-sdk-go-v2 v1.47.1 + github.com/aws/aws-sdk-go-v2 v1.47.3 github.com/aws/aws-sdk-go-v2/credentials v1.20.7 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 + github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4 github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 github.com/moby/moby/api v1.55.0 @@ -20,15 +21,15 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect - github.com/aws/smithy-go v1.28.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/smithy-go v1.28.5 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect diff --git a/integrations/testcontainers-go/go.sum b/integrations/testcontainers-go/go.sum index 2889f67d..8e49eb83 100644 --- a/integrations/testcontainers-go/go.sum +++ b/integrations/testcontainers-go/go.sum @@ -6,16 +6,16 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= -github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= github.com/aws/aws-sdk-go-v2/credentials v1.20.7 h1:mit8rRVnmt+wvT4U6putiWSlwe3eKD2n6ElgyfcP/Mw= github.com/aws/aws-sdk-go-v2/credentials v1.20.7/go.mod h1:/2pPi2qhPug21z6l3M0+AKUqAT1zn2MhRCtnQv8R5Zg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 h1:kEs6rZI/z6aD9JBIjXzaXA2UyGYqhpQrxDnmcpU9fGA= @@ -28,18 +28,20 @@ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 h1:6Hvm github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4/go.mod h1:zv2N29aiQUhG2XZNM9zgwCnAyVBdTBbcIpfNAlNmA20= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 h1:/P13I8ouMzMIa2yY32aUh+GIh0wYbJ/auQb1KAEhB8k= github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0/go.mod h1:jUmFXtUKRVCKTaKap+NgL32pmSkVehamqqMENlGMApk= github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q= github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4 h1:4bY+1jJ15fXOsGAuYjgZYrb1D6GyZ1asQnAa+aNKAns= +github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4/go.mod h1:yHNx8e9O4nK766Ghzg/UXYr3SW8Co4VeKvRWU39VOAk= github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 h1:6ZXe/lG2mbGhw10y/aiKbe/poytLoYJsdbKV1pT9+ws= github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2/go.mod h1:+TDqZ1h8CLkW9ewfQkSPWHYRjm7/wDThKeDlR46qyvE= github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 h1:AWSPqynwEJV8J2pl/p7aizCf7a23/NGHR+rK/KLOxi4= github.com/aws/aws-sdk-go-v2/service/sts v1.51.2/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= -github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= -github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/integrations/testcontainers-go/homecloud.go b/integrations/testcontainers-go/homecloud.go index 89aaeabb..331c42c1 100644 --- a/integrations/testcontainers-go/homecloud.go +++ b/integrations/testcontainers-go/homecloud.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "regexp" "strconv" @@ -146,10 +147,7 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom if err != nil { return c, err } - if host == "localhost" { - host = "127.0.0.1" // keep SDKs from trying IPv6 first or bucket.localhost names - } - c.endpoint = "http://" + host + ":" + strconv.Itoa(o.port) + c.endpoint = "http://" + net.JoinHostPort(endpointHost(host), strconv.Itoa(o.port)) if err := c.waitHealthy(ctx, 30*time.Second); err != nil { return c, err } @@ -159,12 +157,26 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom return c, nil } -// EndpointURL is the URL of the HomeCloud API, e.g. http://127.0.0.1:18080: +// endpointHost names a loopback Docker host "localhost", as `homecloud aws-env` +// does: S3 Control (which the Terraform AWS provider calls for every +// aws_s3_bucket) puts the account ID in front of the endpoint's host, and +// 123456789012.127.0.0.1 does not resolve while 123456789012.localhost does. +// HomeCloud listens on 127.0.0.1 only; clients that try ::1 first fall back to +// it. S3 clients should still use path-style addressing (see AWSConfig). +func endpointHost(host string) string { + switch host { + case "localhost", "127.0.0.1", "::1": + return "localhost" + } + return host +} + +// EndpointURL is the URL of the HomeCloud API, e.g. http://localhost:18080: // what AWS_ENDPOINT_URL or an SDK's base endpoint should be set to. func (c *Container) EndpointURL() string { return c.endpoint } // Endpoint returns the API address as testcontainers.Container.Endpoint does -// ("127.0.0.1:18080", or "http://127.0.0.1:18080" for proto "http"). It +// ("localhost:18080", or "http://localhost:18080" for proto "http"). It // replaces the generic one, which looks for a mapped port and so does not work // with host networking. func (c *Container) Endpoint(_ context.Context, proto string) (string, error) { diff --git a/integrations/testcontainers-go/homecloud_test.go b/integrations/testcontainers-go/homecloud_test.go index 127cde4c..359e9c0e 100644 --- a/integrations/testcontainers-go/homecloud_test.go +++ b/integrations/testcontainers-go/homecloud_test.go @@ -13,6 +13,7 @@ import ( ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/moby/moby/client" @@ -77,6 +78,14 @@ func TestHomeCloud(t *testing.T) { t.Errorf("s3 body %q", b) } + // S3 Control puts the account ID in front of the endpoint host + // (.localhost), which is why the endpoint is not 127.0.0.1. + if _, err := s3control.NewFromConfig(cfg).ListTagsForResource(ctx, &s3control.ListTagsForResourceInput{ + AccountId: aws.String(hc.AccountID()), ResourceArn: aws.String("arn:aws:s3:::tc-go"), + }); err != nil { + t.Fatalf("s3control list tags (endpoint %s): %v", hc.EndpointURL(), err) + } + sq := sqs.NewFromConfig(cfg) q, err := sq.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("tc-go")}) if err != nil { diff --git a/integrations/testcontainers-java/README.md b/integrations/testcontainers-java/README.md index 5d3e41dd..2e020357 100644 --- a/integrations/testcontainers-java/README.md +++ b/integrations/testcontainers-java/README.md @@ -51,7 +51,7 @@ With the Testcontainers JUnit 5 extension, `@Testcontainers` plus | | | | --- | --- | | `new HomeCloudContainer()`, `new HomeCloudContainer(image)` | default image `ghcr.io/solinode/homecloud:latest` | -| `getEndpoint()`, `getEndpointUri()` | `http://127.0.0.1:18080` | +| `getEndpoint()`, `getEndpointUri()` | `http://localhost:18080` | | `getCredentials()`, `getAccessKey()`, `getSecretKey()`, `getRegion()` | root keys and region | | `getCredentialsProvider()` | an `AwsCredentialsProvider` | | `configure(builder)` | endpoint override, credentials and region on any SDK v2 client builder: `hc.configure(SnsClient.builder()).build()` | @@ -77,6 +77,12 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1`. S3 clients should use +path-style addressing (`getS3Client()` does), so bucket names need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel, nor next to a HomeCloud you run yourself on the same Docker host diff --git a/integrations/testcontainers-java/pom.xml b/integrations/testcontainers-java/pom.xml index 9616c82f..347f703b 100644 --- a/integrations/testcontainers-java/pom.xml +++ b/integrations/testcontainers-java/pom.xml @@ -125,6 +125,11 @@ sts test + + software.amazon.awssdk + s3control + test + org.slf4j slf4j-simple diff --git a/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java b/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java index 2c0fb570..b614bb6e 100644 --- a/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java +++ b/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java @@ -265,17 +265,31 @@ private void captureAccount() { // Connection details /** - * The URL of the HomeCloud API, e.g. {@code http://127.0.0.1:18080}: what AWS_ENDPOINT_URL or + * The URL of the HomeCloud API, e.g. {@code http://localhost:18080}: what AWS_ENDPOINT_URL or * an SDK's endpoint override should be. * * @return the endpoint URL */ public String getEndpoint() { - String host = getHost(); - if ("localhost".equals(host)) { - host = "127.0.0.1"; // keeps SDKs from trying IPv6 first or bucket.localhost names + return "http://" + endpointHost(getHost()) + ":" + port; + } + + /** + * Names a loopback Docker host {@code localhost}, as {@code homecloud aws-env} does: S3 Control + * (which the Terraform AWS provider calls for every bucket) puts the account ID in front of + * the endpoint's host, and {@code 123456789012.127.0.0.1} does not resolve. HomeCloud listens + * on 127.0.0.1 only; Java resolves localhost to 127.0.0.1 first, and the SDK's HTTP clients + * try the next address when one refuses the connection. + */ + static String endpointHost(String host) { + switch (host) { + case "localhost": + case "127.0.0.1": + case "::1": + return "localhost"; + default: + return host; } - return "http://" + host + ":" + port; } /** diff --git a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java index 6fbb6ca5..744c829b 100644 --- a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java +++ b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3control.S3ControlClient; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sts.StsClient; @@ -97,6 +98,11 @@ void s3PutGet() { assertEquals("hello", s3.getObjectAsBytes(b -> b.bucket("tc-java").key("a.txt")).asUtf8String()); assertEquals(1, s3.listObjectsV2(b -> b.bucket("tc-java")).keyCount()); } + // S3 Control puts the account ID in front of the endpoint host (.localhost), + // which is why the endpoint is not 127.0.0.1. + try (S3ControlClient control = hc.configure(S3ControlClient.builder()).build()) { + control.listTagsForResource(b -> b.accountId(hc.getAccountId()).resourceArn("arn:aws:s3:::tc-java")); + } } @Test diff --git a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java index f6f465f8..e2e69d9d 100644 --- a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java +++ b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java @@ -20,4 +20,11 @@ void parsesQuotedExports() { assertEquals("us-east-1", env.get("AWS_REGION")); assertEquals("http://127.0.0.1:18080", env.get("AWS_ENDPOINT_URL")); } + + @Test + void namesALoopbackHostLocalhost() { + assertEquals("localhost", HomeCloudContainer.endpointHost("localhost")); + assertEquals("localhost", HomeCloudContainer.endpointHost("127.0.0.1")); + assertEquals("docker.example", HomeCloudContainer.endpointHost("docker.example")); + } } diff --git a/integrations/testcontainers-node/README.md b/integrations/testcontainers-node/README.md index 7894fba8..c5220e45 100644 --- a/integrations/testcontainers-node/README.md +++ b/integrations/testcontainers-node/README.md @@ -38,7 +38,7 @@ ESM and CommonJS, with TypeScript types. Requires Node.js 20+. | --- | --- | | `new HomeCloudContainer(image?)` | default image `ghcr.io/solinode/homecloud:latest` | | `.start()` | starts HomeCloud, waits for the API (and S3, by default), reads the credentials | -| `hc.getEndpoint()` | `http://127.0.0.1:18080` | +| `hc.getEndpoint()` | `http://localhost:18080` | | `hc.getCredentials()` | `{ accessKeyId, secretAccessKey, region }` | | `hc.getRegion()` | `us-east-1` unless configured otherwise | | `hc.getClientConfig()` | `{ endpoint, region, credentials }` for any SDK v3 client | @@ -63,6 +63,13 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1` (Node.js 20+). S3 +clients should use path-style addressing (`getS3ClientConfig()` does), so bucket names +need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel (with vitest, `fileParallelism: false`), nor next to a HomeCloud you diff --git a/integrations/testcontainers-node/package-lock.json b/integrations/testcontainers-node/package-lock.json index 2a0105b5..d07f44d0 100644 --- a/integrations/testcontainers-node/package-lock.json +++ b/integrations/testcontainers-node/package-lock.json @@ -15,6 +15,7 @@ "@aws-sdk/client-dynamodb": "^3.1151.0", "@aws-sdk/client-lambda": "^3.1151.0", "@aws-sdk/client-s3": "^3.1151.0", + "@aws-sdk/client-s3-control": "^3.1151.0", "@aws-sdk/client-sqs": "^3.1151.0", "@aws-sdk/client-sts": "^3.1151.0", "@types/node": "^22.0.0", @@ -108,6 +109,28 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-s3-control": { + "version": "3.1151.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3-control/-/client-s3-control-3.1151.0.tgz", + "integrity": "sha512-2DKNaTgGwkUSJ3cDpRyKCVEXaKKDqEhzSk/BUbdp4UewJUC05kwO7Z9oZrUwHrAjcLmo8ky8kZV+FCcmTHkxTw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-node": "^3.972.86", + "@aws-sdk/middleware-sdk-s3": "^3.972.79", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/middleware-apply-body-checksum": "^4.6.3", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-sqs": { "version": "3.1151.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1151.0.tgz", @@ -1881,6 +1904,21 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-apply-body-checksum": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-apply-body-checksum/-/middleware-apply-body-checksum-4.6.3.tgz", + "integrity": "sha512-c2WI+satDWRRptxuczkq0ZqHqfWm5F7GP7aMS4wAjbs9MPxFwKcaVOsiFAlAB2wzclxCiumcVeCzluJxw6qmfw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/node-http-handler": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", diff --git a/integrations/testcontainers-node/package.json b/integrations/testcontainers-node/package.json index ced9860a..382fd8dd 100644 --- a/integrations/testcontainers-node/package.json +++ b/integrations/testcontainers-node/package.json @@ -60,6 +60,7 @@ "@aws-sdk/client-dynamodb": "^3.1151.0", "@aws-sdk/client-lambda": "^3.1151.0", "@aws-sdk/client-s3": "^3.1151.0", + "@aws-sdk/client-s3-control": "^3.1151.0", "@aws-sdk/client-sqs": "^3.1151.0", "@aws-sdk/client-sts": "^3.1151.0", "@types/node": "^22.0.0", diff --git a/integrations/testcontainers-node/src/index.ts b/integrations/testcontainers-node/src/index.ts index 19dbbb94..6e0f778f 100644 --- a/integrations/testcontainers-node/src/index.ts +++ b/integrations/testcontainers-node/src/index.ts @@ -138,8 +138,7 @@ export class HomeCloudContainer extends GenericContainer { started = await super.start(); const account = this.account() ?? (await readAccount(started)); if (!account) throw new Error("homecloud: no account in the first-start log"); - const host = started.getHost() === "localhost" ? "127.0.0.1" : started.getHost(); - const endpoint = `http://${host}:${this.port}`; + const endpoint = `http://${endpointHost(started.getHost())}:${this.port}`; await waitHealthy(endpoint, 30_000); const credentials = await readCredentials(started); return new StartedHomeCloudContainer(started, endpoint, credentials, account); @@ -172,7 +171,7 @@ export class StartedHomeCloudContainer extends AbstractStartedContainer { super(startedTestContainer); } - /** The API URL, e.g. http://127.0.0.1:18080: what AWS_ENDPOINT_URL should be. */ + /** The API URL, e.g. http://localhost:18080: what AWS_ENDPOINT_URL should be. */ getEndpoint(): string { return this.endpoint; } @@ -274,6 +273,17 @@ export async function removeResources(account: string): Promise { } } +/** + * The host name for the endpoint: a loopback Docker host is named `localhost`, as + * `homecloud aws-env` does. S3 Control (which the Terraform AWS provider calls for every + * bucket) puts the account ID in front of the endpoint's host, and `123456789012.127.0.0.1` + * does not resolve. HomeCloud listens on 127.0.0.1 only; Node.js 20+ tries 127.0.0.1 when + * ::1 refuses the connection. + */ +export function endpointHost(host: string): string { + return host === "localhost" || host === "127.0.0.1" || host === "::1" ? "localhost" : host; +} + /** Parses the `export NAME="value"` lines printed by `homecloud aws-env`. */ export function parseAwsEnv(output: string): Record { const env: Record = {}; diff --git a/integrations/testcontainers-node/test/homecloud.test.ts b/integrations/testcontainers-node/test/homecloud.test.ts index 8c7fbe95..16c22020 100644 --- a/integrations/testcontainers-node/test/homecloud.test.ts +++ b/integrations/testcontainers-node/test/homecloud.test.ts @@ -1,11 +1,12 @@ import { CreateTableCommand, DynamoDBClient, GetItemCommand, PutItemCommand } from "@aws-sdk/client-dynamodb"; import { LambdaClient, ListFunctionsCommand } from "@aws-sdk/client-lambda"; import { CreateBucketCommand, GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { ListTagsForResourceCommand, S3ControlClient } from "@aws-sdk/client-s3-control"; import { CreateQueueCommand, ReceiveMessageCommand, SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { GetCallerIdentityCommand, STSClient } from "@aws-sdk/client-sts"; import { getContainerRuntimeClient } from "testcontainers"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { DEFAULT_IMAGE, HomeCloudContainer, parseAwsEnv, type StartedHomeCloudContainer } from "../src/index.js"; +import { DEFAULT_IMAGE, endpointHost, HomeCloudContainer, parseAwsEnv, type StartedHomeCloudContainer } from "../src/index.js"; // A local build (docker build -t homecloud:test .) or the official image. const IMAGE = process.env.HOMECLOUD_IMAGE || DEFAULT_IMAGE; @@ -17,6 +18,14 @@ describe("parseAwsEnv", () => { }); }); +describe("endpointHost", () => { + it("names a loopback host localhost", () => { + expect(endpointHost("localhost")).toBe("localhost"); + expect(endpointHost("127.0.0.1")).toBe("localhost"); + expect(endpointHost("docker.example")).toBe("docker.example"); + }); +}); + async function expectNoLeftovers(account: string): Promise { const docker = (await getContainerRuntimeClient()).container.dockerode; const filters = JSON.stringify({ label: [`homecloud.account=${account}`] }); @@ -68,6 +77,12 @@ describe("HomeCloudContainer", () => { await s3.send(new PutObjectCommand({ Bucket: "tc-node", Key: "a.txt", Body: "hello" })); const obj = await s3.send(new GetObjectCommand({ Bucket: "tc-node", Key: "a.txt" })); expect(await obj.Body?.transformToString()).toBe("hello"); + // S3 Control puts the account ID in front of the endpoint host (.localhost), + // which is why the endpoint is not 127.0.0.1. + const control = new S3ControlClient(hc.getClientConfig()); + await control.send( + new ListTagsForResourceCommand({ AccountId: hc.getAccountId(), ResourceArn: "arn:aws:s3:::tc-node" }), + ); }); it("SQS", async () => { diff --git a/integrations/testcontainers-python/README.md b/integrations/testcontainers-python/README.md index ec2704b8..59b3e12d 100644 --- a/integrations/testcontainers-python/README.md +++ b/integrations/testcontainers-python/README.md @@ -31,7 +31,7 @@ def test_upload(homecloud): | | | | --- | --- | | `HomeCloudContainer(image=DEFAULT_IMAGE, port=18080, docker_socket="/var/run/docker.sock", wait_for_services=("s3",), startup_timeout=180)` | `wait_for_services` takes `s3`, `ecr`, `route53`; everything else is ready with the API | -| `get_endpoint()` | `http://127.0.0.1:18080` | +| `get_endpoint()` | `http://localhost:18080` | | `get_credentials()` | `Credentials(access_key_id, secret_access_key, region)` | | `get_client(service, **kw)` / `get_resource(service, **kw)` | boto3 client / resource for HomeCloud | | `aws_env()` | `AWS_ENDPOINT_URL`, keys and region for subprocesses (AWS CLI, Terraform) | @@ -46,6 +46,12 @@ gets the Docker socket and **host networking**; the API listens on `127.0.0.1:

None: # Connection details def get_endpoint(self) -> str: - """The API URL, e.g. http://127.0.0.1:18080 (what AWS_ENDPOINT_URL should be).""" + """The API URL, e.g. http://localhost:18080 (what AWS_ENDPOINT_URL should be). + + A loopback Docker host is named ``localhost``, as ``homecloud aws-env`` + does: S3 Control (which the Terraform AWS provider calls for every + bucket) puts the account ID in front of the endpoint's host, and + ``123456789012.127.0.0.1`` does not resolve. HomeCloud listens on + 127.0.0.1 only; clients that try ::1 first fall back to it. + """ host = self.get_container_host_ip() - if host == "localhost": - host = "127.0.0.1" + if host in ("localhost", "127.0.0.1", "::1"): + host = "localhost" return f"http://{host}:{self.port}" def get_credentials(self) -> Credentials: diff --git a/integrations/testcontainers-python/tests/test_homecloud.py b/integrations/testcontainers-python/tests/test_homecloud.py index c792ce32..aa52eb6d 100644 --- a/integrations/testcontainers-python/tests/test_homecloud.py +++ b/integrations/testcontainers-python/tests/test_homecloud.py @@ -47,6 +47,10 @@ def test_s3(homecloud): assert s3.get_object(Bucket="tc-python", Key="a.txt")["Body"].read() == b"hello" bucket = homecloud.get_resource("s3").Bucket("tc-python") assert [o.key for o in bucket.objects.all()] == ["a.txt"] + # S3 Control puts the account ID in front of the endpoint host + # (.localhost), which is why the endpoint is not 127.0.0.1. + control = homecloud.get_client("s3control") + control.list_tags_for_resource(AccountId=homecloud.account_id, ResourceArn="arn:aws:s3:::tc-python") def test_sqs(homecloud): diff --git a/integrations/testcontainers-python/tests/test_options.py b/integrations/testcontainers-python/tests/test_options.py index 4185beba..1a695244 100644 --- a/integrations/testcontainers-python/tests/test_options.py +++ b/integrations/testcontainers-python/tests/test_options.py @@ -26,3 +26,14 @@ def test_command_and_socket(): hc = make(port=19090, docker_socket="/run/user/1000/docker.sock") assert hc._command == ["serve", "--data-dir", "/data", "--addr", "127.0.0.1:19090"] assert hc.volumes["/run/user/1000/docker.sock"]["bind"] == "/var/run/docker.sock" + + +def test_endpoint_names_loopback_localhost(): + for host, want in [ + ("localhost", "http://localhost:18080"), + ("127.0.0.1", "http://localhost:18080"), + ("docker.example", "http://docker.example:18080"), + ]: + hc = make() + with mock.patch.object(HomeCloudContainer, "get_container_host_ip", return_value=host): + assert hc.get_endpoint() == want