From eb0856dc7a43cb5e95ebcada29e4cae1640e4aeb Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:05:42 +0530 Subject: [PATCH 1/4] Tests: one meaning for HC_TEST_TERRAFORM across the Terraform tests Add awstest.Terraform, which skips unless HC_TEST_TERRAFORM is set (and in -short mode), uses the value as the binary when it is an executable, and otherwise looks up terraform, then tofu, on PATH. The IAM, S3, CloudFormation and Route 53 Terraform tests all use it, so HC_TEST_TERRAFORM=1 no longer makes the IAM test run a program called 1, and the IAM test no longer runs unasked. Closes #97 --- CONTRIBUTING.md | 2 +- cli/internal/awsapi/awstest/awstest.go | 31 ++++++++++++++++++++++ cli/internal/svc/cfn/terraform_test.go | 12 +++------ cli/internal/svc/iam/terraform_test.go | 16 +++-------- cli/internal/svc/route53/terraform_test.go | 11 ++------ cli/internal/svc/s3/terraform_test.go | 14 +++------- 6 files changed, 44 insertions(+), 42 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f633a9bb..4c531872 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,7 +110,7 @@ If Docker tests skip with "no free 10.88-119.0.0/16 range", earlier runs leaked | --- | --- | | `HC_TEST_PYTHON` | Python interpreter with boto3 (default `python3`); e.g. `HC_TEST_PYTHON=$PWD/.venv/bin/python` | | `HC_TEST_AWS` | path to the `aws` CLI (default: `aws` on `PATH`) | -| `HC_TEST_TERRAFORM` | set to run the Terraform/OpenTofu tests (they download the AWS provider; set `TF_PLUGIN_CACHE_DIR` to reuse it) | +| `HC_TEST_TERRAFORM` | set to run the Terraform/OpenTofu tests: `1` uses `terraform`, then `tofu`, from `PATH`; a path (e.g. `/opt/tofu/tofu`) uses that binary. They download the AWS provider; set `TF_PLUGIN_CACHE_DIR` to reuse it | | `HC_TEST_NODE_PATH` | a `node_modules` directory with `amazon-cognito-identity-js`, for the Cognito SRP test in Node.js | **End to end.** With a server running, `./scripts/smoke.sh` exercises every service through the CLI and diff --git a/cli/internal/awsapi/awstest/awstest.go b/cli/internal/awsapi/awstest/awstest.go index fc66ccd0..47123415 100644 --- a/cli/internal/awsapi/awstest/awstest.go +++ b/cli/internal/awsapi/awstest/awstest.go @@ -11,6 +11,10 @@ // // The AWS CLI is found on PATH (or HC_TEST_AWS); Python with boto3 is // HC_TEST_PYTHON (default python3). Tests skip when a tool is missing. +// +// The Terraform/OpenTofu tests are opt-in because they download the AWS +// provider: set HC_TEST_TERRAFORM to run them, either to "1" (use terraform, +// then tofu, from PATH) or to the path of the binary to use. See Terraform. package awstest import ( @@ -179,6 +183,33 @@ func (h *Harness) Signed(t *testing.T, service, method, path string, header http return resp.StatusCode, resp.Header, out } +// Terraform returns the Terraform or OpenTofu binary for a Terraform test, or +// skips the test. The tests are opt-in: they run only when HC_TEST_TERRAFORM +// is set, and not in -short mode. A value that names an executable file (for +// example /opt/tofu/tofu) is used as the binary; any other value (such as 1) +// looks up terraform, then tofu, on PATH. Set TF_PLUGIN_CACHE_DIR to reuse the +// provider download across runs. +func Terraform(t *testing.T) string { + t.Helper() + v := os.Getenv("HC_TEST_TERRAFORM") + if v == "" { + t.Skip("set HC_TEST_TERRAFORM=1 (or the path to terraform/tofu) to run") + } + if testing.Short() { + t.Skip("terraform test is slow") + } + if fi, err := os.Stat(v); err == nil && fi.Mode().IsRegular() && fi.Mode().Perm()&0o111 != 0 { + return v + } + for _, name := range []string{"terraform", "tofu"} { + if p, err := exec.LookPath(name); err == nil { + return p + } + } + t.Skipf("HC_TEST_TERRAFORM=%q is not an executable, and neither terraform nor tofu is on PATH", v) + return "" +} + func awsBin(t *testing.T) string { if p := os.Getenv("HC_TEST_AWS"); p != "" { return p diff --git a/cli/internal/svc/cfn/terraform_test.go b/cli/internal/svc/cfn/terraform_test.go index ca4047b3..9ee3acb2 100644 --- a/cli/internal/svc/cfn/terraform_test.go +++ b/cli/internal/svc/cfn/terraform_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" ) // TestTerraformCloudFormationStack applies, re-plans, updates and destroys an @@ -13,15 +15,7 @@ import ( // (HC_TEST_TERRAFORM=1) since it downloads the provider; set TF_PLUGIN_CACHE_DIR // to reuse it. func TestTerraformCloudFormationStack(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) e := newEnv(t, false) dir := t.TempDir() cfg := func(retention string) string { diff --git a/cli/internal/svc/iam/terraform_test.go b/cli/internal/svc/iam/terraform_test.go index 05a8f8ae..55a8978d 100644 --- a/cli/internal/svc/iam/terraform_test.go +++ b/cli/internal/svc/iam/terraform_test.go @@ -12,20 +12,10 @@ import ( ) // TestTerraform applies and destroys a Terraform configuration with the AWS -// provider against HomeCloud's IAM and STS. It needs terraform (HC_TEST_TERRAFORM -// or PATH) and network access to download the provider on first use; set -// TF_PLUGIN_CACHE_DIR to reuse the download. +// provider against HomeCloud's IAM and STS. Opt-in (see awstest.Terraform) since it +// downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse the download. func TestTerraform(t *testing.T) { - bin := os.Getenv("HC_TEST_TERRAFORM") - if bin == "" { - var err error - if bin, err = exec.LookPath("terraform"); err != nil { - t.Skip("terraform not installed (set HC_TEST_TERRAFORM)") - } - } - if testing.Short() { - t.Skip("terraform test is slow") - } + bin := awstest.Terraform(t) h := awstest.New(t) dir := t.TempDir() write := func(policyAction string) { diff --git a/cli/internal/svc/route53/terraform_test.go b/cli/internal/svc/route53/terraform_test.go index a4eb15e8..eb5e5821 100644 --- a/cli/internal/svc/route53/terraform_test.go +++ b/cli/internal/svc/route53/terraform_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" "github.com/homecloudhq/homecloud/cli/internal/svc/acm" ) @@ -14,15 +15,7 @@ import ( // and the hashicorp/aws provider. Opt-in (HC_TEST_TERRAFORM=1) since it // downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse it. func TestTerraform(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) h, _ := setup(t) a := acm.New(h.Env, h.Secrets) a.RegisterAWS() diff --git a/cli/internal/svc/s3/terraform_test.go b/cli/internal/svc/s3/terraform_test.go index f65b039e..3864ba6b 100644 --- a/cli/internal/svc/s3/terraform_test.go +++ b/cli/internal/svc/s3/terraform_test.go @@ -6,21 +6,15 @@ import ( "path/filepath" "strings" "testing" + + "github.com/homecloudhq/homecloud/cli/internal/awsapi/awstest" ) // TestTerraform applies, re-plans and destroys S3 resources with Terraform or -// OpenTofu and the hashicorp/aws provider. Opt-in (HC_TEST_TERRAFORM=1) since +// OpenTofu and the hashicorp/aws provider. Opt-in (see awstest.Terraform) since // it downloads the provider; set TF_PLUGIN_CACHE_DIR to reuse it. func TestTerraform(t *testing.T) { - if os.Getenv("HC_TEST_TERRAFORM") == "" { - t.Skip("set HC_TEST_TERRAFORM=1 to run") - } - bin, err := exec.LookPath("terraform") - if err != nil { - if bin, err = exec.LookPath("tofu"); err != nil { - t.Skip("terraform/tofu not installed") - } - } + bin := awstest.Terraform(t) h, _ := newHarness(t) b := bucketName() dir := t.TempDir() From 860012d77d42482d0e621b92a448e3b511bf5ead Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:05:43 +0530 Subject: [PATCH 2/4] Console: general site description, tab titles for home and sign-in Closes #98 --- console/app/(console)/page.tsx | 2 ++ console/app/layout.tsx | 2 +- console/app/login/layout.tsx | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 console/app/login/layout.tsx diff --git a/console/app/(console)/page.tsx b/console/app/(console)/page.tsx index 1b4570ad..48b0d13c 100644 --- a/console/app/(console)/page.tsx +++ b/console/app/(console)/page.tsx @@ -1,5 +1,7 @@ import { ConsoleHome } from "@/components/home/console-home" +export const metadata = { title: "Home" } + export default function HomePage() { return } diff --git a/console/app/layout.tsx b/console/app/layout.tsx index ca4f4298..55e5f158 100644 --- a/console/app/layout.tsx +++ b/console/app/layout.tsx @@ -13,7 +13,7 @@ export const metadata: Metadata = { default: "HomeCloud Console", template: "%s | HomeCloud Console", }, - description: "Manage your self-hosted HomeCloud: EC2, S3, VPC, IAM, Secrets Manager, CloudWatch and CloudTrail.", + description: "The web console for HomeCloud, a self-hosted AWS-compatible cloud.", applicationName: "HomeCloud Console", icons: { icon: `${process.env.NEXT_PUBLIC_BASE_PATH ?? ""}/favicon.svg` }, } diff --git a/console/app/login/layout.tsx b/console/app/login/layout.tsx new file mode 100644 index 00000000..fcc24a61 --- /dev/null +++ b/console/app/login/layout.tsx @@ -0,0 +1,6 @@ +// The sign-in page is a client component, so its tab title is set here. +export const metadata = { title: "Sign in" } + +export default function LoginLayout({ children }: Readonly<{ children: React.ReactNode }>) { + return children +} From 3b873e9e0e98a8a5df994486f83357224b928b6a Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:19:43 +0530 Subject: [PATCH 3/4] GitHub Action mirror: verify the release signature Same change as solinode/setup-homecloud#2: when a release publishes checksums.txt.sigstore.json (v0.5.0 on), verify it with cosign against the release workflow's identity before trusting checksums.txt; releases before v0.5.0 keep the checksum-only check. New input verify-signature (default true). --- integrations/github-action/README.md | 27 ++++++- integrations/github-action/action.yml | 7 ++ integrations/github-action/main.js | 1 + integrations/github-action/setup.sh | 94 ++++++++++++++++++++--- integrations/github-action/test/inputs.sh | 13 +++- 5 files changed, 127 insertions(+), 15 deletions(-) diff --git a/integrations/github-action/README.md b/integrations/github-action/README.md index 48a1b542..854b300d 100644 --- a/integrations/github-action/README.md +++ b/integrations/github-action/README.md @@ -13,7 +13,7 @@ an AWS account. The action: -1. downloads the HomeCloud release you ask for and verifies it against the release's `checksums.txt`, +1. downloads the HomeCloud release you ask for, verifies the Sigstore signature on its `checksums.txt` (releases from v0.5.0 on are signed) and checks the download against it, 2. starts `homecloud serve` in the background (data in `$RUNNER_TEMP/homecloud-data`, log in `$RUNNER_TEMP/homecloud.log`), 3. waits for `/api/v1/health` and for any services listed in `services-wait`, 4. exports `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (masked), `AWS_REGION`, `AWS_DEFAULT_REGION` and `HOMECLOUD_DATA_DIR` for the following steps, and puts `homecloud` on `PATH`, @@ -44,6 +44,7 @@ jobs: | `port` | `8080` | Port the API listens on, on `127.0.0.1`. | | `services-wait` | (none) | Comma or newline separated services to wait for. `s3`, `ecr` and `route53` start containers in the background; in-process services (`sqs`, `dynamodb`, `lambda`, ...) are ready with the API; an unknown name fails the step. | | `wait-timeout` | `180` | Seconds to wait for the API and the services. | +| `verify-signature` | `true` | Verify the release signature with cosign before trusting `checksums.txt` (see [Release verification](#release-verification)). `false` checks the SHA-256 checksum only. | ### Outputs @@ -126,6 +127,30 @@ provider "aws" { - run: ./run-tests.sh --endpoint "${{ steps.homecloud.outputs.endpoint }}" ``` +## Release verification + +HomeCloud releases from v0.5.0 on publish `checksums.txt.sigstore.json`, a Sigstore bundle +over `checksums.txt` made by keyless signing in HomeCloud's release workflow. When the bundle +exists, the action runs + +```sh +cosign verify-blob checksums.txt --bundle checksums.txt.sigstore.json \ + --certificate-identity https://github.com/solinode/homecloud/.github/workflows/release.yml@refs/tags/ \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com +``` + +and fails the step if the signature does not verify, or if a release from v0.5.0 on has no +bundle. Only then does it trust the SHA-256 in `checksums.txt` for the archive. `latest` is +resolved to its tag first, so the signer identity names the exact release. + +- cosign comes from the runner's `PATH` if it is there (for example installed with + `sigstore/cosign-installer`; cosign 2.4 or newer), otherwise the action downloads cosign + v3.1.3 from its GitHub release and checks it against a SHA-256 pinned in `setup.sh`. +- Verification fetches Sigstore's trust root over the network. On a runner that cannot + reach Sigstore, set `verify-signature: false` to check the checksum only. +- Releases before v0.5.0 are unsigned: they are checked against `checksums.txt` only, as + before. + ## Notes - One HomeCloud per Docker host: it names its helper containers (`homecloud-s3`, ...) and diff --git a/integrations/github-action/action.yml b/integrations/github-action/action.yml index 6000f3e7..5bcc6cca 100644 --- a/integrations/github-action/action.yml +++ b/integrations/github-action/action.yml @@ -25,6 +25,13 @@ inputs: description: Seconds to wait for the API and every service in services-wait. required: false default: "180" + verify-signature: + description: >- + Verify the release's Sigstore signature (checksums.txt.sigstore.json, published from + v0.5.0 on) with cosign before trusting checksums.txt: "true" or "false". Releases + before v0.5.0 are unsigned and get the checksum check only. + required: false + default: "true" outputs: endpoint: diff --git a/integrations/github-action/main.js b/integrations/github-action/main.js index 9fa77135..85f1246d 100644 --- a/integrations/github-action/main.js +++ b/integrations/github-action/main.js @@ -10,6 +10,7 @@ const env = { HC_PORT: input("port", "8080"), HC_SERVICES_WAIT: input("services-wait", ""), HC_WAIT_TIMEOUT: input("wait-timeout", "180"), + HC_VERIFY_SIGNATURE: input("verify-signature", "true"), }; const r = spawnSync("bash", [path.join(__dirname, "setup.sh")], { stdio: "inherit", env }); if (r.error) { diff --git a/integrations/github-action/setup.sh b/integrations/github-action/setup.sh index 1bd79fe7..85387c6d 100755 --- a/integrations/github-action/setup.sh +++ b/integrations/github-action/setup.sh @@ -25,6 +25,11 @@ port=$((10#$port)) case "$timeout" in '' | *[!0-9]*) fail "wait-timeout must be a whole number of seconds, got '$timeout'" ;; esac timeout=$((10#$timeout)) [ "$timeout" -ge 1 ] || fail "wait-timeout must be at least 1 second" +case "$(echo "${HC_VERIFY_SIGNATURE:-true}" | tr '[:upper:]' '[:lower:]')" in + true) verify_sig=1 ;; + false) verify_sig= ;; + *) fail "verify-signature must be true or false, got '$HC_VERIFY_SIGNATURE'" ;; +esac # Services that start containers in the background log a line when they are ready; # the others run inside the HomeCloud process and are ready with the API. @@ -36,7 +41,7 @@ for svc in $(echo "${HC_SERVICES_WAIT:-}" | tr ',\n' ' ' | tr '[:upper:]' '[:lo *) case "$in_process" in *" $svc "*) ;; *) fail "services-wait: unknown service '$svc' (background: s3, ecr, route53; in-process:$in_process)" ;; esac ;; esac done -[ "${HC_VALIDATE_ONLY:-}" = 1 ] && { echo "inputs OK: port=$port wait-timeout=$timeout waits=${waits[*]:-}"; exit 0; } +[ "${HC_VALIDATE_ONLY:-}" = 1 ] && { echo "inputs OK: port=$port wait-timeout=$timeout waits=${waits[*]:-} verify-signature=${verify_sig:-0}"; exit 0; } command -v docker >/dev/null 2>&1 || fail "HomeCloud needs Docker on the runner (ubuntu-* runners have it)" docker info >/dev/null 2>&1 || fail "Docker is installed but not running" @@ -49,24 +54,91 @@ case "$(uname -m)" in esac name="homecloud-$os-$arch" if [ "$version" = latest ]; then - base="https://github.com/$REPO/releases/latest/download" -else - case "$version" in v*) ;; *) version="v$version" ;; esac - base="https://github.com/$REPO/releases/download/$version" + # Resolve "latest" to its tag: the signature names the tag it was made for, and + # every file then comes from the same release. + version=$(curl -fsSL --retry 3 -o /dev/null -w '%{url_effective}' "https://github.com/$REPO/releases/latest" | sed -n 's#.*/releases/tag/##p') || true + [ -n "$version" ] || fail "could not determine the latest HomeCloud release" fi +case "$version" in v*) ;; *) version="v$version" ;; esac +base="https://github.com/$REPO/releases/download/$version" + +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +# signed_release VERSION: true for v0.5.0 and later, which are all signed. +signed_release() { + local v=${1#v} major rest minor + v=${v%%-*} + major=${v%%.*} + rest=${v#*.} + minor=${rest%%.*} + case "$major$minor" in *[!0-9]* | "") return 1 ;; esac + [ "$major" -gt 0 ] || [ "$minor" -ge 5 ] +} + +# cosign for the signature check: the runner's own (for example from +# sigstore/cosign-installer), else this pinned release, checked against its SHA-256. +COSIGN_VERSION=v3.1.3 +cosign_sha256() { + case "$1" in + linux-amd64) echo 4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 ;; + linux-arm64) echo c5d324e091826b0d7a78eb16fef316450b4eb9aaec045611c08ba06f5e73220a ;; + darwin-amd64) echo 2347488e5d5b25336644024dfeca5601b190e91197a71a917bda44744aff106c ;; + darwin-arm64) echo 5cf948c2f4dfe59687bdd0b8523709067383e03982cc543475c8a7dc70e92a76 ;; + esac +} +find_cosign() { + if cosign=$(command -v cosign); then + return + fi + local dir="$tmp/homecloud-cosign" want got + want=$(cosign_sha256 "$os-$arch") + [ -n "$want" ] || fail "no pinned cosign for $os/$arch: install cosign first, or set verify-signature: false" + mkdir -p "$dir" + curl -fsSL --retry 3 "https://github.com/sigstore/cosign/releases/download/$COSIGN_VERSION/cosign-$os-$arch" -o "$dir/cosign" || + fail "download of cosign $COSIGN_VERSION failed" + got=$(sha256 "$dir/cosign") + [ "$got" = "$want" ] || fail "checksum mismatch for cosign $COSIGN_VERSION (expected $want, got $got)" + chmod +x "$dir/cosign" + cosign="$dir/cosign" +} echo "::group::Install HomeCloud ($version, $os/$arch)" dl="$tmp/homecloud-dl" rm -rf "$dl" && mkdir -p "$dl" "$bin_dir" curl -fsSL --retry 3 "$base/$name.tar.gz" -o "$dl/$name.tar.gz" || fail "download $base/$name.tar.gz failed" curl -fsSL --retry 3 "$base/checksums.txt" -o "$dl/checksums.txt" || fail "download $base/checksums.txt failed" -expected=$(awk -v f="$name.tar.gz" '$2 == f || $2 == "*" f {print $1}' "$dl/checksums.txt") -[ -n "$expected" ] || fail "checksums.txt has no entry for $name.tar.gz" -if command -v sha256sum >/dev/null 2>&1; then - actual=$(sha256sum "$dl/$name.tar.gz" | cut -d' ' -f1) + +# Releases from v0.5.0 on sign checksums.txt with Sigstore keyless signing in the +# release workflow (checksums.txt.sigstore.json). Verify it when it exists, so the +# checksum below is trusted; older releases are unsigned and get the checksum only. +if [ -z "$verify_sig" ]; then + echo "verify-signature is false: checking the SHA-256 checksum only" +elif curl -fsSL --retry 3 "$base/checksums.txt.sigstore.json" -o "$dl/checksums.txt.sigstore.json" 2>/dev/null; then + find_cosign + echo "verifying the release signature with $cosign" + identity="https://github.com/$REPO/.github/workflows/release.yml@refs/tags/$version" + if ! "$cosign" verify-blob "$dl/checksums.txt" --bundle "$dl/checksums.txt.sigstore.json" \ + --certificate-identity "$identity" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com >"$dl/cosign.log" 2>&1; then + cat "$dl/cosign.log" + fail "signature verification of checksums.txt failed for $version (expected signer $identity)" + fi + echo "signature OK (checksums.txt signed by $identity)" +elif signed_release "$version"; then + fail "$version should be signed, but $base/checksums.txt.sigstore.json could not be downloaded (set verify-signature: false to skip the check)" else - actual=$(shasum -a 256 "$dl/$name.tar.gz" | cut -d' ' -f1) + echo "$version is not signed (releases before v0.5.0 were not): checking the SHA-256 checksum only" fi + +expected=$(awk -v f="$name.tar.gz" '$2 == f || $2 == "*" f {print $1}' "$dl/checksums.txt") +[ -n "$expected" ] || fail "checksums.txt has no entry for $name.tar.gz" +actual=$(sha256 "$dl/$name.tar.gz") [ "$expected" = "$actual" ] || fail "checksum mismatch for $name.tar.gz (expected $expected, got $actual)" echo "checksum OK ($actual)" tar -xzf "$dl/$name.tar.gz" -C "$dl" @@ -75,6 +147,8 @@ rm -rf "$dl" [ -n "${GITHUB_PATH:-}" ] && echo "$bin_dir" >> "$GITHUB_PATH" "$bin_dir/homecloud" version echo "::endgroup::" +# Tests of the download and verification stop here (test.yml). +[ "${HC_INSTALL_ONLY:-}" = 1 ] && exit 0 echo "Starting homecloud serve on 127.0.0.1:$port (data: $data_dir, log: $log)" mkdir -p "$data_dir" diff --git a/integrations/github-action/test/inputs.sh b/integrations/github-action/test/inputs.sh index e2386bc6..c0bbc0ed 100755 --- a/integrations/github-action/test/inputs.sh +++ b/integrations/github-action/test/inputs.sh @@ -5,16 +5,16 @@ set -u setup="$(cd "$(dirname "$0")/.." && pwd)/setup.sh" fails=0 -# expect ok|fail PORT TIMEOUT SERVICES +# expect ok|fail PORT TIMEOUT SERVICES [VERIFY_SIGNATURE] expect() { local want=$1 out got=ok - out=$(HC_VALIDATE_ONLY=1 HC_PORT="$2" HC_WAIT_TIMEOUT="$3" HC_SERVICES_WAIT="$4" \ + out=$(HC_VALIDATE_ONLY=1 HC_PORT="$2" HC_WAIT_TIMEOUT="$3" HC_SERVICES_WAIT="$4" HC_VERIFY_SIGNATURE="${5-true}" \ GITHUB_OUTPUT=/dev/null GITHUB_ENV=/dev/null GITHUB_STATE= RUNNER_TEMP="${TMPDIR:-/tmp}" bash "$setup" 2>&1) || got=fail if [ "$got" != "$want" ]; then - echo "FAIL: port='$2' timeout='$3' services='$4': want $want, got $got: $out" + echo "FAIL: port='$2' timeout='$3' services='$4' verify='${5-true}': want $want, got $got: $out" fails=$((fails + 1)) else - echo "ok: port='$2' timeout='$3' services='$4' -> $want" + echo "ok: port='$2' timeout='$3' services='$4' verify='${5-true}' -> $want" fi } @@ -30,5 +30,10 @@ expect fail http 180 "" expect fail 70000 180 "" expect fail 8080 180 "s33" expect fail 8080 180 "s3,nosuchservice" +expect ok 8080 180 "" false +expect ok 8080 180 "" FALSE +expect ok 8080 180 "" "" # empty means the default (true) +expect fail 8080 180 "" no +expect fail 8080 180 "" 1 [ "$fails" -eq 0 ] && echo "all input checks passed" || { echo "$fails input checks failed"; exit 1; } From 6a042c2182959d5b94d09627963eab8b87dfe58d Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:42:17 +0530 Subject: [PATCH 4/4] Testcontainers: hand out http://localhost: in all four modules The Go, Python, Java and Node modules turned a localhost Docker host into 127.0.0.1. Like homecloud aws-env since #125, name a loopback host localhost instead: S3 Control (which the Terraform AWS provider calls for every aws_s3_bucket) puts the account ID in front of the endpoint's host, and 123456789012.127.0.0.1 does not resolve. HomeCloud listens on 127.0.0.1 and clients that try ::1 first fall back to it; the modules' S3 helpers already use path-style addressing, so bucket names never need to resolve. Each module's integration test now calls S3 Control ListTagsForResource, and Python, Java and Node unit-test the host mapping. READMEs and docs/integrations.md explain the choice. --- docs/integrations.md | 4 +- integrations/testcontainers-go/README.md | 8 +++- integrations/testcontainers-go/go.mod | 11 +++--- integrations/testcontainers-go/go.sum | 22 ++++++----- integrations/testcontainers-go/homecloud.go | 24 +++++++++--- .../testcontainers-go/homecloud_test.go | 9 +++++ integrations/testcontainers-java/README.md | 8 +++- integrations/testcontainers-java/pom.xml | 5 +++ .../testcontainers/HomeCloudContainer.java | 24 +++++++++--- .../HomeCloudContainerTest.java | 6 +++ .../testcontainers/ParseAwsEnvTest.java | 7 ++++ integrations/testcontainers-node/README.md | 9 ++++- .../testcontainers-node/package-lock.json | 38 +++++++++++++++++++ integrations/testcontainers-node/package.json | 1 + integrations/testcontainers-node/src/index.ts | 16 ++++++-- .../test/homecloud.test.ts | 17 ++++++++- integrations/testcontainers-python/README.md | 8 +++- .../src/testcontainers_homecloud/__init__.py | 13 +++++-- .../tests/test_homecloud.py | 4 ++ .../tests/test_options.py | 11 ++++++ 20 files changed, 207 insertions(+), 38 deletions(-) diff --git a/docs/integrations.md b/docs/integrations.md index 25f60cca..9154a749 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -162,7 +162,9 @@ with types. See [integrations/testcontainers-node/README.md](../integrations/tes All four modules start the image (default `ghcr.io/solinode/homecloud:latest`, configurable) with the Docker socket mounted and **host networking**, and the API on -`127.0.0.1:18080` of the Docker host (configurable; not 8080, which is often taken). +`127.0.0.1:18080` of the Docker host (configurable; not 8080, which is often taken). The +endpoint they hand out names it `http://localhost:18080`, for the reason in +[SDK notes](#sdk-notes). HomeCloud then reaches the containers it starts on the host's loopback ports, and they call it back through `host.docker.internal`, exactly as when it runs on the host. This works on Linux (GitHub Actions included), OrbStack, and Docker Desktop with host diff --git a/integrations/testcontainers-go/README.md b/integrations/testcontainers-go/README.md index 81e58307..0f875ae8 100644 --- a/integrations/testcontainers-go/README.md +++ b/integrations/testcontainers-go/README.md @@ -32,7 +32,7 @@ func TestUpload(t *testing.T) { | | | | --- | --- | | `Run(ctx, image, opts...)` | starts HomeCloud and waits for the API (and S3, by default) | -| `hc.EndpointURL()` | `http://127.0.0.1:18080` | +| `hc.EndpointURL()` | `http://localhost:18080` | | `hc.Endpoint(ctx, "http")` | same, through the `testcontainers.Container` interface | | `hc.Credentials()` | root `AccessKeyID`, `SecretAccessKey`, `Region` | | `hc.AWSConfig()` | `aws.Config` for any `NewFromConfig` | @@ -55,6 +55,12 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1`. S3 clients should use +path-style addressing (`UsePathStyle`), so bucket names need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel, nor next to a HomeCloud you run yourself on the same Docker host diff --git a/integrations/testcontainers-go/go.mod b/integrations/testcontainers-go/go.mod index 489ec01d..52d935df 100644 --- a/integrations/testcontainers-go/go.mod +++ b/integrations/testcontainers-go/go.mod @@ -3,11 +3,12 @@ module github.com/solinode/homecloud/integrations/testcontainers-go go 1.25.0 require ( - github.com/aws/aws-sdk-go-v2 v1.47.1 + github.com/aws/aws-sdk-go-v2 v1.47.3 github.com/aws/aws-sdk-go-v2/credentials v1.20.7 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 + github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4 github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 github.com/moby/moby/api v1.55.0 @@ -20,15 +21,15 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect - github.com/aws/smithy-go v1.28.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/smithy-go v1.28.5 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect diff --git a/integrations/testcontainers-go/go.sum b/integrations/testcontainers-go/go.sum index 2889f67d..8e49eb83 100644 --- a/integrations/testcontainers-go/go.sum +++ b/integrations/testcontainers-go/go.sum @@ -6,16 +6,16 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= -github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= github.com/aws/aws-sdk-go-v2/credentials v1.20.7 h1:mit8rRVnmt+wvT4U6putiWSlwe3eKD2n6ElgyfcP/Mw= github.com/aws/aws-sdk-go-v2/credentials v1.20.7/go.mod h1:/2pPi2qhPug21z6l3M0+AKUqAT1zn2MhRCtnQv8R5Zg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.1 h1:kEs6rZI/z6aD9JBIjXzaXA2UyGYqhpQrxDnmcpU9fGA= @@ -28,18 +28,20 @@ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 h1:6Hvm github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4/go.mod h1:zv2N29aiQUhG2XZNM9zgwCnAyVBdTBbcIpfNAlNmA20= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0 h1:/P13I8ouMzMIa2yY32aUh+GIh0wYbJ/auQb1KAEhB8k= github.com/aws/aws-sdk-go-v2/service/lambda v1.111.0/go.mod h1:jUmFXtUKRVCKTaKap+NgL32pmSkVehamqqMENlGMApk= github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q= github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4 h1:4bY+1jJ15fXOsGAuYjgZYrb1D6GyZ1asQnAa+aNKAns= +github.com/aws/aws-sdk-go-v2/service/s3control v1.79.4/go.mod h1:yHNx8e9O4nK766Ghzg/UXYr3SW8Co4VeKvRWU39VOAk= github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2 h1:6ZXe/lG2mbGhw10y/aiKbe/poytLoYJsdbKV1pT9+ws= github.com/aws/aws-sdk-go-v2/service/sqs v1.52.2/go.mod h1:+TDqZ1h8CLkW9ewfQkSPWHYRjm7/wDThKeDlR46qyvE= github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 h1:AWSPqynwEJV8J2pl/p7aizCf7a23/NGHR+rK/KLOxi4= github.com/aws/aws-sdk-go-v2/service/sts v1.51.2/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= -github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= -github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/integrations/testcontainers-go/homecloud.go b/integrations/testcontainers-go/homecloud.go index 89aaeabb..331c42c1 100644 --- a/integrations/testcontainers-go/homecloud.go +++ b/integrations/testcontainers-go/homecloud.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "regexp" "strconv" @@ -146,10 +147,7 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom if err != nil { return c, err } - if host == "localhost" { - host = "127.0.0.1" // keep SDKs from trying IPv6 first or bucket.localhost names - } - c.endpoint = "http://" + host + ":" + strconv.Itoa(o.port) + c.endpoint = "http://" + net.JoinHostPort(endpointHost(host), strconv.Itoa(o.port)) if err := c.waitHealthy(ctx, 30*time.Second); err != nil { return c, err } @@ -159,12 +157,26 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom return c, nil } -// EndpointURL is the URL of the HomeCloud API, e.g. http://127.0.0.1:18080: +// endpointHost names a loopback Docker host "localhost", as `homecloud aws-env` +// does: S3 Control (which the Terraform AWS provider calls for every +// aws_s3_bucket) puts the account ID in front of the endpoint's host, and +// 123456789012.127.0.0.1 does not resolve while 123456789012.localhost does. +// HomeCloud listens on 127.0.0.1 only; clients that try ::1 first fall back to +// it. S3 clients should still use path-style addressing (see AWSConfig). +func endpointHost(host string) string { + switch host { + case "localhost", "127.0.0.1", "::1": + return "localhost" + } + return host +} + +// EndpointURL is the URL of the HomeCloud API, e.g. http://localhost:18080: // what AWS_ENDPOINT_URL or an SDK's base endpoint should be set to. func (c *Container) EndpointURL() string { return c.endpoint } // Endpoint returns the API address as testcontainers.Container.Endpoint does -// ("127.0.0.1:18080", or "http://127.0.0.1:18080" for proto "http"). It +// ("localhost:18080", or "http://localhost:18080" for proto "http"). It // replaces the generic one, which looks for a mapped port and so does not work // with host networking. func (c *Container) Endpoint(_ context.Context, proto string) (string, error) { diff --git a/integrations/testcontainers-go/homecloud_test.go b/integrations/testcontainers-go/homecloud_test.go index 127cde4c..359e9c0e 100644 --- a/integrations/testcontainers-go/homecloud_test.go +++ b/integrations/testcontainers-go/homecloud_test.go @@ -13,6 +13,7 @@ import ( ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/moby/moby/client" @@ -77,6 +78,14 @@ func TestHomeCloud(t *testing.T) { t.Errorf("s3 body %q", b) } + // S3 Control puts the account ID in front of the endpoint host + // (.localhost), which is why the endpoint is not 127.0.0.1. + if _, err := s3control.NewFromConfig(cfg).ListTagsForResource(ctx, &s3control.ListTagsForResourceInput{ + AccountId: aws.String(hc.AccountID()), ResourceArn: aws.String("arn:aws:s3:::tc-go"), + }); err != nil { + t.Fatalf("s3control list tags (endpoint %s): %v", hc.EndpointURL(), err) + } + sq := sqs.NewFromConfig(cfg) q, err := sq.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("tc-go")}) if err != nil { diff --git a/integrations/testcontainers-java/README.md b/integrations/testcontainers-java/README.md index 5d3e41dd..2e020357 100644 --- a/integrations/testcontainers-java/README.md +++ b/integrations/testcontainers-java/README.md @@ -51,7 +51,7 @@ With the Testcontainers JUnit 5 extension, `@Testcontainers` plus | | | | --- | --- | | `new HomeCloudContainer()`, `new HomeCloudContainer(image)` | default image `ghcr.io/solinode/homecloud:latest` | -| `getEndpoint()`, `getEndpointUri()` | `http://127.0.0.1:18080` | +| `getEndpoint()`, `getEndpointUri()` | `http://localhost:18080` | | `getCredentials()`, `getAccessKey()`, `getSecretKey()`, `getRegion()` | root keys and region | | `getCredentialsProvider()` | an `AwsCredentialsProvider` | | `configure(builder)` | endpoint override, credentials and region on any SDK v2 client builder: `hc.configure(SnsClient.builder()).build()` | @@ -77,6 +77,12 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1`. S3 clients should use +path-style addressing (`getS3Client()` does), so bucket names need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel, nor next to a HomeCloud you run yourself on the same Docker host diff --git a/integrations/testcontainers-java/pom.xml b/integrations/testcontainers-java/pom.xml index 9616c82f..347f703b 100644 --- a/integrations/testcontainers-java/pom.xml +++ b/integrations/testcontainers-java/pom.xml @@ -125,6 +125,11 @@ sts test + + software.amazon.awssdk + s3control + test + org.slf4j slf4j-simple diff --git a/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java b/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java index 2c0fb570..b614bb6e 100644 --- a/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java +++ b/integrations/testcontainers-java/src/main/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainer.java @@ -265,17 +265,31 @@ private void captureAccount() { // Connection details /** - * The URL of the HomeCloud API, e.g. {@code http://127.0.0.1:18080}: what AWS_ENDPOINT_URL or + * The URL of the HomeCloud API, e.g. {@code http://localhost:18080}: what AWS_ENDPOINT_URL or * an SDK's endpoint override should be. * * @return the endpoint URL */ public String getEndpoint() { - String host = getHost(); - if ("localhost".equals(host)) { - host = "127.0.0.1"; // keeps SDKs from trying IPv6 first or bucket.localhost names + return "http://" + endpointHost(getHost()) + ":" + port; + } + + /** + * Names a loopback Docker host {@code localhost}, as {@code homecloud aws-env} does: S3 Control + * (which the Terraform AWS provider calls for every bucket) puts the account ID in front of + * the endpoint's host, and {@code 123456789012.127.0.0.1} does not resolve. HomeCloud listens + * on 127.0.0.1 only; Java resolves localhost to 127.0.0.1 first, and the SDK's HTTP clients + * try the next address when one refuses the connection. + */ + static String endpointHost(String host) { + switch (host) { + case "localhost": + case "127.0.0.1": + case "::1": + return "localhost"; + default: + return host; } - return "http://" + host + ":" + port; } /** diff --git a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java index 6fbb6ca5..744c829b 100644 --- a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java +++ b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/HomeCloudContainerTest.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3control.S3ControlClient; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sts.StsClient; @@ -97,6 +98,11 @@ void s3PutGet() { assertEquals("hello", s3.getObjectAsBytes(b -> b.bucket("tc-java").key("a.txt")).asUtf8String()); assertEquals(1, s3.listObjectsV2(b -> b.bucket("tc-java")).keyCount()); } + // S3 Control puts the account ID in front of the endpoint host (.localhost), + // which is why the endpoint is not 127.0.0.1. + try (S3ControlClient control = hc.configure(S3ControlClient.builder()).build()) { + control.listTagsForResource(b -> b.accountId(hc.getAccountId()).resourceArn("arn:aws:s3:::tc-java")); + } } @Test diff --git a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java index f6f465f8..e2e69d9d 100644 --- a/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java +++ b/integrations/testcontainers-java/src/test/java/io/github/solinode/homecloud/testcontainers/ParseAwsEnvTest.java @@ -20,4 +20,11 @@ void parsesQuotedExports() { assertEquals("us-east-1", env.get("AWS_REGION")); assertEquals("http://127.0.0.1:18080", env.get("AWS_ENDPOINT_URL")); } + + @Test + void namesALoopbackHostLocalhost() { + assertEquals("localhost", HomeCloudContainer.endpointHost("localhost")); + assertEquals("localhost", HomeCloudContainer.endpointHost("127.0.0.1")); + assertEquals("docker.example", HomeCloudContainer.endpointHost("docker.example")); + } } diff --git a/integrations/testcontainers-node/README.md b/integrations/testcontainers-node/README.md index 7894fba8..c5220e45 100644 --- a/integrations/testcontainers-node/README.md +++ b/integrations/testcontainers-node/README.md @@ -38,7 +38,7 @@ ESM and CommonJS, with TypeScript types. Requires Node.js 20+. | --- | --- | | `new HomeCloudContainer(image?)` | default image `ghcr.io/solinode/homecloud:latest` | | `.start()` | starts HomeCloud, waits for the API (and S3, by default), reads the credentials | -| `hc.getEndpoint()` | `http://127.0.0.1:18080` | +| `hc.getEndpoint()` | `http://localhost:18080` | | `hc.getCredentials()` | `{ accessKeyId, secretAccessKey, region }` | | `hc.getRegion()` | `us-east-1` unless configured otherwise | | `hc.getClientConfig()` | `{ endpoint, region, credentials }` for any SDK v3 client | @@ -63,6 +63,13 @@ listens on `127.0.0.1:` of the Docker host. This works on Linux (including Actions), OrbStack, and Docker Desktop with host networking turned on (Settings > Resources > Network). +The endpoint names that address `localhost`, as `homecloud aws-env` does: S3 Control puts +the account ID in front of the endpoint's host (`123456789012.localhost`), and the +Terraform AWS provider calls it for every `aws_s3_bucket`; `123456789012.127.0.0.1` does +not resolve. Clients that try `::1` first fall back to `127.0.0.1` (Node.js 20+). S3 +clients should use path-style addressing (`getS3ClientConfig()` does), so bucket names +need not resolve. + Only one HomeCloud can run per Docker host: its helper containers have fixed names (`homecloud-s3`, ...) and host ports (9500, 9501, 5500, 8053). Do not run tests that use it in parallel (with vitest, `fileParallelism: false`), nor next to a HomeCloud you diff --git a/integrations/testcontainers-node/package-lock.json b/integrations/testcontainers-node/package-lock.json index 2a0105b5..d07f44d0 100644 --- a/integrations/testcontainers-node/package-lock.json +++ b/integrations/testcontainers-node/package-lock.json @@ -15,6 +15,7 @@ "@aws-sdk/client-dynamodb": "^3.1151.0", "@aws-sdk/client-lambda": "^3.1151.0", "@aws-sdk/client-s3": "^3.1151.0", + "@aws-sdk/client-s3-control": "^3.1151.0", "@aws-sdk/client-sqs": "^3.1151.0", "@aws-sdk/client-sts": "^3.1151.0", "@types/node": "^22.0.0", @@ -108,6 +109,28 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-s3-control": { + "version": "3.1151.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3-control/-/client-s3-control-3.1151.0.tgz", + "integrity": "sha512-2DKNaTgGwkUSJ3cDpRyKCVEXaKKDqEhzSk/BUbdp4UewJUC05kwO7Z9oZrUwHrAjcLmo8ky8kZV+FCcmTHkxTw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-node": "^3.972.86", + "@aws-sdk/middleware-sdk-s3": "^3.972.79", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/middleware-apply-body-checksum": "^4.6.3", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-sqs": { "version": "3.1151.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1151.0.tgz", @@ -1881,6 +1904,21 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-apply-body-checksum": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-apply-body-checksum/-/middleware-apply-body-checksum-4.6.3.tgz", + "integrity": "sha512-c2WI+satDWRRptxuczkq0ZqHqfWm5F7GP7aMS4wAjbs9MPxFwKcaVOsiFAlAB2wzclxCiumcVeCzluJxw6qmfw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/node-http-handler": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", diff --git a/integrations/testcontainers-node/package.json b/integrations/testcontainers-node/package.json index ced9860a..382fd8dd 100644 --- a/integrations/testcontainers-node/package.json +++ b/integrations/testcontainers-node/package.json @@ -60,6 +60,7 @@ "@aws-sdk/client-dynamodb": "^3.1151.0", "@aws-sdk/client-lambda": "^3.1151.0", "@aws-sdk/client-s3": "^3.1151.0", + "@aws-sdk/client-s3-control": "^3.1151.0", "@aws-sdk/client-sqs": "^3.1151.0", "@aws-sdk/client-sts": "^3.1151.0", "@types/node": "^22.0.0", diff --git a/integrations/testcontainers-node/src/index.ts b/integrations/testcontainers-node/src/index.ts index 19dbbb94..6e0f778f 100644 --- a/integrations/testcontainers-node/src/index.ts +++ b/integrations/testcontainers-node/src/index.ts @@ -138,8 +138,7 @@ export class HomeCloudContainer extends GenericContainer { started = await super.start(); const account = this.account() ?? (await readAccount(started)); if (!account) throw new Error("homecloud: no account in the first-start log"); - const host = started.getHost() === "localhost" ? "127.0.0.1" : started.getHost(); - const endpoint = `http://${host}:${this.port}`; + const endpoint = `http://${endpointHost(started.getHost())}:${this.port}`; await waitHealthy(endpoint, 30_000); const credentials = await readCredentials(started); return new StartedHomeCloudContainer(started, endpoint, credentials, account); @@ -172,7 +171,7 @@ export class StartedHomeCloudContainer extends AbstractStartedContainer { super(startedTestContainer); } - /** The API URL, e.g. http://127.0.0.1:18080: what AWS_ENDPOINT_URL should be. */ + /** The API URL, e.g. http://localhost:18080: what AWS_ENDPOINT_URL should be. */ getEndpoint(): string { return this.endpoint; } @@ -274,6 +273,17 @@ export async function removeResources(account: string): Promise { } } +/** + * The host name for the endpoint: a loopback Docker host is named `localhost`, as + * `homecloud aws-env` does. S3 Control (which the Terraform AWS provider calls for every + * bucket) puts the account ID in front of the endpoint's host, and `123456789012.127.0.0.1` + * does not resolve. HomeCloud listens on 127.0.0.1 only; Node.js 20+ tries 127.0.0.1 when + * ::1 refuses the connection. + */ +export function endpointHost(host: string): string { + return host === "localhost" || host === "127.0.0.1" || host === "::1" ? "localhost" : host; +} + /** Parses the `export NAME="value"` lines printed by `homecloud aws-env`. */ export function parseAwsEnv(output: string): Record { const env: Record = {}; diff --git a/integrations/testcontainers-node/test/homecloud.test.ts b/integrations/testcontainers-node/test/homecloud.test.ts index 8c7fbe95..16c22020 100644 --- a/integrations/testcontainers-node/test/homecloud.test.ts +++ b/integrations/testcontainers-node/test/homecloud.test.ts @@ -1,11 +1,12 @@ import { CreateTableCommand, DynamoDBClient, GetItemCommand, PutItemCommand } from "@aws-sdk/client-dynamodb"; import { LambdaClient, ListFunctionsCommand } from "@aws-sdk/client-lambda"; import { CreateBucketCommand, GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { ListTagsForResourceCommand, S3ControlClient } from "@aws-sdk/client-s3-control"; import { CreateQueueCommand, ReceiveMessageCommand, SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { GetCallerIdentityCommand, STSClient } from "@aws-sdk/client-sts"; import { getContainerRuntimeClient } from "testcontainers"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { DEFAULT_IMAGE, HomeCloudContainer, parseAwsEnv, type StartedHomeCloudContainer } from "../src/index.js"; +import { DEFAULT_IMAGE, endpointHost, HomeCloudContainer, parseAwsEnv, type StartedHomeCloudContainer } from "../src/index.js"; // A local build (docker build -t homecloud:test .) or the official image. const IMAGE = process.env.HOMECLOUD_IMAGE || DEFAULT_IMAGE; @@ -17,6 +18,14 @@ describe("parseAwsEnv", () => { }); }); +describe("endpointHost", () => { + it("names a loopback host localhost", () => { + expect(endpointHost("localhost")).toBe("localhost"); + expect(endpointHost("127.0.0.1")).toBe("localhost"); + expect(endpointHost("docker.example")).toBe("docker.example"); + }); +}); + async function expectNoLeftovers(account: string): Promise { const docker = (await getContainerRuntimeClient()).container.dockerode; const filters = JSON.stringify({ label: [`homecloud.account=${account}`] }); @@ -68,6 +77,12 @@ describe("HomeCloudContainer", () => { await s3.send(new PutObjectCommand({ Bucket: "tc-node", Key: "a.txt", Body: "hello" })); const obj = await s3.send(new GetObjectCommand({ Bucket: "tc-node", Key: "a.txt" })); expect(await obj.Body?.transformToString()).toBe("hello"); + // S3 Control puts the account ID in front of the endpoint host (.localhost), + // which is why the endpoint is not 127.0.0.1. + const control = new S3ControlClient(hc.getClientConfig()); + await control.send( + new ListTagsForResourceCommand({ AccountId: hc.getAccountId(), ResourceArn: "arn:aws:s3:::tc-node" }), + ); }); it("SQS", async () => { diff --git a/integrations/testcontainers-python/README.md b/integrations/testcontainers-python/README.md index ec2704b8..59b3e12d 100644 --- a/integrations/testcontainers-python/README.md +++ b/integrations/testcontainers-python/README.md @@ -31,7 +31,7 @@ def test_upload(homecloud): | | | | --- | --- | | `HomeCloudContainer(image=DEFAULT_IMAGE, port=18080, docker_socket="/var/run/docker.sock", wait_for_services=("s3",), startup_timeout=180)` | `wait_for_services` takes `s3`, `ecr`, `route53`; everything else is ready with the API | -| `get_endpoint()` | `http://127.0.0.1:18080` | +| `get_endpoint()` | `http://localhost:18080` | | `get_credentials()` | `Credentials(access_key_id, secret_access_key, region)` | | `get_client(service, **kw)` / `get_resource(service, **kw)` | boto3 client / resource for HomeCloud | | `aws_env()` | `AWS_ENDPOINT_URL`, keys and region for subprocesses (AWS CLI, Terraform) | @@ -46,6 +46,12 @@ gets the Docker socket and **host networking**; the API listens on `127.0.0.1:

None: # Connection details def get_endpoint(self) -> str: - """The API URL, e.g. http://127.0.0.1:18080 (what AWS_ENDPOINT_URL should be).""" + """The API URL, e.g. http://localhost:18080 (what AWS_ENDPOINT_URL should be). + + A loopback Docker host is named ``localhost``, as ``homecloud aws-env`` + does: S3 Control (which the Terraform AWS provider calls for every + bucket) puts the account ID in front of the endpoint's host, and + ``123456789012.127.0.0.1`` does not resolve. HomeCloud listens on + 127.0.0.1 only; clients that try ::1 first fall back to it. + """ host = self.get_container_host_ip() - if host == "localhost": - host = "127.0.0.1" + if host in ("localhost", "127.0.0.1", "::1"): + host = "localhost" return f"http://{host}:{self.port}" def get_credentials(self) -> Credentials: diff --git a/integrations/testcontainers-python/tests/test_homecloud.py b/integrations/testcontainers-python/tests/test_homecloud.py index c792ce32..aa52eb6d 100644 --- a/integrations/testcontainers-python/tests/test_homecloud.py +++ b/integrations/testcontainers-python/tests/test_homecloud.py @@ -47,6 +47,10 @@ def test_s3(homecloud): assert s3.get_object(Bucket="tc-python", Key="a.txt")["Body"].read() == b"hello" bucket = homecloud.get_resource("s3").Bucket("tc-python") assert [o.key for o in bucket.objects.all()] == ["a.txt"] + # S3 Control puts the account ID in front of the endpoint host + # (.localhost), which is why the endpoint is not 127.0.0.1. + control = homecloud.get_client("s3control") + control.list_tags_for_resource(AccountId=homecloud.account_id, ResourceArn="arn:aws:s3:::tc-python") def test_sqs(homecloud): diff --git a/integrations/testcontainers-python/tests/test_options.py b/integrations/testcontainers-python/tests/test_options.py index 4185beba..1a695244 100644 --- a/integrations/testcontainers-python/tests/test_options.py +++ b/integrations/testcontainers-python/tests/test_options.py @@ -26,3 +26,14 @@ def test_command_and_socket(): hc = make(port=19090, docker_socket="/run/user/1000/docker.sock") assert hc._command == ["serve", "--data-dir", "/data", "--addr", "127.0.0.1:19090"] assert hc.volumes["/run/user/1000/docker.sock"]["bind"] == "/var/run/docker.sock" + + +def test_endpoint_names_loopback_localhost(): + for host, want in [ + ("localhost", "http://localhost:18080"), + ("127.0.0.1", "http://localhost:18080"), + ("docker.example", "http://docker.example:18080"), + ]: + hc = make() + with mock.patch.object(HomeCloudContainer, "get_container_host_ip", return_value=host): + assert hc.get_endpoint() == want