From db4613246ebf3fb4f3d816c3d830b0af9de3cc2c Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:40:38 +0530 Subject: [PATCH 1/2] Fixes for cdk bootstrap/deploy, Pulumi and the Go SDK - IAM: add the AWSCloudFormationReadOnlyAccess managed policy; allow 128-character policy, group and instance profile names; set aws:ResourceAccount for users and role sessions. - Lambda: support LoggingConfig (log group, format, levels) over the AWS API and in CloudFormation; add nodejs24.x, python3.14 and java25. - CloudFormation: DeleteChangeSet of a missing change set succeeds when the stack exists, as in AWS. - awsJson responses carry X-Amz-Crc32. - The self-signed certificate covers *.localhost and *.. - Logs, metrics and the Lambda reaper only see this installation's containers. --- cli/internal/awsapi/awsapi.go | 22 +++++-- cli/internal/awsapi/crc32_test.go | 28 +++++++++ cli/internal/dockertest/dockertest_test.go | 35 +++++++++++ cli/internal/runtime/docker.go | 16 +++++ cli/internal/server/tls.go | 8 ++- cli/internal/server/tls_test.go | 14 +++++ cli/internal/svc/cfn/aws.go | 10 ++++ cli/internal/svc/cfn/aws_test.go | 6 ++ cli/internal/svc/cfn/awstypes_lambda.go | 10 ++++ cli/internal/svc/cfn/awstypes_lambda_test.go | 4 ++ cli/internal/svc/cfn/awstypes_update.go | 9 +-- cli/internal/svc/cloudwatch/logs.go | 4 +- cli/internal/svc/cloudwatch/metrics.go | 2 +- cli/internal/svc/iam/iam.go | 19 ++++-- cli/internal/svc/iam/policy.go | 1 + cli/internal/svc/iam/policy_test.go | 29 +++++++++ cli/internal/svc/iam/resourceaccount_test.go | 29 +++++++++ cli/internal/svc/iam/roles.go | 3 +- cli/internal/svc/lambda/aws.go | 13 ++-- cli/internal/svc/lambda/engine.go | 6 +- cli/internal/svc/lambda/lambda.go | 62 ++++++++++++++++++++ cli/internal/svc/lambda/lambda_test.go | 32 +++++++++- cli/internal/svc/lambda/runtimes.go | 6 +- 23 files changed, 336 insertions(+), 32 deletions(-) create mode 100644 cli/internal/awsapi/crc32_test.go create mode 100644 cli/internal/svc/iam/resourceaccount_test.go diff --git a/cli/internal/awsapi/awsapi.go b/cli/internal/awsapi/awsapi.go index 2a25126b..4f431aa5 100644 --- a/cli/internal/awsapi/awsapi.go +++ b/cli/internal/awsapi/awsapi.go @@ -20,10 +20,12 @@ import ( "encoding/json" "errors" "fmt" + "hash/crc32" "io" "log" "net/http" "net/url" + "strconv" "strings" "sync" "time" @@ -572,12 +574,22 @@ func (q *Req) fail(err error) { } func (q *Req) writeJSON(status int, v any) { - q.W.Header().Set("Content-Type", "application/x-amz-json-"+q.Svc.JSONVersion) - q.W.WriteHeader(status) if v == nil { v = struct{}{} } - _ = json.NewEncoder(q.W).Encode(v) + q.writeJSONBody(status, "application/x-amz-json-"+q.Svc.JSONVersion, v) +} + +// writeJSONBody writes an awsJson response with its X-Amz-Crc32 header: the +// AWS SDK for Go v2 checks DynamoDB response bodies against it and, when the +// header is missing, compares them with 0 and logs a warning on every call. +func (q *Req) writeJSONBody(status int, contentType string, v any) { + var buf bytes.Buffer + _ = json.NewEncoder(&buf).Encode(v) + q.W.Header().Set("Content-Type", contentType) + q.W.Header().Set("X-Amz-Crc32", strconv.FormatUint(uint64(crc32.ChecksumIEEE(buf.Bytes())), 10)) + q.W.WriteHeader(status) + _, _ = q.W.Write(buf.Bytes()) } func (q *Req) writeJSONError(e *Error) { @@ -593,13 +605,11 @@ func (q *Req) writeJSONError(e *Error) { if q.Svc != nil { ver = q.Svc.JSONVersion } - q.W.Header().Set("Content-Type", "application/x-amz-json-"+ver) - q.W.WriteHeader(e.Status) body := map[string]any{"__type": e.Code, "message": e.Message} for k, v := range e.Fields { body[k] = v } - _ = json.NewEncoder(q.W).Encode(body) + q.writeJSONBody(e.Status, "application/x-amz-json-"+ver, body) } func (q *Req) writeRESTJSONError(e *Error) { diff --git a/cli/internal/awsapi/crc32_test.go b/cli/internal/awsapi/crc32_test.go new file mode 100644 index 00000000..79c8ecc5 --- /dev/null +++ b/cli/internal/awsapi/crc32_test.go @@ -0,0 +1,28 @@ +package awsapi + +import ( + "hash/crc32" + "net/http/httptest" + "strconv" + "testing" +) + +// awsJson responses carry X-Amz-Crc32 of the body, as DynamoDB's do: the Go v2 +// SDK validates DynamoDB bodies against it (and warns on every call without it). +func TestJSONResponsesCarryCRC32(t *testing.T) { + for name, write := range map[string]func(q *Req){ + "result": func(q *Req) { q.writeJSON(200, map[string]any{"Item": map[string]any{"id": map[string]string{"S": "1"}}}) }, + "empty": func(q *Req) { q.writeJSON(200, nil) }, + "error": func(q *Req) { q.writeJSONError(&Error{Status: 400, Code: "ResourceNotFoundException", Message: "no table"}) }, + } { + w := httptest.NewRecorder() + write(&Req{W: w, Svc: &Service{Name: "dynamodb", JSONVersion: "1.0"}}) + want := strconv.FormatUint(uint64(crc32.ChecksumIEEE(w.Body.Bytes())), 10) + if got := w.Header().Get("X-Amz-Crc32"); got != want || w.Body.Len() == 0 { + t.Errorf("%s: X-Amz-Crc32 %q, body CRC %s (%q)", name, got, want, w.Body) + } + if ct := w.Header().Get("Content-Type"); ct != "application/x-amz-json-1.0" { + t.Errorf("%s: Content-Type %q", name, ct) + } + } +} diff --git a/cli/internal/dockertest/dockertest_test.go b/cli/internal/dockertest/dockertest_test.go index eb34679f..230442dd 100644 --- a/cli/internal/dockertest/dockertest_test.go +++ b/cli/internal/dockertest/dockertest_test.go @@ -1,6 +1,7 @@ package dockertest import ( + "context" "os/exec" "testing" "time" @@ -108,3 +109,37 @@ func TestRemoveOnlyMatchingAccount(t *testing.T) { t.Error("Remove deleted another test account's volume") } } + +// OwnContainers leaves out containers of other installations and of other +// tests on the same Docker host (log groups, metrics and the Lambda reaper use it). +func TestOwnContainers(t *testing.T) { + d := Start(t) + other := NewAccount() + t.Cleanup(func() { Remove(d, other, t.Logf) }) + if err := d.EnsureImage(context.Background(), "alpine:3.20"); err != nil { + t.Skip("cannot pull alpine: ", err) + } + mk := func(name string, labels map[string]string) string { + t.Helper() + c, err := d.C.CreateContainer(docker.CreateContainerOptions{Name: name, Config: &docker.Config{Image: "alpine:3.20", Labels: labels}}) + if err != nil { + t.Fatal(err) + } + return c.ID + } + suffix := core.RandHex(4) + mine := mk("hctest-own-"+suffix, runtime.Labels("lambda", "fn", nil)) + theirs := mk("hctest-theirs-"+suffix, map[string]string{core.LabelManaged: "true", core.LabelService: "lambda", + core.LabelResource: "fn", core.LabelAccount: other}) + cs, err := d.OwnContainers() + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, c := range cs { + seen[c.ID] = true + } + if !seen[mine] || seen[theirs] { + t.Fatalf("OwnContainers: mine %v, other account's %v", seen[mine], seen[theirs]) + } +} diff --git a/cli/internal/runtime/docker.go b/cli/internal/runtime/docker.go index 4976625d..b0d1febe 100644 --- a/cli/internal/runtime/docker.go +++ b/cli/internal/runtime/docker.go @@ -451,6 +451,22 @@ func (d *Docker) ManagedContainers() ([]docker.APIContainers, error) { return d.C.ListContainers(docker.ListContainersOptions{All: true, Filters: map[string][]string{"label": {core.LabelManaged + "=true"}}}) } +// OwnContainers lists the managed containers of this installation (see Mine), +// leaving out those of another account or a running test on the same host. +func (d *Docker) OwnContainers() ([]docker.APIContainers, error) { + cs, err := d.ManagedContainers() + if err != nil { + return nil, err + } + out := cs[:0] + for _, c := range cs { + if Mine(c.Labels) { + out = append(out, c) + } + } + return out, nil +} + // ---- stats ---- type Usage struct { diff --git a/cli/internal/server/tls.go b/cli/internal/server/tls.go index 93a57514..87fe2c7f 100644 --- a/cli/internal/server/tls.go +++ b/cli/internal/server/tls.go @@ -39,13 +39,15 @@ func SelfSignedCert(dataDir, publicHost string) (string, string, error) { ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, IsCA: true, - DNSNames: []string{"localhost"}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + // S3 Control (which the Terraform and Pulumi AWS providers call for bucket + // tags) puts the account ID in front of the host: 123456789012.localhost. + DNSNames: []string{"localhost", "*.localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, } if ip := net.ParseIP(publicHost); ip != nil { tmpl.IPAddresses = append(tmpl.IPAddresses, ip) } else if publicHost != "" && publicHost != "localhost" { - tmpl.DNSNames = append(tmpl.DNSNames, publicHost) + tmpl.DNSNames = append(tmpl.DNSNames, publicHost, "*."+publicHost) } der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) if err != nil { diff --git a/cli/internal/server/tls_test.go b/cli/internal/server/tls_test.go index ea1db12e..aacd3e7a 100644 --- a/cli/internal/server/tls_test.go +++ b/cli/internal/server/tls_test.go @@ -1,7 +1,9 @@ package server import ( + "crypto/x509" "encoding/json" + "encoding/pem" "net" "net/http" "os" @@ -40,6 +42,18 @@ func TestSelfSignedTLS(t *testing.T) { if err := c.Do("GET", "/", nil, &out); err != nil || !out["ok"] { t.Fatalf("https call through the trusted self-signed cert failed: %v %v", out, err) } + // Account-prefixed S3 Control hosts are covered too. + pemBytes, _ := os.ReadFile(cert) + blk, _ := pem.Decode(pemBytes) + parsed, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + t.Fatal(err) + } + for _, h := range []string{"localhost", "123456789012.localhost", "homelab.local", "123456789012.homelab.local", "127.0.0.1"} { + if err := parsed.VerifyHostname(h); err != nil { + t.Errorf("%s: %v", h, err) + } + } // A second call must reuse the existing certificate rather than regenerate it. cert2, _, _ := SelfSignedCert(dir, "homelab.local") if cert2 != cert { diff --git a/cli/internal/svc/cfn/aws.go b/cli/internal/svc/cfn/aws.go index 5048c391..3c471fe3 100644 --- a/cli/internal/svc/cfn/aws.go +++ b/cli/internal/svc/cfn/aws.go @@ -876,7 +876,17 @@ func (s *Service) awsExecuteChangeSet(q *awsapi.Req) (any, error) { } func (s *Service) awsDeleteChangeSet(q *awsapi.Req) (any, error) { + ref := q.Param("ChangeSetName") cs, err := s.csAuth(q, "cloudformation:DeleteChangeSet") + var ce *core.Error + if err != nil && errors.As(err, &ce) && ce.Code == "ChangeSetNotFound" && ref != "" && !strings.HasPrefix(ref, "arn:") { + // As in AWS, deleting a change set by name succeeds when the stack + // exists but the change set does not: cdk diff and cdk deploy + // clear their change set name this way before creating it. + if _, serr := s.authStack(q, "cloudformation:DeleteChangeSet", "StackName"); serr == nil { + return nil, nil + } + } if err != nil { return nil, err } diff --git a/cli/internal/svc/cfn/aws_test.go b/cli/internal/svc/cfn/aws_test.go index 9cfcd6a5..2de9f5bc 100644 --- a/cli/internal/svc/cfn/aws_test.go +++ b/cli/internal/svc/cfn/aws_test.go @@ -523,6 +523,12 @@ Outputs: if o, err := e.AWSErr(t, "cloudformation", "describe-change-set", "--stack-name", "dep", "--change-set-name", "preview"); err == nil || !strings.Contains(o, "ChangeSetNotFound") { t.Fatalf("deleted change set: %v %s", err, o) } + // Deleting a missing change set by name succeeds while the stack exists (cdk diff relies on it), + // and fails for a missing stack. + e.AWS(t, "cloudformation", "delete-change-set", "--stack-name", "dep", "--change-set-name", "preview") + if o, err := e.AWSErr(t, "cloudformation", "delete-change-set", "--stack-name", "nostack", "--change-set-name", "preview"); err == nil { + t.Fatalf("delete change set of a missing stack: %s", o) + } // An identical template makes a FAILED change set that cannot be executed. e.AWS(t, "cloudformation", "create-change-set", "--stack-name", "dep", "--change-set-name", "same", "--template-body", "file://"+tf, "--parameters", "ParameterKey=Retention,ParameterValue=120") d = e.AWSJSON(t, "cloudformation", "describe-change-set", "--stack-name", "dep", "--change-set-name", "same") diff --git a/cli/internal/svc/cfn/awstypes_lambda.go b/cli/internal/svc/cfn/awstypes_lambda.go index e6202971..6cb15225 100644 --- a/cli/internal/svc/cfn/awstypes_lambda.go +++ b/cli/internal/svc/cfn/awstypes_lambda.go @@ -172,6 +172,16 @@ func lambdaFunctionProps(x *xctx, in map[string]any) (map[string]any, error) { if dl := mv(in, "DeadLetterConfig"); dl != nil && has(dl, "TargetArn") { out["dead_letter_target"] = sv(dl, "TargetArn") } + // CDK's `logGroup` prop sets LoggingConfig.LogGroup. + if lc := mv(in, "LoggingConfig"); lc != nil { + cfg := map[string]any{} + for _, k := range []string{"LogFormat", "LogGroup", "ApplicationLogLevel", "SystemLogLevel"} { + if has(lc, k) { + cfg[k] = sv(lc, k) + } + } + out["logging_config"] = cfg + } return out, nil } diff --git a/cli/internal/svc/cfn/awstypes_lambda_test.go b/cli/internal/svc/cfn/awstypes_lambda_test.go index 44513874..db5837b3 100644 --- a/cli/internal/svc/cfn/awstypes_lambda_test.go +++ b/cli/internal/svc/cfn/awstypes_lambda_test.go @@ -115,6 +115,7 @@ Resources: Variables: {STAGE: prod, N: 3} Tags: - {Key: team, Value: platform} + LoggingConfig: {LogGroup: fnstack-logs} Code: ZipFile: | def handler(event, context): @@ -203,6 +204,9 @@ func TestLambdaStack(t *testing.T) { !strings.HasSuffix(cfg["Role"].(string), ":role/"+lastPart(cfg["Role"].(string))) { t.Fatalf("configuration: %v", cfg) } + if lc, _ := cfg["LoggingConfig"].(map[string]any); lc["LogGroup"] != "fnstack-logs" || lc["LogFormat"] != "Text" { + t.Fatalf("LoggingConfig: %v", cfg["LoggingConfig"]) + } vars := cfg["Environment"].(map[string]any)["Variables"].(map[string]any) if vars["STAGE"] != "prod" || vars["N"] != "3" { t.Fatalf("environment: %v", vars) diff --git a/cli/internal/svc/cfn/awstypes_update.go b/cli/internal/svc/cfn/awstypes_update.go index a65837a2..8606953a 100644 --- a/cli/internal/svc/cfn/awstypes_update.go +++ b/cli/internal/svc/cfn/awstypes_update.go @@ -450,7 +450,7 @@ func init() { // ---- Lambda ---- configure("AWS::Lambda::Function", "FunctionName", []string{"Code", "Runtime", "Handler", "Description", "Timeout", "MemorySize", "Environment", "Role", - "Layers", "Architectures", "Tags", "VpcConfig", "DeadLetterConfig", "ImageConfig", "ReservedConcurrentExecutions"}, + "Layers", "Architectures", "Tags", "VpcConfig", "DeadLetterConfig", "ImageConfig", "ReservedConcurrentExecutions", "LoggingConfig"}, func(x *xctx, r *Resource, old, in map[string]any) (map[string]any, error) { np, err := translate(x, "AWS::Lambda::Function", r, in) if err != nil { @@ -472,10 +472,10 @@ func init() { } } if differs(old, in, "Runtime", "Handler", "Description", "Timeout", "MemorySize", "Environment", "Role", "Layers", "Architectures", "Tags", "VpcConfig", - "DeadLetterConfig", "ImageConfig") { + "DeadLetterConfig", "ImageConfig", "LoggingConfig") { cfg := map[string]any{} for _, k := range []string{"runtime", "handler", "description", "timeout_seconds", "memory_mb", "environment", "role", "layers", "architectures", "tags", - "subnet_id", "security_group_ids", "dead_letter_target", "image_config"} { + "subnet_id", "security_group_ids", "dead_letter_target", "image_config", "logging_config"} { if v, ok := np[k]; ok { cfg[k] = v } @@ -483,7 +483,8 @@ func init() { // what the template dropped goes back to the default for k, reset := range map[string]map[string]any{"Description": {"description": ""}, "Timeout": {"timeout_seconds": 3}, "MemorySize": {"memory_mb": 128}, "Environment": {"environment": map[string]string{}}, "Layers": {"layers": []string{}}, "Tags": {"tags": core.Tags{}}, - "VpcConfig": {"subnet_id": "", "security_group_ids": []string{}}, "DeadLetterConfig": {"dead_letter_target": ""}} { + "VpcConfig": {"subnet_id": "", "security_group_ids": []string{}}, "DeadLetterConfig": {"dead_letter_target": ""}, + "LoggingConfig": {"logging_config": map[string]any{}}} { if has(old, k) && !has(in, k) { for rk, rv := range reset { cfg[rk] = rv diff --git a/cli/internal/svc/cloudwatch/logs.go b/cli/internal/svc/cloudwatch/logs.go index ef4f4f3c..80344875 100644 --- a/cli/internal/svc/cloudwatch/logs.go +++ b/cli/internal/svc/cloudwatch/logs.go @@ -494,7 +494,7 @@ func (s *Service) containerGroups() []LogGroup { if s.env.Docker == nil { return out } - cs, err := s.env.Docker.ManagedContainers() + cs, err := s.env.Docker.OwnContainers() if err != nil { return out } @@ -515,7 +515,7 @@ func (s *Service) containerFor(group string) string { return "" } svcName, res, _ := strings.Cut(rest, "/") - cs, err := s.env.Docker.ManagedContainers() + cs, err := s.env.Docker.OwnContainers() if err != nil { return "" } diff --git a/cli/internal/svc/cloudwatch/metrics.go b/cli/internal/svc/cloudwatch/metrics.go index 776e1d79..ec792f4d 100644 --- a/cli/internal/svc/cloudwatch/metrics.go +++ b/cli/internal/svc/cloudwatch/metrics.go @@ -287,7 +287,7 @@ func (s *Service) collect(ctx context.Context) { if s.env.Docker == nil { return } - cs, err := s.env.Docker.ManagedContainers() + cs, err := s.env.Docker.OwnContainers() if err != nil { log.Printf("cloudwatch: list containers: %v", err) return diff --git a/cli/internal/svc/iam/iam.go b/cli/internal/svc/iam/iam.go index e656395a..26282256 100644 --- a/cli/internal/svc/iam/iam.go +++ b/cli/internal/svc/iam/iam.go @@ -408,6 +408,8 @@ func (s *Service) userContext(u User) CondContext { c := CondContext{ "aws:username": {u.Name}, "aws:userid": {u.ID}, "aws:principalarn": {u.ARN}, "aws:principalaccount": {s.env.AccountID}, "aws:principaltype": {typ}, + // Every resource a server holds is in its one account. + "aws:resourceaccount": {s.env.AccountID}, } for k, v := range u.Tags { c["aws:principaltag/"+strings.ToLower(k)] = []string{v} @@ -474,13 +476,22 @@ func (s *Service) boundaryDoc(arn string) *PolicyDocument { } var ( - nameRe = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`) - pathRe = regexp.MustCompile(`^/([\x21-\x7E]+/)?$`) + nameRe = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`) + longNameRe = regexp.MustCompile(`^[\w+=,.@-]{1,128}$`) + pathRe = regexp.MustCompile(`^/([\x21-\x7E]+/)?$`) ) +// validName checks an IAM name. As in AWS, user and role names allow 64 +// characters; policy (managed and inline), group and instance profile names +// allow 128 (the CDK bootstrap stack names an inline policy with 72). func validName(kind, n string) error { - if !nameRe.MatchString(n) { - return core.Errf(http.StatusBadRequest, "ValidationError", "%s name %q is invalid: use 1-64 letters, digits or +=,.@_-", kind, n) + re, max := nameRe, 64 + switch kind { + case "policy", "group", "instance profile": + re, max = longNameRe, 128 + } + if !re.MatchString(n) { + return core.Errf(http.StatusBadRequest, "ValidationError", "%s name %q is invalid: use 1-%d letters, digits or +=,.@_-", kind, n, max) } return nil } diff --git a/cli/internal/svc/iam/policy.go b/cli/internal/svc/iam/policy.go index e8473906..908f7f2f 100644 --- a/cli/internal/svc/iam/policy.go +++ b/cli/internal/svc/iam/policy.go @@ -509,6 +509,7 @@ var builtinPolicies = []builtinPolicy{ {"ElasticLoadBalancingFullAccess", "/", "Full access to load balancers and target groups.", doc("Allow", "elasticloadbalancing:*")}, {"AmazonElasticFileSystemFullAccess", "/", "Full access to shared file systems.", doc("Allow", "elasticfilesystem:*")}, {"AmazonAPIGatewayAdministrator", "/", "Full access to HTTP APIs.", doc("Allow", "apigateway:*")}, + {"AWSCloudFormationReadOnlyAccess", "/", "Read access to stacks, change sets and templates.", doc("Allow", "cloudformation:Describe*", "cloudformation:EstimateTemplateCost", "cloudformation:Get*", "cloudformation:List*", "cloudformation:ValidateTemplate", "cloudformation:Detect*")}, {"AWSCloudFormationFullAccess", "/", "Full access to stacks (resources are created with the caller's own permissions).", doc("Allow", "cloudformation:*")}, {"AmazonCognitoPowerUser", "/", "Full access to user pools, app clients and pool users.", doc("Allow", "cognito-idp:*", "cognito-identity:*", "cognito-sync:*")}, {"AWSStepFunctionsFullAccess", "/", "Full access to state machines and executions.", doc("Allow", "states:*")}, diff --git a/cli/internal/svc/iam/policy_test.go b/cli/internal/svc/iam/policy_test.go index 2f0e0a5b..58d6d5f1 100644 --- a/cli/internal/svc/iam/policy_test.go +++ b/cli/internal/svc/iam/policy_test.go @@ -2,6 +2,7 @@ package iam import ( "encoding/json" + "strings" "testing" "time" ) @@ -86,6 +87,28 @@ func TestValidate(t *testing.T) { mustDoc(t, `{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:*","Resource":"*"}}`) } +// Name limits follow AWS: users and roles 64 characters, policies, groups and +// instance profiles 128. cdk bootstrap creates a 72-character inline policy name. +func TestValidNameLimits(t *testing.T) { + cdk := "cdk-hnb659fds-file-publishing-role-default-policy-123456789012-us-east-1" + if err := validName("policy", cdk); err != nil { + t.Errorf("policy %s: %v", cdk, err) + } + if validName("policy", strings.Repeat("p", 129)) == nil { + t.Error("129-character policy name accepted") + } + for _, k := range []string{"group", "instance profile"} { + if err := validName(k, strings.Repeat("g", 128)); err != nil { + t.Errorf("%s: %v", k, err) + } + } + for _, k := range []string{"user", "role"} { + if validName(k, strings.Repeat("u", 65)) == nil { + t.Errorf("65-character %s name accepted", k) + } + } +} + func TestReadOnlyExcludesSecrets(t *testing.T) { bp, ok := builtin("ReadOnlyAccess") if !ok { @@ -118,6 +141,12 @@ func TestBuiltinsValid(t *testing.T) { t.Errorf("legacy name %s is also a built-in", old) } } + // cdk bootstrap attaches this one to its deployment action role. + cf, ok := builtin("AWSCloudFormationReadOnlyAccess") + if !ok || evaluate([]PolicyDocument{cf.Doc}, "cloudformation:DescribeStacks", "*", nil) != allow || + evaluate([]PolicyDocument{cf.Doc}, "cloudformation:CreateStack", "*", nil) != implicitDeny { + t.Error("AWSCloudFormationReadOnlyAccess missing or wrong") + } pu, _ := builtin("PowerUserAccess") if evaluate([]PolicyDocument{pu.Doc}, "iam:CreateUser", "*", nil) != implicitDeny || evaluate([]PolicyDocument{pu.Doc}, "s3:PutObject", "*", nil) != allow || diff --git a/cli/internal/svc/iam/resourceaccount_test.go b/cli/internal/svc/iam/resourceaccount_test.go new file mode 100644 index 00000000..ea4745ee --- /dev/null +++ b/cli/internal/svc/iam/resourceaccount_test.go @@ -0,0 +1,29 @@ +package iam + +import ( + "testing" + + "github.com/homecloudhq/homecloud/cli/internal/svc/svctest" +) + +// The CDK bootstrap stack scopes its publishing roles' policies with +// aws:ResourceAccount; every resource a server holds is in its account. +func TestResourceAccountConditionKey(t *testing.T) { + env := svctest.Env(t) + s := New(env) + d := mustDoc(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetBucket*", + "Resource":"arn:aws:s3:::assets","Condition":{"StringEquals":{"aws:ResourceAccount":["`+env.AccountID+`"]}}}]}`) + other := mustDoc(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetBucket*", + "Resource":"arn:aws:s3:::assets","Condition":{"StringEquals":{"aws:ResourceAccount":"999999999999"}}}]}`) + for name, ctx := range map[string]CondContext{ + "role": s.roleContext(Role{Name: "publish", ARN: "arn:aws:iam::" + env.AccountID + ":role/publish"}, tempCred{SessionName: "cdk"}), + "user": s.userContext(User{Name: "dev", ARN: "arn:aws:iam::" + env.AccountID + ":user/dev"}), + } { + if evaluate([]PolicyDocument{d}, "s3:GetBucketLocation", "arn:aws:s3:::assets", ctx) != allow { + t.Errorf("%s: aws:ResourceAccount does not match the server's account", name) + } + if evaluate([]PolicyDocument{other}, "s3:GetBucketLocation", "arn:aws:s3:::assets", ctx) != implicitDeny { + t.Errorf("%s: aws:ResourceAccount matches another account", name) + } + } +} diff --git a/cli/internal/svc/iam/roles.go b/cli/internal/svc/iam/roles.go index 0db01de6..a2b7e9dd 100644 --- a/cli/internal/svc/iam/roles.go +++ b/cli/internal/svc/iam/roles.go @@ -132,7 +132,8 @@ func (s *Service) roleContext(r Role, t tempCred) CondContext { c := CondContext{ "aws:userid": {r.ID + ":" + t.SessionName}, "aws:principalarn": {r.ARN}, "aws:principalaccount": {s.env.AccountID}, "aws:principaltype": {"AssumedRole"}, - "aws:tokenissuetime": {t.Expires.Add(-time.Hour).UTC().Format(time.RFC3339)}, + "aws:resourceaccount": {s.env.AccountID}, // every resource is in the server's one account + "aws:tokenissuetime": {t.Expires.Add(-time.Hour).UTC().Format(time.RFC3339)}, } if t.Issued != nil { c["aws:tokenissuetime"] = []string{t.Issued.UTC().Format(time.RFC3339)} diff --git a/cli/internal/svc/lambda/aws.go b/cli/internal/svc/lambda/aws.go index 513d1048..8c3eb0b5 100644 --- a/cli/internal/svc/lambda/aws.go +++ b/cli/internal/svc/lambda/aws.go @@ -216,7 +216,7 @@ func (s *Service) functionConfig(f Function) map[string]any { "State": f.State, "LastUpdateStatus": f.LastUpdateStatus, "PackageType": f.PackageType, "Architectures": f.Architectures, "EphemeralStorage": map[string]int{"Size": 512}, "SnapStart": map[string]string{"ApplyOn": "None", "OptimizationStatus": "Off"}, - "LoggingConfig": map[string]string{"LogFormat": "Text", "LogGroup": f.LogGroup}, + "LoggingConfig": f.logging(), } if c["PackageType"] == "" { c["PackageType"] = "Zip" @@ -288,15 +288,16 @@ type awsConfigIn struct { DeadLetterConfig *struct { TargetArn string `json:"TargetArn"` } `json:"DeadLetterConfig"` - Layers *[]string `json:"Layers"` - ImageConfig *ImageConfig `json:"ImageConfig"` - Architectures []string `json:"Architectures"` - RevisionId string `json:"RevisionId"` + Layers *[]string `json:"Layers"` + ImageConfig *ImageConfig `json:"ImageConfig"` + Architectures []string `json:"Architectures"` + RevisionId string `json:"RevisionId"` + LoggingConfig *LoggingConfig `json:"LoggingConfig"` } func (a awsConfigIn) config() configInput { in := configInput{Runtime: a.Runtime, Handler: a.Handler, Description: a.Description, MemoryMB: a.MemorySize, - TimeoutSec: a.Timeout, Role: a.Role, Layers: a.Layers, ImageConfig: a.ImageConfig, Architectures: a.Architectures} + TimeoutSec: a.Timeout, Role: a.Role, Layers: a.Layers, ImageConfig: a.ImageConfig, Architectures: a.Architectures, Logging: a.LoggingConfig} if a.VpcConfig != nil { sn := "" if len(a.VpcConfig.SubnetIds) > 0 { diff --git a/cli/internal/svc/lambda/engine.go b/cli/internal/svc/lambda/engine.go index 00048706..012696c3 100644 --- a/cli/internal/svc/lambda/engine.go +++ b/cli/internal/svc/lambda/engine.go @@ -511,6 +511,10 @@ func (s *Service) startEnv(ctx context.Context, f Function, slot int) (*execEnv, if s.env.ContainerAPI != "" { env["AWS_ENDPOINT_URL"] = s.env.ContainerAPI } + if f.LogFormat == "JSON" { + // The AWS base images' runtimes read these to log structured JSON. + env["AWS_LAMBDA_LOG_FORMAT"], env["AWS_LAMBDA_LOG_LEVEL"] = "JSON", f.AppLogLevel + } for k, v := range f.Environment { env[k] = v } @@ -674,7 +678,7 @@ func (s *Service) post(ctx context.Context, e *execEnv, reqID, clientCtx string, func (s *Service) Run(ctx context.Context) { if s.env.Docker != nil { // Environments from a previous server run are stale. - if cs, err := s.env.Docker.ManagedContainers(); err == nil { + if cs, err := s.env.Docker.OwnContainers(); err == nil { for _, c := range cs { if c.Labels["homecloud.service"] == "lambda" { _ = s.env.Docker.Remove(c.ID) diff --git a/cli/internal/svc/lambda/lambda.go b/cli/internal/svc/lambda/lambda.go index 0e1a1fcc..22fb1ea4 100644 --- a/cli/internal/svc/lambda/lambda.go +++ b/cli/internal/svc/lambda/lambda.go @@ -14,6 +14,7 @@ package lambda import ( "archive/zip" "bytes" + "cmp" "context" "crypto/sha256" "encoding/base64" @@ -79,6 +80,58 @@ type ImageConfig struct { WorkingDirectory string `json:"WorkingDirectory,omitempty"` } +// LoggingConfig is where a function's logs go and in which format, as in +// AWS's LoggingConfig. An empty LogGroup means /aws/lambda/. +type LoggingConfig struct { + LogFormat string `json:"LogFormat,omitempty"` // Text (default) | JSON + LogGroup string `json:"LogGroup,omitempty"` + ApplicationLogLevel string `json:"ApplicationLogLevel,omitempty"` // JSON format only + SystemLogLevel string `json:"SystemLogLevel,omitempty"` // JSON format only +} + +var ( + logGroupRe = regexp.MustCompile(`^[.\-_/#A-Za-z0-9]{1,512}$`) + appLevels = map[string]bool{"TRACE": true, "DEBUG": true, "INFO": true, "WARN": true, "ERROR": true, "FATAL": true} + sysLevels = map[string]bool{"DEBUG": true, "INFO": true, "WARN": true} +) + +// applyLogging validates a logging configuration and applies it to f. +func applyLogging(f *Function, l LoggingConfig) error { + switch l.LogFormat { + case "": + l.LogFormat = "Text" + case "Text", "JSON": + default: + return core.BadRequest("LogFormat must be Text or JSON") + } + if l.LogFormat == "Text" && (l.ApplicationLogLevel != "" || l.SystemLogLevel != "") { + return core.BadRequest("ApplicationLogLevel and SystemLogLevel require LogFormat JSON") + } + if l.ApplicationLogLevel != "" && !appLevels[l.ApplicationLogLevel] { + return core.BadRequest("ApplicationLogLevel must be TRACE, DEBUG, INFO, WARN, ERROR or FATAL") + } + if l.SystemLogLevel != "" && !sysLevels[l.SystemLogLevel] { + return core.BadRequest("SystemLogLevel must be DEBUG, INFO or WARN") + } + if l.LogFormat == "JSON" { + l.ApplicationLogLevel = cmp.Or(l.ApplicationLogLevel, "INFO") + l.SystemLogLevel = cmp.Or(l.SystemLogLevel, "INFO") + } + if l.LogGroup == "" { + l.LogGroup = "/aws/lambda/" + f.Name + } else if !logGroupRe.MatchString(l.LogGroup) { + return core.BadRequest("LogGroup %q must be 1-512 letters, digits or ._-/#", l.LogGroup) + } + f.LogGroup, f.LogFormat, f.AppLogLevel, f.SysLogLevel = l.LogGroup, l.LogFormat, l.ApplicationLogLevel, l.SystemLogLevel + return nil +} + +// logging is f's LoggingConfig as AWS reports it. +func (f Function) logging() LoggingConfig { + return LoggingConfig{LogFormat: cmp.Or(f.LogFormat, "Text"), LogGroup: f.LogGroup, + ApplicationLogLevel: f.AppLogLevel, SystemLogLevel: f.SysLogLevel} +} + // Function is a function's $LATEST configuration, or (in cVersions) an // immutable published version of it. type Function struct { @@ -99,6 +152,9 @@ type Function struct { LastUpdateStatusReason string `json:"last_update_status_reason,omitempty"` URL FunctionURL `json:"function_url"` LogGroup string `json:"log_group"` + LogFormat string `json:"log_format,omitempty"` // "" = Text + AppLogLevel string `json:"application_log_level,omitempty"` + SysLogLevel string `json:"system_log_level,omitempty"` SubnetID string `json:"subnet_id,omitempty"` SecurityGroupIDs []string `json:"security_group_ids,omitempty"` Role string `json:"role,omitempty"` // execution role ARN @@ -436,6 +492,7 @@ type configInput struct { Architectures []string `json:"architectures"` ImageConfig *ImageConfig `json:"image_config"` DeadLetterTarget *string `json:"dead_letter_target"` + Logging *LoggingConfig `json:"logging_config"` } // applyConfig validates and applies the parts of a configuration change that @@ -512,6 +569,11 @@ func applyConfig(f *Function, in configInput) error { } f.ImageConfig = in.ImageConfig } + if in.Logging != nil { + if err := applyLogging(f, *in.Logging); err != nil { + return err + } + } if in.Layers != nil { if f.isImage() && len(*in.Layers) > 0 { return core.BadRequest("Layers are not supported for functions with package type Image") diff --git a/cli/internal/svc/lambda/lambda_test.go b/cli/internal/svc/lambda/lambda_test.go index 1cf679bc..e2f393aa 100644 --- a/cli/internal/svc/lambda/lambda_test.go +++ b/cli/internal/svc/lambda/lambda_test.go @@ -129,8 +129,38 @@ func TestApplyConfig(t *testing.T) { } } +// LoggingConfig (CDK's `logGroup`, Terraform's logging_config) picks the log +// group and format; dropping the group goes back to /aws/lambda/. +func TestApplyLogging(t *testing.T) { + f := Function{Name: "fn", Runtime: "nodejs22.x", Handler: "index.handler", LogGroup: "/aws/lambda/fn"} + for i, l := range []LoggingConfig{ + {LogFormat: "XML"}, + {LogFormat: "Text", ApplicationLogLevel: "INFO"}, + {LogFormat: "JSON", ApplicationLogLevel: "LOUD"}, + {LogFormat: "JSON", SystemLogLevel: "TRACE"}, + {LogGroup: "bad group name"}, + } { + g := f + if err := applyConfig(&g, configInput{Logging: &l}); err == nil { + t.Errorf("case %d: expected an error", i) + } + } + if err := applyConfig(&f, configInput{Logging: &LoggingConfig{LogGroup: "App-ConsumerLogs/x"}}); err != nil { + t.Fatal(err) + } + if got := f.logging(); got != (LoggingConfig{LogFormat: "Text", LogGroup: "App-ConsumerLogs/x"}) { + t.Errorf("custom group: %+v", got) + } + if err := applyConfig(&f, configInput{Logging: &LoggingConfig{LogFormat: "JSON", ApplicationLogLevel: "DEBUG"}}); err != nil { + t.Fatal(err) + } + if got := f.logging(); got != (LoggingConfig{LogFormat: "JSON", LogGroup: "/aws/lambda/fn", ApplicationLogLevel: "DEBUG", SystemLogLevel: "INFO"}) { + t.Errorf("json: %+v", got) + } +} + func TestRuntimes(t *testing.T) { - for _, name := range []string{"python3.9", "python3.13", "nodejs18.x", "nodejs22.x", "java17", "java21", "ruby3.3", "dotnet8", "provided.al2023", "provided.al2"} { + for _, name := range []string{"python3.9", "python3.13", "python3.14", "nodejs18.x", "nodejs22.x", "nodejs24.x", "java17", "java21", "java25", "ruby3.3", "dotnet8", "provided.al2023", "provided.al2"} { rt, ok := findRuntime(name) if !ok || !strings.HasPrefix(rt.Image, "public.ecr.aws/lambda/") { t.Errorf("runtime %s: %+v", name, rt) diff --git a/cli/internal/svc/lambda/runtimes.go b/cli/internal/svc/lambda/runtimes.go index e6050209..aaf2c4d0 100644 --- a/cli/internal/svc/lambda/runtimes.go +++ b/cli/internal/svc/lambda/runtimes.go @@ -77,9 +77,9 @@ func provided(tag, label string) Runtime { } var runtimes = []Runtime{ - py("3.13"), py("3.12"), py("3.11"), py("3.10"), py("3.9"), - node("22"), node("20"), node("18"), - java("21"), java("17"), + py("3.14"), py("3.13"), py("3.12"), py("3.11"), py("3.10"), py("3.9"), + node("24"), node("22"), node("20"), node("18"), + java("25"), java("21"), java("17"), {Name: "ruby3.3", Label: "Ruby 3.3", Image: baseImages + "ruby:3.3", DefaultHandler: "lambda_function.lambda_handler", DefaultFile: "lambda_function.rb", Template: rubyTemplate, dotted: true}, {Name: "dotnet8", Label: ".NET 8", Image: baseImages + "dotnet:8", DefaultHandler: "Function::Function.Handler::FunctionHandler"}, From de53d854de75dc117e4ac788e912162f756599f9 Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:40:38 +0530 Subject: [PATCH 2/2] Examples and docs: CDK, Pulumi and the JavaScript, Go and Java SDKs Closes #99, closes #100, closes #101. --- CHANGELOG.md | 10 ++ README.md | 8 +- docs/aws-compat.md | 83 ++++++++++++- examples/.gitignore | 16 +++ examples/cdk/basic/README.md | 115 +++++++++++++++++++ examples/cdk/basic/bin/app.ts | 9 ++ examples/cdk/basic/cdk.json | 8 ++ examples/cdk/basic/lib/stack.ts | 65 +++++++++++ examples/cdk/basic/package.json | 21 ++++ examples/cdk/basic/tsconfig.json | 14 +++ examples/pulumi/basic/Pulumi.yaml | 13 +++ examples/pulumi/basic/README.md | 103 +++++++++++++++++ examples/pulumi/basic/index.ts | 83 +++++++++++++ examples/pulumi/basic/package.json | 14 +++ examples/pulumi/basic/tsconfig.json | 17 +++ examples/sdk/README.md | 27 +++++ examples/sdk/go/go.mod | 30 +++++ examples/sdk/go/go.sum | 42 +++++++ examples/sdk/go/main.go | 94 +++++++++++++++ examples/sdk/java/pom.xml | 47 ++++++++ examples/sdk/java/src/main/java/Example.java | 70 +++++++++++ examples/sdk/javascript/index.mjs | 60 ++++++++++ examples/sdk/javascript/package.json | 15 +++ 23 files changed, 957 insertions(+), 7 deletions(-) create mode 100644 examples/.gitignore create mode 100644 examples/cdk/basic/README.md create mode 100644 examples/cdk/basic/bin/app.ts create mode 100644 examples/cdk/basic/cdk.json create mode 100644 examples/cdk/basic/lib/stack.ts create mode 100644 examples/cdk/basic/package.json create mode 100644 examples/cdk/basic/tsconfig.json create mode 100644 examples/pulumi/basic/Pulumi.yaml create mode 100644 examples/pulumi/basic/README.md create mode 100644 examples/pulumi/basic/index.ts create mode 100644 examples/pulumi/basic/package.json create mode 100644 examples/pulumi/basic/tsconfig.json create mode 100644 examples/sdk/README.md create mode 100644 examples/sdk/go/go.mod create mode 100644 examples/sdk/go/go.sum create mode 100644 examples/sdk/go/main.go create mode 100644 examples/sdk/java/pom.xml create mode 100644 examples/sdk/java/src/main/java/Example.java create mode 100644 examples/sdk/javascript/index.mjs create mode 100644 examples/sdk/javascript/package.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c754238..f753ed6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +- **CDK, Pulumi and SDK examples, verified:** [`examples/cdk/basic`](examples/cdk/basic/README.md) (`cdk bootstrap` with the default template, `cdk deploy`, `cdk diff`, `cdk destroy`; HomeCloud must be served over HTTPS because the bootstrap bucket policy denies plain HTTP), [`examples/pulumi/basic`](examples/pulumi/basic/README.md) (`pulumi up`, `preview`, `refresh`, `destroy` with a local backend) and [`examples/sdk`](examples/sdk) (JavaScript v3, Go v2, Java v2: S3, SQS, DynamoDB, STS), with configuration snippets in [docs/aws-compat.md](docs/aws-compat.md#sdks) (#99, #100, #101). +- IAM: the AWS managed policy `AWSCloudFormationReadOnlyAccess`; policy, group and instance profile names may be up to 128 characters, as in AWS (users and roles stay at 64); the `aws:ResourceAccount` condition key is set (to the server's account) for users and role sessions. +- Lambda: `LoggingConfig` (log group, `Text`/`JSON` format, application and system log levels) over the AWS API and in CloudFormation (`AWS::Lambda::Function` `LoggingConfig`, CDK's `logGroup`); it was ignored and logs always went to `/aws/lambda/`. New runtimes `nodejs24.x`, `python3.14` and `java25`. +- CloudFormation: `DeleteChangeSet` with the name of a change set that does not exist succeeds when the stack exists, as in AWS (`cdk diff` and `cdk deploy` rely on it). +- awsJson responses carry `X-Amz-Crc32`, as DynamoDB's do; the AWS SDK for Go v2 logged a checksum warning on every DynamoDB call without it. +- The `--tls-self-signed` certificate also covers `*.localhost` and `*.`, so S3 Control calls (`.localhost`, made by the Terraform and Pulumi AWS providers) verify over HTTPS. Delete `/tls/` to replace a certificate made by an earlier version. +- CloudWatch Logs, CloudWatch metrics and the Lambda environment reaper only see this installation's containers; on a Docker host shared with test runs they listed (and the reaper removed) containers of other accounts. + ## 0.5.0 (2026-10-14) Trust and operations: signed releases, safe upgrades with rollback, backups that restore everything, Prometheus metrics and a status page, and Testcontainers for Java and Node. diff --git a/README.md b/README.md index 3f65bd95..054c00aa 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,11 @@ Or use the ready-made pieces in **[docs/integrations.md](docs/integrations.md)** | **boto3** (Python SDK) | Tested: the compatibility test suite runs boto3 in CI, including Cognito SRP sign-in through pycognito | | **Terraform / OpenTofu** (`hashicorp/aws` provider) | Tested: opt-in Terraform tests in the suite (IAM, S3, Route 53, CloudFormation and more; `HC_TEST_TERRAFORM=1`), and [`examples/terraform/shop`](examples/terraform/shop/README.md) applies, re-plans clean and destroys on a fresh install. Nightly: 17 of 17 popular `terraform-aws-modules` scenarios apply, re-plan clean and destroy ([results](docs/compatibility.md)). | | **CloudFormation** (`aws cloudformation deploy`) | Tested: stacks, change sets and the boto3 waiters ([details](docs/aws-compat.md#cloudformation)) | -| **AWS CDK** | Partly: CDK-synthesized templates deploy through CloudFormation change sets; `cdk bootstrap` / `cdk deploy` end to end is not yet verified | +| **AWS CDK** | Verified by hand: `cdk bootstrap` (default template), `cdk deploy`, `cdk diff` and `cdk destroy` with HomeCloud served over HTTPS ([`examples/cdk/basic`](examples/cdk/basic/README.md)); constructs that need CloudFormation custom resources do not deploy yet | | **Testcontainers** (Go, Python, Java, Node.js) | Modules in [`integrations/`](docs/integrations.md) start HomeCloud from a test and clean up after it; the Java and Node.js modules are tested in CI with S3, SQS, DynamoDB, Lambda and STS through the AWS SDK for Java v2 and the JavaScript SDK v3 | -| **Other AWS SDKs** (Go, .NET, …) and **Pulumi** | Expected to work (same protocols and SigV4), not yet covered by tests. Reports welcome | +| **Pulumi** (`@pulumi/aws`) | Verified by hand: `pulumi up`, `preview` (no changes), `refresh` and `destroy` ([`examples/pulumi/basic`](examples/pulumi/basic/README.md)) | +| **AWS SDKs for JavaScript v3, Go v2, Java v2** | Verified by hand: S3, SQS, DynamoDB and STS ([configuration](docs/aws-compat.md#sdks), [`examples/sdk`](examples/sdk)); the Java and Node.js Testcontainers modules also run the Java and JavaScript SDKs in CI | +| **Other AWS SDKs** (.NET, Rust, …) | Expected to work (same protocols and SigV4), not yet covered by tests. Reports welcome | Run `homecloud aws-env` to get the environment variables. For Terraform, point the provider's `endpoints {}` block at HomeCloud and use `s3_use_path_style = true`; the [shop example](examples/terraform/shop/README.md) shows a complete provider block. @@ -229,7 +231,7 @@ Releases are signed (from v0.5.0): the release workflow signs `checksums.txt` an - **Shipped in [0.4.0](https://github.com/solinode/homecloud/releases/tag/v0.4.0):** VM-backed EC2 instances (QEMU, KVM when available), the container image `ghcr.io/solinode/homecloud`, a GitHub Action and testcontainers modules for Go and Python, a nightly Terraform modules compatibility suite ([17 of 17 pass](docs/compatibility.md)), and fixes from two security audits ([platform](docs/security-audit-2026-10.md), [VM instances](docs/security-audit-vm-2026-10.md)). - **Shipped in [0.5.0](https://github.com/solinode/homecloud/releases/tag/v0.5.0):** signed releases (Sigstore) with SBOMs and build provenance, verified by the installer and `homecloud upgrade`; automatic backups before upgrades and `homecloud upgrade --rollback`; restores that bring back databases, caches, load balancers and instances on a new machine; Prometheus metrics and a system status page ([docs/monitoring.md](docs/monitoring.md)); Testcontainers modules for Java and Node. -- **Next:** closing the AWS API gaps users hit most, verified CDK and Pulumi support, native VM acceleration on macOS, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms. +- **Next:** closing the AWS API gaps users hit most, native VM acceleration on macOS, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms. - **Planned:** multi-node clusters ([#55](https://github.com/solinode/homecloud/issues/55), [design](docs/design/multi-node.md)), edge compute and hardware integrations ([#56](https://github.com/solinode/homecloud/issues/56), [design](docs/design/edge.md)). See the [open issues](https://github.com/solinode/homecloud/issues) for everything planned. diff --git a/docs/aws-compat.md b/docs/aws-compat.md index f47ba982..6d3221c1 100644 --- a/docs/aws-compat.md +++ b/docs/aws-compat.md @@ -2,9 +2,10 @@ HomeCloud speaks the AWS wire protocols, so the AWS CLI, the AWS SDKs and tools built on them work against it. The test suite drives the real AWS CLI, boto3 and Terraform; CloudFormation -(including CDK-synthesized templates) is covered too. Other SDKs and Pulumi use the same -protocols but are not tested yet. Point them at your HomeCloud endpoint and use a HomeCloud -access key: +(including CDK-synthesized templates) is covered too. The JavaScript v3, Go v2 and Java v2 SDKs +([below](#sdks)), [`cdk bootstrap` and `cdk deploy`](../examples/cdk/basic/README.md) and +[Pulumi](../examples/pulumi/basic/README.md) have been run against it by hand. Point them at your +HomeCloud endpoint and use a HomeCloud access key: ```bash export AWS_ENDPOINT_URL=http://localhost:8080 # your HomeCloud API address @@ -25,6 +26,67 @@ Temporary credentials work as in AWS: create an IAM role whose trust policy allo instances with a role receive credentials and `AWS_ENDPOINT_URL` in their environment, so SDK calls from inside them reach HomeCloud. +## SDKs + +Every SDK below reads `AWS_ENDPOINT_URL`, the keys and `AWS_REGION` from the environment, so after +`eval "$(homecloud aws-env)"` the default client configuration already reaches HomeCloud. The one +setting each needs on top is **path-style S3 addressing**, because HomeCloud serves buckets by path. +The snippets set the endpoint explicitly to show where it goes. Runnable programs that create a +bucket, a queue and a table, round-trip an object, a message and an item, call +`sts get-caller-identity` and clean up are in [`examples/sdk`](../examples/sdk); each was run +against HomeCloud over HTTP and over HTTPS (`--tls-self-signed`). + +**JavaScript v3** (`@aws-sdk/client-*` 3.1151.0, Node.js 24), [`examples/sdk/javascript`](../examples/sdk/javascript/index.mjs): + +```js +import { S3Client } from "@aws-sdk/client-s3"; +import { SQSClient } from "@aws-sdk/client-sqs"; + +const config = { + endpoint: process.env.AWS_ENDPOINT_URL, // http://localhost:8080 + region: "us-east-1", + credentials: { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY }, +}; +const s3 = new S3Client({ ...config, forcePathStyle: true }); +const sqs = new SQSClient(config); // DynamoDB, STS, ... the same way +``` + +With HTTPS and a self-signed certificate, Node.js does not read `AWS_CA_BUNDLE`; export +`NODE_EXTRA_CA_CERTS="$AWS_CA_BUNDLE"`. + +**Go v2** (`github.com/aws/aws-sdk-go-v2` 1.47.3, `service/s3` 1.114.3), [`examples/sdk/go`](../examples/sdk/go/main.go): + +```go +cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-east-1")) +cfg.BaseEndpoint = aws.String(os.Getenv("AWS_ENDPOINT_URL")) // optional: AWS_ENDPOINT_URL is read anyway + +s3c := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }) +q := sqs.NewFromConfig(cfg) // DynamoDB, STS, ... the same way +``` + +`AWS_CA_BUNDLE` is honored for HTTPS. + +**Java v2** (`software.amazon.awssdk` 2.55.16, Java 17), [`examples/sdk/java`](../examples/sdk/java/src/main/java/Example.java): + +```java +URI endpoint = URI.create(System.getenv("AWS_ENDPOINT_URL")); +var creds = StaticCredentialsProvider.create(AwsBasicCredentials.create( + System.getenv("AWS_ACCESS_KEY_ID"), System.getenv("AWS_SECRET_ACCESS_KEY"))); + +S3Client s3 = S3Client.builder().endpointOverride(endpoint).region(Region.US_EAST_1) + .credentialsProvider(creds).forcePathStyle(true).build(); +SqsClient sqs = SqsClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1) + .credentialsProvider(creds).build(); // DynamoDB, STS, ... the same way +``` + +For HTTPS with a self-signed certificate, import it into a trust store +(`keytool -importcert -file "$AWS_CA_BUNDLE" -keystore homecloud.p12 -storepass changeit -noprompt`) and +run with `-Djavax.net.ssl.trustStore=homecloud.p12 -Djavax.net.ssl.trustStorePassword=changeit`. + +Nothing failed in these runs once HomeCloud sent the `X-Amz-Crc32` header on JSON responses: the Go SDK +checks DynamoDB response bodies against it and logged `response did not match expected checksum` on +every DynamoDB call without it. + ## Policies and authorization Every request is decided the way AWS does within one account: an explicit `Deny` in the @@ -84,7 +146,7 @@ that is not itself a trusted proxy is the source address, and `X-Forwarded-Proto | STS | sts | awsQuery | GetCallerIdentity, AssumeRole, GetSessionToken | | IAM | iam | awsQuery | Users, groups, roles, managed policies (with versions, `arn:aws:iam::aws:policy/...`), inline policies, access keys, instance profiles, permissions boundaries, conditions, simulation | | S3 | s3 | restXml | Objects, multipart, presigned URLs (SigV4 and SigV2), chunked/trailer uploads, versioning, lifecycle, bucket policies with condition keys, tagging, website; path and virtual-host addressing | -| Lambda | lambda | restJson1 | Functions (zip, S3, container image), invoke (sync/async/tail), versions, aliases, layers, permissions, concurrency, destinations, event source mappings (SQS, DynamoDB streams), function URLs | +| Lambda | lambda | restJson1 | Functions (zip, S3, container image), invoke (sync/async/tail), versions, aliases, layers, permissions, concurrency, destinations, event source mappings (SQS, DynamoDB streams), function URLs, `LoggingConfig` (custom log group; `JSON` format makes the runtime log application lines as JSON; the START, END and REPORT lines stay text) | | DynamoDB | dynamodb | awsJson 1.0 | Tables, GSIs/LSIs, all item ops with expressions, batch, transactions, PartiQL, TTL, streams | | SQS | sqs | awsJson 1.0 + awsQuery | Standard and FIFO queues, DLQs and redrive, batch ops, long polling, queue policies | | SNS | sns | awsQuery | Topics (incl. FIFO), subscriptions (sqs, lambda, http/s, email), filter policies, topic policies, signed messages | @@ -237,6 +299,19 @@ clean against a fresh install; the README shows how to check each piece. [Terraform compatibility](compatibility.md) runs popular community modules (`terraform-aws-modules`) the same way, nightly, and publishes which ones pass. +## CDK and Pulumi + +[`examples/cdk/basic`](../examples/cdk/basic/README.md) runs the stock CDK CLI end to end: `cdk bootstrap` +(the default modern bootstrap template, unchanged), `cdk deploy`, `cdk diff` and `cdk destroy` of a stack +with an SQS queue feeding a Lambda function that writes to a DynamoDB table. HomeCloud must be served over +HTTPS (`homecloud serve --tls-self-signed` is enough) because the bootstrap staging bucket's policy denies +requests without TLS; constructs that need CloudFormation custom resources (`autoDeleteObjects`, +`BucketDeployment`, `logRetention`) do not deploy yet. + +[`examples/pulumi/basic`](../examples/pulumi/basic/README.md) deploys S3, SQS, DynamoDB, IAM and a +queue-triggered Lambda function with `@pulumi/aws` and a local state backend; `pulumi up`, `preview`, +`refresh` and `destroy` run clean over HTTP and HTTPS. + ## Adding operations (for contributors) The `internal/awsapi` package verifies signatures, resolves the service from the credential diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 00000000..b1225040 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,16 @@ +# Node.js (CDK, Pulumi, SDK examples) +node_modules/ +package-lock.json +*.js +*.d.ts +# CDK +cdk.out/ +cdk.context.json +outputs.json +# Pulumi stack settings (written by `pulumi stack init` / `pulumi config set`) +Pulumi.*.yaml +# Terraform +.terraform/ +*.tfstate* +# Java +target/ diff --git a/examples/cdk/basic/README.md b/examples/cdk/basic/README.md new file mode 100644 index 00000000..3dbcd695 --- /dev/null +++ b/examples/cdk/basic/README.md @@ -0,0 +1,115 @@ +# AWS CDK on HomeCloud + +A small CDK app (TypeScript) deployed with the stock CDK CLI: `cdk bootstrap`, `cdk deploy`, `cdk diff` +and `cdk destroy` all run against HomeCloud. Nothing in the app is HomeCloud specific; the same code +deploys to AWS. + +What it builds (`lib/stack.ts`): + +| Resource | Notes | +| --- | --- | +| S3 bucket | versioned, `RemovalPolicy.DESTROY` | +| DynamoDB table | on-demand, partition key `id` | +| SQS queue | 60 s visibility timeout | +| Lambda function | Node.js 22, inline code (`lambda.Code.fromInline`), subscribed to the queue, writes each message to the table | +| CloudWatch log group | the function's `logGroup`, so `cdk destroy` removes it | +| IAM | the function's role and the policies CDK generates for the queue and the table | + +Verified with CDK CLI 2.1145.0, `aws-cdk-lib` 2.273.0, Node.js 24 and HomeCloud 0.5.x (the bootstrap +stack needs the fixes listed at the end, so use a build from `main` until the next release). + +## Prerequisites + +* A running HomeCloud **served over HTTPS** (see [why](#why-https)). The built-in self-signed certificate + is enough: + + ```bash + homecloud serve --tls-self-signed + ``` + + It covers `localhost`, `*.localhost`, `127.0.0.1` and `--public-host`. +* Node.js 18 or newer and the AWS CLI (only to check the result). + +## Run it + +```bash +eval "$(homecloud aws-env)" # AWS_ENDPOINT_URL=https://localhost:8080, keys, region, AWS_CA_BUNDLE +export NODE_EXTRA_CA_CERTS="$AWS_CA_BUNDLE" # the CDK CLI is Node.js: trust HomeCloud's certificate + +cd examples/cdk/basic +npm install +npx cdk bootstrap # creates the CDKToolkit stack (staging bucket, ECR repository, roles, SSM parameter) +npx cdk deploy --require-approval never --outputs-file outputs.json +``` + +The CDK CLI finds the account with `sts get-caller-identity`, as on AWS, and uses `us-east-1` from +`AWS_REGION`. No `--endpoint` flags, custom bootstrap template or legacy synthesizer is needed: the +default synthesizer assumes the bootstrap roles (`cdk-hnb659fds-deploy-role-...`, +`...-file-publishing-role-...`), uploads the template to the staging bucket and deploys it through a +CloudFormation change set. + +## Check that it works + +```bash +QUEUE=$(jq -r .HomeCloudBasic.QueueUrl outputs.json) +TABLE=$(jq -r .HomeCloudBasic.TableName outputs.json) +LOGS=$(jq -r .HomeCloudBasic.LogGroupName outputs.json) + +aws sqs send-message --queue-url "$QUEUE" --message-body '{"order":1}' +aws logs tail "$LOGS" --since 5m # ... INFO order received: {"order":1} +aws dynamodb scan --table-name "$TABLE" --query 'Items[].body.S' # ["{\"order\":1}"] +``` + +`npx cdk diff` then reports no differences, and a second `npx cdk deploy` reports `(no changes)`. + +## Tear down + +The bucket is versioned, and CloudFormation (on AWS as on HomeCloud) does not delete a bucket that still +holds objects. Empty it first if you put anything in it, then destroy: + +```bash +BUCKET=$(jq -r .HomeCloudBasic.BucketName outputs.json) +aws s3api delete-objects --bucket "$BUCKET" --delete "$(aws s3api list-object-versions --bucket "$BUCKET" \ + --query '{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" +npx cdk destroy --force +``` + +`cdk destroy` removes the stack, the function's containers and everything above. The `CDKToolkit` +stack stays, as on AWS. + +## Why HTTPS + +The bootstrap stack's staging bucket has CDK's standard bucket policy, which denies every request made +without TLS (`aws:SecureTransport` is `false`). HomeCloud enforces bucket policies, so over plain HTTP the +asset upload in `cdk deploy` fails with + +``` +Bucket named 'cdk-hnb659fds-assets--us-east-1' exists, but we dont have access to it. +``` + +Serve HomeCloud with `--tls-self-signed` (or `--tls-cert`/`--tls-key`), as above. Buckets of your own +stacks with `enforceSSL: true` behave the same way. + +## Notes + +* Custom resources are not supported by HomeCloud's CloudFormation yet, so constructs that rely on one + fail to deploy: `autoDeleteObjects: true` on a bucket, `BucketDeployment`, `AwsCustomResource`, log + retention on a function's implicit log group (`logRetention`; use `logGroup` as here). Nested stacks + and `AWS::Serverless` transforms are not supported either. +* The function's code is inline, so the only file asset is the template itself. Zip assets + (`Code.fromAsset`) and container image assets (pushed to the bootstrap ECR repository) go through the + same publishing roles but are not covered by this example yet. + +## Fixed in HomeCloud while verifying this example + +* The AWS managed policy `AWSCloudFormationReadOnlyAccess`, which the bootstrap stack attaches to its + deployment action role, existed only as a missing-policy error. +* IAM rejected policy names longer than 64 characters; AWS allows 128 for policies (and groups and + instance profiles). The bootstrap stack names an inline policy with 72. +* The `aws:ResourceAccount` condition key, which scopes the publishing roles' policies, was never set, + so the roles were denied access to their own staging bucket. +* `AWS::Lambda::Function` `LoggingConfig` (CDK's `logGroup`) was ignored and logs went to + `/aws/lambda/`; Lambda now supports `LoggingConfig` (log group, format and levels) over the AWS + API and in templates. +* `DeleteChangeSet` on a change set name that does not exist now succeeds when the stack exists, as in + AWS; `cdk diff` relies on it and fell back to a template-only diff before. diff --git a/examples/cdk/basic/bin/app.ts b/examples/cdk/basic/bin/app.ts new file mode 100644 index 00000000..430c2793 --- /dev/null +++ b/examples/cdk/basic/bin/app.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import * as cdk from 'aws-cdk-lib'; +import { BasicStack } from '../lib/stack'; + +const app = new cdk.App(); +new BasicStack(app, 'HomeCloudBasic', { + // The account and region come from the credentials (`aws sts get-caller-identity`), as on AWS. + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, +}); diff --git a/examples/cdk/basic/cdk.json b/examples/cdk/basic/cdk.json new file mode 100644 index 00000000..e1a2912e --- /dev/null +++ b/examples/cdk/basic/cdk.json @@ -0,0 +1,8 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/app.ts", + "watch": { + "include": ["**"], + "exclude": ["README.md", "cdk*.json", "**/*.d.ts", "**/*.js", "tsconfig.json", "package*.json", "node_modules"] + }, + "context": {} +} diff --git a/examples/cdk/basic/lib/stack.ts b/examples/cdk/basic/lib/stack.ts new file mode 100644 index 00000000..b1934970 --- /dev/null +++ b/examples/cdk/basic/lib/stack.ts @@ -0,0 +1,65 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; +import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; + +// An SQS queue feeding a Lambda function that writes each message to a DynamoDB table, +// plus an S3 bucket. Nothing here is HomeCloud specific: the same app deploys to AWS. +export class BasicStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + const bucket = new s3.Bucket(this, 'Uploads', { + versioned: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + const table = new dynamodb.Table(this, 'Orders', { + partitionKey: { name: 'id', type: dynamodb.AttributeType.STRING }, + billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + const queue = new sqs.Queue(this, 'Jobs', { + visibilityTimeout: cdk.Duration.seconds(60), + }); + + const logGroup = new logs.LogGroup(this, 'ConsumerLogs', { + retention: logs.RetentionDays.ONE_WEEK, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + const consumer = new lambda.Function(this, 'Consumer', { + runtime: lambda.Runtime.NODEJS_22_X, + handler: 'index.handler', + timeout: cdk.Duration.seconds(30), + logGroup, + environment: { TABLE_NAME: table.tableName }, + code: lambda.Code.fromInline(` +const { DynamoDBClient, PutItemCommand } = require('@aws-sdk/client-dynamodb'); +const db = new DynamoDBClient({}); +exports.handler = async (event) => { + for (const r of event.Records) { + console.log('order received:', r.body); + await db.send(new PutItemCommand({ + TableName: process.env.TABLE_NAME, + Item: { id: { S: r.messageId }, body: { S: r.body } }, + })); + } +}; +`), + }); + table.grantWriteData(consumer); + consumer.addEventSource(new SqsEventSource(queue, { batchSize: 5 })); + + new cdk.CfnOutput(this, 'BucketName', { value: bucket.bucketName }); + new cdk.CfnOutput(this, 'TableName', { value: table.tableName }); + new cdk.CfnOutput(this, 'QueueUrl', { value: queue.queueUrl }); + new cdk.CfnOutput(this, 'FunctionName', { value: consumer.functionName }); + new cdk.CfnOutput(this, 'LogGroupName', { value: logGroup.logGroupName }); + } +} diff --git a/examples/cdk/basic/package.json b/examples/cdk/basic/package.json new file mode 100644 index 00000000..e2d2aa2a --- /dev/null +++ b/examples/cdk/basic/package.json @@ -0,0 +1,21 @@ +{ + "name": "homecloud-cdk-basic", + "version": "0.1.0", + "private": true, + "bin": { + "app": "bin/app.ts" + }, + "scripts": { + "cdk": "cdk" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "aws-cdk": "2.1145.0", + "ts-node": "^10.9.2", + "typescript": "~5.6.0" + }, + "dependencies": { + "aws-cdk-lib": "2.273.0", + "constructs": "^10.4.0" + } +} diff --git a/examples/cdk/basic/tsconfig.json b/examples/cdk/basic/tsconfig.json new file mode 100644 index 00000000..232160b2 --- /dev/null +++ b/examples/cdk/basic/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "lib": ["es2022"], + "declaration": false, + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "exclude": ["node_modules", "cdk.out"] +} diff --git a/examples/pulumi/basic/Pulumi.yaml b/examples/pulumi/basic/Pulumi.yaml new file mode 100644 index 00000000..df7e572b --- /dev/null +++ b/examples/pulumi/basic/Pulumi.yaml @@ -0,0 +1,13 @@ +name: homecloud-basic +description: S3, SQS, DynamoDB, IAM and Lambda on HomeCloud with the stock AWS provider +runtime: + name: nodejs + options: + typescript: true +config: + # HomeCloud's region. The endpoint comes from AWS_ENDPOINT_URL (`homecloud aws-env`). + aws:region: + value: us-east-1 + # HomeCloud serves buckets by path, so bucket names need not resolve as host names. + aws:s3UsePathStyle: + value: true diff --git a/examples/pulumi/basic/README.md b/examples/pulumi/basic/README.md new file mode 100644 index 00000000..15e06180 --- /dev/null +++ b/examples/pulumi/basic/README.md @@ -0,0 +1,103 @@ +# Pulumi on HomeCloud + +A small Pulumi program (TypeScript) written with the stock `@pulumi/aws` provider. Nothing in `index.ts` +is HomeCloud specific: the provider is pointed at HomeCloud by the environment and two settings in +`Pulumi.yaml`, so the same program deploys to AWS. + +What it builds: + +| Resource | Notes | +| --- | --- | +| S3 bucket and object | `forceDestroy`, so `pulumi destroy` empties it | +| DynamoDB table | on-demand, hash key `id` | +| SQS queue | 60 s visibility timeout | +| IAM role | trusted by `lambda.amazonaws.com`, with `AWSLambdaSQSQueueExecutionRole` and an inline `dynamodb:PutItem` policy | +| Lambda function | Node.js 24, inline code, `loggingConfig` pointing at the log group below | +| Event source mapping | the queue triggers the function, which writes each message to the table | +| CloudWatch log group | declared so `pulumi destroy` removes it | + +Verified with Pulumi CLI 3.268.0, `@pulumi/aws` 7.50.0 (provider plugin 7.50.0), Node.js 24 and HomeCloud +0.5.x built from `main` (Node.js 24 functions and `loggingConfig` need it). `pulumi up`, `pulumi preview` +(no changes), `pulumi refresh` (no changes) and `pulumi destroy` all run clean on a fresh install, over +HTTP and over HTTPS (`--tls-self-signed`). + +## Prerequisites + +* A running HomeCloud (`homecloud serve`). +* The Pulumi CLI and Node.js 18 or newer. No Pulumi account is needed: state is kept in a local + directory. + +## Run it + +```bash +eval "$(homecloud aws-env)" # AWS_ENDPOINT_URL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION +export PULUMI_BACKEND_URL=file://$HOME/.pulumi-local # local state; or `pulumi login --local` +export PULUMI_CONFIG_PASSPHRASE= # no secrets in this program + +cd examples/pulumi/basic +npm install +pulumi stack init dev +pulumi up --yes # about half a minute +``` + +### Provider settings, and why + +* `AWS_ENDPOINT_URL` (from `homecloud aws-env`) sends every AWS call to HomeCloud. The provider reads + it like the AWS CLI does, so no per-service endpoint list is needed. +* `aws:s3UsePathStyle: true` (in `Pulumi.yaml`): HomeCloud serves buckets by path, so bucket names need + not resolve as host names. +* `aws:region: us-east-1` (in `Pulumi.yaml`): HomeCloud's region. +* No `skipCredentialsValidation`, `skipRequestingAccountId` or `skipMetadataApiCheck` is needed: + HomeCloud implements STS, so the provider's startup checks pass unchanged. +* With HTTPS, `homecloud aws-env` also exports `AWS_CA_BUNDLE`, which the provider honors. + +If you prefer an explicit `aws:endpoints` block in the stack configuration instead of the environment +variable, list every service the program uses **plus `sts` and `s3control`** (the provider reads bucket +tags through S3 Control), and set `aws:skipCredentialsValidation`, because Pulumi's own credential check +ignores `aws:endpoints` and asks the real AWS STS: + +```bash +for s in cloudwatchlogs dynamodb iam lambda s3 s3control sqs sts; do + pulumi config set --path "aws:endpoints[0].$s" "$AWS_ENDPOINT_URL" +done +pulumi config set aws:skipCredentialsValidation true +``` + +Use a host name for the endpoint, not an IP address: S3 Control puts the account ID in front of the host +(`123456789012.localhost` resolves, `123456789012.127.0.0.1` does not). `homecloud aws-env` already prints +`localhost` for a local server. + +## Check that it works + +```bash +aws s3 cp "s3://$(pulumi stack output bucketName)/README.txt" - # uploaded by Pulumi +aws sqs send-message --queue-url "$(pulumi stack output queueUrl)" --message-body '{"order":1}' +aws logs tail "$(pulumi stack output logGroupName)" --since 5m # ... INFO order received: {"order":1} +aws dynamodb scan --table-name "$(pulumi stack output tableName)" --query 'Items[].body.S' +``` + +A second `pulumi preview --expect-no-changes` and a `pulumi refresh` report no changes. + +## Tear down + +```bash +pulumi destroy --yes +pulumi stack rm dev --yes +``` + +## Notes + +* The provider waits for SQS queue attributes to settle after creating and deleting a queue, so those + two steps take about 25 seconds each, as they do on AWS. +* The log group is declared in the program. Lambda creates its group on first invoke and, on AWS as on + HomeCloud, leaves it behind when the function is deleted, so `destroy` would otherwise not leave the + account empty. + +## Fixed in HomeCloud while verifying this example + +* Lambda accepts the `nodejs24.x`, `python3.14` and `java25` runtimes. +* Lambda supports `LoggingConfig` (custom log group, `Text`/`JSON` format and log levels); it was + ignored before. +* The self-signed certificate of `--tls-self-signed` covers `*.localhost` and `*.`, so S3 + Control calls (`.localhost`) verify over HTTPS. A certificate generated by an earlier version + is kept: delete `/tls/` to get a new one. diff --git a/examples/pulumi/basic/index.ts b/examples/pulumi/basic/index.ts new file mode 100644 index 00000000..447d2474 --- /dev/null +++ b/examples/pulumi/basic/index.ts @@ -0,0 +1,83 @@ +import * as pulumi from "@pulumi/pulumi"; +import * as aws from "@pulumi/aws"; + +// An S3 bucket with an object, an SQS queue that triggers a Lambda function, and a +// DynamoDB table the function writes each message to. Nothing here is HomeCloud +// specific: the provider is pointed at HomeCloud by AWS_ENDPOINT_URL and Pulumi.yaml. + +const bucket = new aws.s3.Bucket("uploads", { forceDestroy: true }); + +new aws.s3.BucketObject("readme", { + bucket: bucket.id, + key: "README.txt", + content: "uploaded by Pulumi\n", + contentType: "text/plain", +}); + +const table = new aws.dynamodb.Table("orders", { + billingMode: "PAY_PER_REQUEST", + hashKey: "id", + attributes: [{ name: "id", type: "S" }], +}); + +const queue = new aws.sqs.Queue("jobs", { visibilityTimeoutSeconds: 60 }); + +const role = new aws.iam.Role("consumer-role", { + assumeRolePolicy: JSON.stringify({ + Version: "2012-10-17", + Statement: [{ Effect: "Allow", Principal: { Service: "lambda.amazonaws.com" }, Action: "sts:AssumeRole" }], + }), +}); + +new aws.iam.RolePolicyAttachment("consumer-sqs", { + role: role.name, + policyArn: aws.iam.ManagedPolicy.AWSLambdaSQSQueueExecutionRole, +}); + +new aws.iam.RolePolicy("consumer-table", { + role: role.id, + policy: table.arn.apply(arn => JSON.stringify({ + Version: "2012-10-17", + Statement: [{ Effect: "Allow", Action: ["dynamodb:PutItem"], Resource: arn }], + })), +}); + +// Declared so `pulumi destroy` removes it: Lambda creates the group on first invoke and, +// on AWS as on HomeCloud, leaves it behind when the function is deleted. +const logGroup = new aws.cloudwatch.LogGroup("consumer-logs", { retentionInDays: 7 }); + +const consumer = new aws.lambda.Function("consumer", { + runtime: aws.lambda.Runtime.NodeJS24dX, + handler: "index.handler", + role: role.arn, + timeout: 30, + loggingConfig: { logFormat: "Text", logGroup: logGroup.name }, + environment: { variables: { TABLE_NAME: table.name } }, + code: new pulumi.asset.AssetArchive({ + "index.js": new pulumi.asset.StringAsset(` +const { DynamoDBClient, PutItemCommand } = require("@aws-sdk/client-dynamodb"); +const db = new DynamoDBClient({}); +exports.handler = async (event) => { + for (const r of event.Records) { + console.log("order received:", r.body); + await db.send(new PutItemCommand({ + TableName: process.env.TABLE_NAME, + Item: { id: { S: r.messageId }, body: { S: r.body } }, + })); + } +}; +`), + }), +}); + +new aws.lambda.EventSourceMapping("consumer-trigger", { + eventSourceArn: queue.arn, + functionName: consumer.arn, + batchSize: 5, +}); + +export const bucketName = bucket.id; +export const queueUrl = queue.url; +export const tableName = table.name; +export const functionName = consumer.name; +export const logGroupName = logGroup.name; diff --git a/examples/pulumi/basic/package.json b/examples/pulumi/basic/package.json new file mode 100644 index 00000000..0535e156 --- /dev/null +++ b/examples/pulumi/basic/package.json @@ -0,0 +1,14 @@ +{ + "name": "homecloud-pulumi-basic", + "version": "0.1.0", + "private": true, + "main": "index.ts", + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "~5.6.0" + }, + "dependencies": { + "@pulumi/aws": "7.50.0", + "@pulumi/pulumi": "3.268.0" + } +} diff --git a/examples/pulumi/basic/tsconfig.json b/examples/pulumi/basic/tsconfig.json new file mode 100644 index 00000000..36934b8a --- /dev/null +++ b/examples/pulumi/basic/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "strict": true, + "outDir": "bin", + "target": "es2022", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "files": ["index.ts"] +} diff --git a/examples/sdk/README.md b/examples/sdk/README.md new file mode 100644 index 00000000..709c327b --- /dev/null +++ b/examples/sdk/README.md @@ -0,0 +1,27 @@ +# AWS SDKs against HomeCloud + +Each program calls `sts get-caller-identity`, then creates an S3 bucket, an SQS queue and a DynamoDB +table, puts and gets an object, a message and an item, and deletes what it made. The configuration each +SDK needs is explained in [docs/aws-compat.md](../../docs/aws-compat.md#sdks). + +```bash +eval "$(homecloud aws-env)" + +cd examples/sdk +(cd javascript && npm install && node index.mjs) # AWS SDK for JavaScript v3 +(cd go && go run .) # AWS SDK for Go v2 +(cd java && mvn -q compile exec:java) # AWS SDK for Java v2 (Java 17+, Maven) +``` + +Expected output (the account differs): + +``` +caller: arn:aws:iam::123456789012:user/root +s3: hello from Go +sqs: hello from Go +dynamodb: hello from Go +``` + +For a server started with `--tls-self-signed`, `homecloud aws-env` also exports `AWS_CA_BUNDLE`; the Go +SDK reads it, Node.js needs `NODE_EXTRA_CA_CERTS="$AWS_CA_BUNDLE"`, and Java needs a trust store (see +the docs). diff --git a/examples/sdk/go/go.mod b/examples/sdk/go/go.mod new file mode 100644 index 00000000..7ca5832e --- /dev/null +++ b/examples/sdk/go/go.mod @@ -0,0 +1,30 @@ +module github.com/homecloudhq/homecloud/examples/sdk/go + +go 1.27.1 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/aws/aws-sdk-go-v2/config v1.33.9 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 + github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/smithy-go v1.28.5 // indirect +) diff --git a/examples/sdk/go/go.sum b/examples/sdk/go/go.sum new file mode 100644 index 00000000..1bb59597 --- /dev/null +++ b/examples/sdk/go/go.sum @@ -0,0 +1,42 @@ +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= +github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= +github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.3 h1:840B8kzqfJDUHUCvSUhoc8MmnOtmGIwEg4WIFiJSy68= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.70.3/go.mod h1:OYCu7bfkQi8JHCVxkQr0n6e/eL/iiJG0YXFN5fOP3v0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.6 h1:GxZx+rmWZ1/20VC0K6PJE6KFgIlwovwpJeR4pNKRlLY= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.6/go.mod h1:AyLP2PNMRAVl7yq1/qoryCn28uRxMEDN4Cb3hTBUkHI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 h1:D+Zmdt3T3P770MrqTmZvalbj6ADO7xA7n+qnNakGvrI= +github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4/go.mod h1:dUYH3ia3qEO5ogaWDLp2tjtb4u4Y9xRbae7tOr1ClIA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= diff --git a/examples/sdk/go/main.go b/examples/sdk/go/main.go new file mode 100644 index 00000000..dac3a288 --- /dev/null +++ b/examples/sdk/go/main.go @@ -0,0 +1,94 @@ +// AWS SDK for Go v2 against HomeCloud: STS, S3, SQS and DynamoDB. +// +// eval "$(homecloud aws-env)" +// go run . +package main + +import ( + "context" + "fmt" + "io" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +func main() { + ctx := context.Background() + // LoadDefaultConfig reads AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION and + // AWS_ENDPOINT_URL from the environment. BaseEndpoint is set explicitly to show it. + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-east-1")) + must(err) + cfg.BaseEndpoint = aws.String(os.Getenv("AWS_ENDPOINT_URL")) // e.g. http://localhost:8080 + + id, err := sts.NewFromConfig(cfg).GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + must(err) + fmt.Println("caller:", *id.Arn) + + suffix := strconv.FormatInt(time.Now().UnixNano(), 36) + + // S3: buckets by path, not by host name. + s3c := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }) + bucket := aws.String("sdk-go-" + suffix) + _, err = s3c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: bucket}) + must(err) + _, err = s3c.PutObject(ctx, &s3.PutObjectInput{Bucket: bucket, Key: aws.String("hello.txt"), Body: strings.NewReader("hello from Go")}) + must(err) + obj, err := s3c.GetObject(ctx, &s3.GetObjectInput{Bucket: bucket, Key: aws.String("hello.txt")}) + must(err) + body, _ := io.ReadAll(obj.Body) + fmt.Println("s3:", string(body)) + _, err = s3c.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: bucket, Key: aws.String("hello.txt")}) + must(err) + _, err = s3c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: bucket}) + must(err) + + q := sqs.NewFromConfig(cfg) + qu, err := q.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("sdk-go-" + suffix)}) + must(err) + _, err = q.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: qu.QueueUrl, MessageBody: aws.String("hello from Go")}) + must(err) + rm, err := q.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: qu.QueueUrl, WaitTimeSeconds: 5}) + must(err) + if len(rm.Messages) == 0 { + log.Fatal("sqs: no message") + } + fmt.Println("sqs:", *rm.Messages[0].Body) + _, err = q.DeleteQueue(ctx, &sqs.DeleteQueueInput{QueueUrl: qu.QueueUrl}) + must(err) + + db := dynamodb.NewFromConfig(cfg) + table := aws.String("sdk-go-" + suffix) + _, err = db.CreateTable(ctx, &dynamodb.CreateTableInput{ + TableName: table, BillingMode: ddbtypes.BillingModePayPerRequest, + AttributeDefinitions: []ddbtypes.AttributeDefinition{{AttributeName: aws.String("id"), AttributeType: ddbtypes.ScalarAttributeTypeS}}, + KeySchema: []ddbtypes.KeySchemaElement{{AttributeName: aws.String("id"), KeyType: ddbtypes.KeyTypeHash}}, + }) + must(err) + must(dynamodb.NewTableExistsWaiter(db).Wait(ctx, &dynamodb.DescribeTableInput{TableName: table}, 2*time.Minute)) + _, err = db.PutItem(ctx, &dynamodb.PutItemInput{TableName: table, Item: map[string]ddbtypes.AttributeValue{ + "id": &ddbtypes.AttributeValueMemberS{Value: "1"}, "msg": &ddbtypes.AttributeValueMemberS{Value: "hello from Go"}}}) + must(err) + item, err := db.GetItem(ctx, &dynamodb.GetItemInput{TableName: table, Key: map[string]ddbtypes.AttributeValue{ + "id": &ddbtypes.AttributeValueMemberS{Value: "1"}}}) + must(err) + fmt.Println("dynamodb:", item.Item["msg"].(*ddbtypes.AttributeValueMemberS).Value) + _, err = db.DeleteTable(ctx, &dynamodb.DeleteTableInput{TableName: table}) + must(err) +} + +func must(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/examples/sdk/java/pom.xml b/examples/sdk/java/pom.xml new file mode 100644 index 00000000..61822ca5 --- /dev/null +++ b/examples/sdk/java/pom.xml @@ -0,0 +1,47 @@ + + + 4.0.0 + dev.homecloud.examples + sdk-java + 0.1.0 + + + 17 + UTF-8 + 2.55.16 + + + + + + software.amazon.awssdk + bom + ${aws.sdk.version} + pom + import + + + + + + software.amazon.awssdks3 + software.amazon.awssdksqs + software.amazon.awssdkdynamodb + software.amazon.awssdksts + org.slf4jslf4j-nop2.0.16 + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + Example + + + + + diff --git a/examples/sdk/java/src/main/java/Example.java b/examples/sdk/java/src/main/java/Example.java new file mode 100644 index 00000000..07301840 --- /dev/null +++ b/examples/sdk/java/src/main/java/Example.java @@ -0,0 +1,70 @@ +// AWS SDK for Java v2 against HomeCloud: STS, S3, SQS and DynamoDB. +// +// eval "$(homecloud aws-env)" +// mvn -q compile exec:java +import java.net.URI; +import java.util.Map; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.BillingMode; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sts.StsClient; + +public class Example { + public static void main(String[] args) { + // The SDK also reads these from the environment (AWS_ENDPOINT_URL since 2.21); + // they are spelled out here to show what each one is. + URI endpoint = URI.create(System.getenv("AWS_ENDPOINT_URL")); // e.g. http://localhost:8080 + Region region = Region.US_EAST_1; + var creds = StaticCredentialsProvider.create(AwsBasicCredentials.create( + System.getenv("AWS_ACCESS_KEY_ID"), System.getenv("AWS_SECRET_ACCESS_KEY"))); + + try (var sts = StsClient.builder().endpointOverride(endpoint).region(region).credentialsProvider(creds).build()) { + System.out.println("caller: " + sts.getCallerIdentity().arn()); + } + + String suffix = Long.toString(System.currentTimeMillis(), 36); + + // S3: buckets by path, not by host name. + try (var s3 = S3Client.builder().endpointOverride(endpoint).region(region).credentialsProvider(creds) + .forcePathStyle(true).build()) { + String bucket = "sdk-java-" + suffix; + s3.createBucket(b -> b.bucket(bucket)); + s3.putObject(b -> b.bucket(bucket).key("hello.txt"), RequestBody.fromString("hello from Java")); + System.out.println("s3: " + s3.getObjectAsBytes(b -> b.bucket(bucket).key("hello.txt")).asUtf8String()); + s3.deleteObject(b -> b.bucket(bucket).key("hello.txt")); + s3.deleteBucket(b -> b.bucket(bucket)); + } + + try (var sqs = SqsClient.builder().endpointOverride(endpoint).region(region).credentialsProvider(creds).build()) { + String url = sqs.createQueue(b -> b.queueName("sdk-java-" + suffix)).queueUrl(); + sqs.sendMessage(b -> b.queueUrl(url).messageBody("hello from Java")); + var msgs = sqs.receiveMessage(b -> b.queueUrl(url).waitTimeSeconds(5)).messages(); + System.out.println("sqs: " + msgs.get(0).body()); + sqs.deleteQueue(b -> b.queueUrl(url)); + } + + try (var ddb = DynamoDbClient.builder().endpointOverride(endpoint).region(region).credentialsProvider(creds).build()) { + String table = "sdk-java-" + suffix; + ddb.createTable(b -> b.tableName(table).billingMode(BillingMode.PAY_PER_REQUEST) + .attributeDefinitions(AttributeDefinition.builder().attributeName("id").attributeType(ScalarAttributeType.S).build()) + .keySchema(KeySchemaElement.builder().attributeName("id").keyType(KeyType.HASH).build())); + ddb.waiter().waitUntilTableExists(b -> b.tableName(table)); + ddb.putItem(b -> b.tableName(table).item(Map.of( + "id", AttributeValue.fromS("1"), "msg", AttributeValue.fromS("hello from Java")))); + var item = ddb.getItem(b -> b.tableName(table).key(Map.of("id", AttributeValue.fromS("1")))).item(); + System.out.println("dynamodb: " + item.get("msg").s()); + ddb.deleteTable(b -> b.tableName(table)); + } + } +} diff --git a/examples/sdk/javascript/index.mjs b/examples/sdk/javascript/index.mjs new file mode 100644 index 00000000..aec95a18 --- /dev/null +++ b/examples/sdk/javascript/index.mjs @@ -0,0 +1,60 @@ +// AWS SDK for JavaScript v3 against HomeCloud: STS, S3, SQS and DynamoDB. +// +// eval "$(homecloud aws-env)" +// npm install && node index.mjs +import { STSClient, GetCallerIdentityCommand } from "@aws-sdk/client-sts"; +import { + S3Client, CreateBucketCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand, +} from "@aws-sdk/client-s3"; +import { + SQSClient, CreateQueueCommand, SendMessageCommand, ReceiveMessageCommand, DeleteQueueCommand, +} from "@aws-sdk/client-sqs"; +import { + DynamoDBClient, CreateTableCommand, PutItemCommand, GetItemCommand, DeleteTableCommand, waitUntilTableExists, +} from "@aws-sdk/client-dynamodb"; + +// The SDK reads AWS_ENDPOINT_URL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION +// from the environment by itself; they are spelled out here to show what each one is. +const config = { + endpoint: process.env.AWS_ENDPOINT_URL, // e.g. http://localhost:8080 + region: process.env.AWS_REGION ?? "us-east-1", + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, // a HomeCloud access key + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, +}; + +const sts = new STSClient(config); +const s3 = new S3Client({ ...config, forcePathStyle: true }); // buckets by path, not by host name +const sqs = new SQSClient(config); +const ddb = new DynamoDBClient(config); + +const id = await sts.send(new GetCallerIdentityCommand({})); +console.log("caller:", id.Arn); + +const suffix = Date.now().toString(36); +const bucket = `sdk-js-${suffix}`; +await s3.send(new CreateBucketCommand({ Bucket: bucket })); +await s3.send(new PutObjectCommand({ Bucket: bucket, Key: "hello.txt", Body: "hello from JavaScript" })); +const obj = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: "hello.txt" })); +console.log("s3:", await obj.Body.transformToString()); +await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: "hello.txt" })); +await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + +const { QueueUrl } = await sqs.send(new CreateQueueCommand({ QueueName: `sdk-js-${suffix}` })); +await sqs.send(new SendMessageCommand({ QueueUrl, MessageBody: "hello from JavaScript" })); +const recv = await sqs.send(new ReceiveMessageCommand({ QueueUrl, WaitTimeSeconds: 5 })); +console.log("sqs:", recv.Messages?.[0]?.Body); +await sqs.send(new DeleteQueueCommand({ QueueUrl })); + +const table = `sdk-js-${suffix}`; +await ddb.send(new CreateTableCommand({ + TableName: table, BillingMode: "PAY_PER_REQUEST", + AttributeDefinitions: [{ AttributeName: "id", AttributeType: "S" }], + KeySchema: [{ AttributeName: "id", KeyType: "HASH" }], +})); +await waitUntilTableExists({ client: ddb, maxWaitTime: 120 }, { TableName: table }); +await ddb.send(new PutItemCommand({ TableName: table, Item: { id: { S: "1" }, msg: { S: "hello from JavaScript" } } })); +const item = await ddb.send(new GetItemCommand({ TableName: table, Key: { id: { S: "1" } } })); +console.log("dynamodb:", item.Item.msg.S); +await ddb.send(new DeleteTableCommand({ TableName: table })); diff --git a/examples/sdk/javascript/package.json b/examples/sdk/javascript/package.json new file mode 100644 index 00000000..23677a79 --- /dev/null +++ b/examples/sdk/javascript/package.json @@ -0,0 +1,15 @@ +{ + "name": "homecloud-sdk-javascript", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node index.mjs" + }, + "dependencies": { + "@aws-sdk/client-dynamodb": "3.1151.0", + "@aws-sdk/client-s3": "3.1151.0", + "@aws-sdk/client-sqs": "3.1151.0", + "@aws-sdk/client-sts": "3.1151.0" + } +}