diff --git a/b2/compose.go b/b2/compose.go index d69334d..daaa69f 100644 --- a/b2/compose.go +++ b/b2/compose.go @@ -112,6 +112,13 @@ func (options *composeOptions) validate() (contentType string, info map[string]s return contentType, info, nil } +func validateComposePlatformSize(size, intLimit uint64) error { + if size > intLimit { + return fmt.Errorf("b2: Compose uploaded part exceeds this platform's maximum buffer size") + } + return nil +} + func (p ComposePart) validate(nonFinal bool) error { const minPartSize int64 = 5e6 const maxPartSize int64 = 5e9 @@ -125,8 +132,8 @@ func (p ComposePart) validate(nonFinal bool) error { if p.size > maxPartSize { return fmt.Errorf("b2: Compose uploaded part exceeds the 5 GB B2 part limit") } - if uint64(p.size) > uint64(^uint(0)>>1) { - return fmt.Errorf("b2: Compose uploaded part exceeds this platform's maximum buffer size") + if err := validateComposePlatformSize(uint64(p.size), uint64(^uint(0)>>1)); err != nil { + return err } if nonFinal && p.size < minPartSize { return fmt.Errorf("b2: Compose nonfinal part must be at least %d bytes", minPartSize) diff --git a/b2/compose_extended_wire_test.go b/b2/compose_extended_wire_test.go index cb864ee..e820a9b 100644 --- a/b2/compose_extended_wire_test.go +++ b/b2/compose_extended_wire_test.go @@ -31,6 +31,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/Backblaze/blazer/internal/b2wiretest" ) const composeMinPartSize = 5e6 @@ -60,6 +62,7 @@ type composeWire struct { starts []map[string]any calls []string copies []map[string]any + copyTimes []time.Time uploads []composeUpload finishes []map[string]any } @@ -149,6 +152,7 @@ func (w *composeWire) serveHTTP(rw http.ResponseWriter, r *http.Request) { } w.mu.Lock() w.copies = append(w.copies, body) + w.copyTimes = append(w.copyTimes, time.Now()) w.calls = append(w.calls, fmt.Sprintf("copy:%d", partNumber)) fail := w.failCopyPart == partNumber status := w.copyFailStatus @@ -160,7 +164,7 @@ func (w *composeWire) serveHTTP(rw http.ResponseWriter, r *http.Request) { w.mu.Unlock() if status != 0 { if status == http.StatusTooManyRequests { - rw.Header().Set("Retry-After", "1") + rw.Header().Set("Retry-After", "2") } rw.WriteHeader(status) code := "service_unavailable" @@ -290,6 +294,34 @@ func TestComposeRejectsMoreThan10000PartsBeforeStarting(t *testing.T) { } } +func TestComposeAccepts10000PartsValidation(t *testing.T) { + // https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api + wire := b2wiretest.New(t) + wire.Reply("b2_list_buckets", http.StatusOK, `{"buckets":[]}`) + wire.Reply("b2_create_bucket", http.StatusOK, noReplicationBucketJSON) + // Stop after validation, avoiding 10000 network copies in this boundary test. + wire.Reply("b2_start_large_file", http.StatusBadRequest, `{"status":400,"code":"bad_request","message":"validated 10000 parts"}`) + client, err := NewClient(context.Background(), "account", "key", APIBase(wire.URL())) + if err != nil { + t.Fatal(err) + } + bucket, err := client.NewBucket(context.Background(), "n", nil) + if err != nil { + t.Fatal(err) + } + parts := make([]ComposePart, 10000) + for i := range parts { + parts[i] = PartFromCopy("source", "bytes=0-4999999") + } + _, err = bucket.Compose(context.Background(), "at-limit", parts) + if err == nil || err.Error() != "b2_start_large_file: 400: validated 10000 parts" { + t.Fatalf("Compose error = %v, want scripted error after successful validation", err) + } + if got := len(wire.Calls("b2_start_large_file")); got != 1 { + t.Fatalf("start requests = %d, want 1", got) + } +} + func TestComposeAllSourceMixesPreservePartHashOrder(t *testing.T) { first := bytes.Repeat([]byte("x"), composeMinPartSize) firstHash := fmt.Sprintf("%x", sha1.Sum(first)) @@ -735,18 +767,18 @@ func TestComposeCopyPartReauthAndBackoff(t *testing.T) { for _, tc := range []struct { name string status, wantAuth int + minRetryWait time.Duration }{ - {"expired token", http.StatusUnauthorized, 2}, - {"request timeout", http.StatusRequestTimeout, 1}, - {"service unavailable", http.StatusServiceUnavailable, 1}, - {"gateway timeout", http.StatusGatewayTimeout, 1}, - {"rate limited", http.StatusTooManyRequests, 1}, + {"expired token", http.StatusUnauthorized, 2, 0}, + {"request timeout", http.StatusRequestTimeout, 1, time.Second}, + {"service unavailable", http.StatusServiceUnavailable, 1, time.Second}, + {"gateway timeout", http.StatusGatewayTimeout, 1, time.Second}, + {"rate limited", http.StatusTooManyRequests, 1, 2 * time.Second}, } { t.Run(tc.name, func(t *testing.T) { wire := newComposeWire(t) wire.copyFailStatus = tc.status bucket := wire.bucket(t) - started := time.Now() _, err := bucket.Compose(context.Background(), "retry-copy", []ComposePart{ PartFromCopy("source-one", "bytes=0-4999999"), PartFromCopy("source-two", "bytes=0-2"), @@ -754,11 +786,16 @@ func TestComposeCopyPartReauthAndBackoff(t *testing.T) { if err != nil { t.Fatal(err) } - if tc.status == http.StatusTooManyRequests && time.Since(started) < 900*time.Millisecond { - t.Fatal("copy_part retry ignored Retry-After: 1") - } wire.mu.Lock() defer wire.mu.Unlock() + if tc.minRetryWait > 0 { + if len(wire.copyTimes) < 2 { + t.Fatalf("copy_part status %d attempts = %d, want at least 2", tc.status, len(wire.copyTimes)) + } + if waited := wire.copyTimes[1].Sub(wire.copyTimes[0]); waited < tc.minRetryWait { + t.Fatalf("copy_part status %d retry waited %v, want at least %v", tc.status, waited, tc.minRetryWait) + } + } if len(wire.copies) != 3 || wire.authCalls != tc.wantAuth || !reflect.DeepEqual(wire.calls, []string{"start", "copy:1", "copy:1", "copy:2", "finish"}) { t.Fatalf("copies=%d auth=%d calls=%v", len(wire.copies), wire.authCalls, wire.calls) } diff --git a/b2/compose_fuzz_test.go b/b2/compose_fuzz_test.go index c00b8a0..628e427 100644 --- a/b2/compose_fuzz_test.go +++ b/b2/compose_fuzz_test.go @@ -1,6 +1,8 @@ package b2 import ( + "fmt" + "math" "regexp" "strconv" "strings" @@ -12,6 +14,20 @@ func TestComposeUploadedPartPlatformSize(t *testing.T) { if (err != nil) != (strconv.IntSize == 32) { t.Fatalf("2 GiB part on %d-bit platform: %v", strconv.IntSize, err) } + for _, size := range []uint64{2_147_483_646, 2_147_483_647, 2_147_483_648} { + err := validateComposePlatformSize(size, math.MaxInt32) + if size <= 2_147_483_647 && err != nil { + t.Fatalf("32-bit validation of %d = %v, want nil", size, err) + } + if size == 2_147_483_648 && (err == nil || err.Error() != "b2: Compose uploaded part exceeds this platform's maximum buffer size") { + t.Fatalf("32-bit validation of %d = %v, want platform-size error", size, err) + } + } + // The production call site still supplies the native int bound. + err = validateComposePlatformSize(2_147_483_648, uint64(^uint(0)>>1)) + if (err != nil) != (strconv.IntSize == 32) { + t.Fatalf("native limit on %d-bit platform = %v", strconv.IntSize, err) + } } func FuzzComposeRange(f *testing.F) { @@ -23,11 +39,74 @@ func FuzzComposeRange(f *testing.F) { if p.validate(nonFinal) == nil && value != "" && !regexp.MustCompile(`^bytes=[0-9]+-[0-9]+$`).MatchString(value) { t.Fatalf("accepted noncanonical range %q", value) } - // Canonical ranges of at least 5 MB must always validate. + // The documented 5 MB nonfinal minimum and 5 GB maximum are inclusive. + // https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api n := int64(len(value)) + for _, tc := range []struct { + value string + nonFinal bool + wantErr bool + }{ + {fmt.Sprintf("bytes=%d-%d", n, n+4_999_998), true, true}, + {fmt.Sprintf("bytes=%d-%d", n+1, n), nonFinal, true}, + {fmt.Sprintf("bytes=%d-%d", n, n+5_000_000_000), nonFinal, true}, + {"bytes=9000000-9000010", true, true}, + {"bytes=9000000-13999999", true, false}, + {"bytes=9000000-14999999", true, false}, + } { + if err := PartFromCopy("source", tc.value).validate(tc.nonFinal); (err != nil) != tc.wantErr { + t.Fatalf("validate(%q, nonFinal=%t) error = %v, want error=%t", tc.value, tc.nonFinal, err, tc.wantErr) + } + } + // Canonical ranges of at least 5 MB must always validate. canonical := PartFromCopy("source", "bytes="+strconv.FormatInt(n, 10)+"-"+strconv.FormatInt(n+4_999_999, 10)) if err := canonical.validate(nonFinal); err != nil { t.Fatal(err) } }) } + +func TestComposeRejectsZeroUploadedPart(t *testing.T) { + err := PartFromReader(strings.NewReader(""), 0).validate(false) + if err == nil || err.Error() != "b2: Compose uploaded part needs a positive size and no copy source" { + t.Fatalf("validate zero-size part = %v, want positive-size error", err) + } +} + +func TestComposeSingleByteRange(t *testing.T) { + // https://www.backblaze.com/apidocs/b2-copy-part (inclusive Range endpoints). + start, end, err := parseComposeRange("bytes=5-5") + if start != 5 || end != 5 || err != nil { + t.Fatalf("parseComposeRange = (%d, %d, %v), want (5, 5, nil)", start, end, err) + } +} + +func TestComposeRangeOrdering(t *testing.T) { + start, end, err := parseComposeRange("bytes=6-5") + if start != 0 || end != 0 || err == nil || err.Error() != `b2: Compose invalid copy range "bytes=6-5"` { + t.Fatalf("parseComposeRange = (%d, %d, %v), want reversed-range rejection", start, end, err) + } +} + +func TestComposeCopiedKnownTotalBoundary(t *testing.T) { + // Isolate aggregation from the separate per-part limit: B2's file cap is 10 TB. + // https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api + for _, tc := range []struct { + rangeEnd string + wantSize int64 + wantErr bool + }{ + {"5000000000098", 4_999_999_999_999, false}, + {"5000000000100", 5_000_000_000_001, true}, + } { + part := PartFromCopy("source", "bytes=100-"+tc.rangeEnd) + if size, known := part.knownSize(); !known || size != tc.wantSize { + t.Fatalf("knownSize = %d, %t; want %d, true", size, known, tc.wantSize) + } + err := validateComposeTotalSize([]ComposePart{part, part}) + const want = "b2: Compose part 2 brings the known total over the 10 TB (10,000,000,000,000 bytes) large-file limit" + if tc.wantErr && (err == nil || err.Error() != want) || !tc.wantErr && err != nil { + t.Fatalf("total error = %v, want over-limit=%t", err, tc.wantErr) + } + } +} diff --git a/b2/create_cors_filelock_test.go b/b2/create_cors_filelock_test.go index e3ca649..81666c0 100644 --- a/b2/create_cors_filelock_test.go +++ b/b2/create_cors_filelock_test.go @@ -1,13 +1,34 @@ package b2 import ( + "context" + "net/http" "reflect" "testing" "github.com/Backblaze/blazer/base" - "github.com/Backblaze/blazer/internal/b2types" + "github.com/Backblaze/blazer/internal/b2wiretest" ) +// Literal b2_create_bucket response, independent of the SDK's JSON structs. +// https://www.backblaze.com/apidocs/b2-create-bucket +const attrsWireBucketJSON = `{"bucketId":"bucket-id","bucketName":"bucket","bucketType":"allPrivate","bucketInfo":{},"lifecycleRules":[],"revision":1,"corsRules":[{"corsRuleName":"browser","allowedOrigins":["https://example.com"],"allowedHeaders":["authorization"],"allowedOperations":["b2_download_file_by_name"],"exposeHeaders":["x-bz-file-name"],"maxAgeSeconds":60}],"fileLockConfiguration":{"isClientAuthorizedToRead":true,"value":{"isFileLockEnabled":true,"defaultRetention":{"mode":"governance","period":{"duration":7,"unit":"days"}}}}}` + +func attrsWireBucket(t *testing.T) *b2Bucket { + t.Helper() + wire := b2wiretest.New(t) + wire.Reply("b2_create_bucket", http.StatusOK, attrsWireBucketJSON) + root, err := base.AuthorizeAccount(context.Background(), "account", "key", base.SetAPIBase(wire.URL())) + if err != nil { + t.Fatal(err) + } + bucket, err := root.CreateBucket(context.Background(), "bucket", "allPrivate", nil, nil, nil, nil, false, nil) + if err != nil { + t.Fatal(err) + } + return &b2Bucket{b: bucket} +} + func TestBucketAttrsCORSAndFileLock(t *testing.T) { wantCORS := []CORSRule{{ Name: "browser", @@ -17,17 +38,7 @@ func TestBucketAttrsCORSAndFileLock(t *testing.T) { ExposeHeaders: []string{"x-bz-file-name"}, MaxAgeSeconds: 60, }} - bucket := &b2Bucket{b: &base.Bucket{ - CORSRules: []b2types.CORSRule{{ - Name: wantCORS[0].Name, - AllowedOrigins: wantCORS[0].AllowedOrigins, - AllowedHeaders: wantCORS[0].AllowedHeaders, - AllowedOperations: wantCORS[0].AllowedOperations, - ExposeHeaders: wantCORS[0].ExposeHeaders, - MaxAgeSeconds: wantCORS[0].MaxAgeSeconds, - }}, - FileLockEnabled: true, - }} + bucket := attrsWireBucket(t) attrs := bucket.attrs() if !reflect.DeepEqual(attrs.CORSRules, wantCORS) { diff --git a/b2/default_retention_test.go b/b2/default_retention_test.go index f4f7d54..f5c6e2f 100644 --- a/b2/default_retention_test.go +++ b/b2/default_retention_test.go @@ -16,21 +16,11 @@ package b2 import ( "testing" - - "github.com/Backblaze/blazer/base" - "github.com/Backblaze/blazer/internal/b2types" ) func TestBucketAttrsDefaultRetention(t *testing.T) { - bucket := &b2Bucket{b: &base.Bucket{ - DefaultRetention: &b2types.Retention{ - Mode: "governance", - Period: &b2types.RetentionPeriod{ - Duration: 7, - Unit: "days", - }, - }, - }} + // https://www.backblaze.com/apidocs/b2-create-bucket (fileLockConfiguration). + bucket := attrsWireBucket(t) attrs := bucket.attrs() if attrs.DefaultRetention == nil { diff --git a/b2/delete_bypass_wire_test.go b/b2/delete_bypass_wire_test.go index 56f44c9..eb7f8fe 100644 --- a/b2/delete_bypass_wire_test.go +++ b/b2/delete_bypass_wire_test.go @@ -58,6 +58,9 @@ func TestObjectDeleteBypassGovernanceReachesTheWire(t *testing.T) { if v, ok := sent["bypassGovernance"]; ok { t.Errorf("plain Delete sent bypassGovernance = %v, want the field omitted", v) } + if sent["fileId"] != "id" || sent["fileName"] != "name" { + t.Errorf("plain Delete body = %v, want fileId=id and fileName=name", sent) + } sent = nil if err := newObject().Delete(ctx, BypassGovernance()); err != nil { @@ -66,4 +69,7 @@ func TestObjectDeleteBypassGovernanceReachesTheWire(t *testing.T) { if sent["bypassGovernance"] != true { t.Errorf("Delete(BypassGovernance) sent bypassGovernance = %v, want true", sent["bypassGovernance"]) } + if sent["fileId"] != "id" || sent["fileName"] != "name" { + t.Errorf("bypass Delete body = %v, want fileId=id and fileName=name", sent) + } } diff --git a/b2/downloadbyid_wire_test.go b/b2/downloadbyid_wire_test.go deleted file mode 100644 index d088b36..0000000 --- a/b2/downloadbyid_wire_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package b2 - -import ( - "context" - "fmt" - "io" - "net/http" - "net/http/httptest" - "testing" -) - -func TestBucketDownloadFileByIDReachesTheWire(t *testing.T) { - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/b2api/v4/b2_authorize_account": - _, _ = fmt.Fprintf(w, updateAuthJSON, srv.URL) - case "/b2api/v4/b2_list_buckets": - _, _ = fmt.Fprint(w, `{"buckets":[]}`) - case "/b2api/v4/b2_create_bucket": - _, _ = fmt.Fprint(w, noReplicationBucketJSON) - case "/b2api/v4/b2_download_file_by_id": - if r.Method != http.MethodGet { - t.Errorf("method = %s, want GET", r.Method) - } - if got := r.URL.Query().Get("fileId"); got != "file-id" { - t.Errorf("fileId = %q, want file-id", got) - } - if got := r.Header.Get("Authorization"); got != "t" { - t.Errorf("Authorization = %q, want t", got) - } - w.Header().Set("Content-Length", "2") - w.Header().Set("X-Bz-File-Id", "file-id") - _, _ = fmt.Fprint(w, "ok") - default: - t.Errorf("unexpected request: %s", r.URL.String()) - http.NotFound(w, r) - } - })) - defer srv.Close() - - ctx := context.Background() - client, err := NewClient(ctx, "a", "k", APIBase(srv.URL)) - if err != nil { - t.Fatal(err) - } - bucket, err := client.NewBucket(ctx, "n", &BucketAttrs{Type: Private}) - if err != nil { - t.Fatal(err) - } - reader, err := bucket.DownloadFileByID(ctx, "file-id") - if err != nil { - t.Fatal(err) - } - got, readErr := io.ReadAll(reader) - closeErr := reader.Close() - if readErr != nil { - t.Fatal(readErr) - } - if closeErr != nil { - t.Fatal(closeErr) - } - if string(got) != "ok" { - t.Errorf("body = %q, want ok", got) - } -} diff --git a/b2/iterator_advance_test.go b/b2/iterator_advance_test.go index ad172e1..19c10f9 100644 --- a/b2/iterator_advance_test.go +++ b/b2/iterator_advance_test.go @@ -3,6 +3,7 @@ package b2 import ( "context" "fmt" + "runtime" "strings" "sync/atomic" "testing" @@ -14,6 +15,7 @@ import ( type scriptRoot struct { *testRoot names func(call int, cont string) (fs []b2FileInterface, next string, handled bool) + versions func(call int, name, id string) (fs []b2FileInterface, nextName, nextID string) unfinished func(call int, cont string) (fs []b2FileInterface, next string, handled bool) calls int32 } @@ -31,6 +33,15 @@ type scriptBucket struct { r *scriptRoot } +func (b *scriptBucket) listFileVersions(ctx context.Context, count int, name, id, pfx, del string) ([]b2FileInterface, string, string, error) { + if b.r.versions != nil { + n := int(atomic.AddInt32(&b.r.calls, 1)) + fs, nextName, nextID := b.r.versions(n, name, id) + return fs, nextName, nextID, nil + } + return b.testBucket.listFileVersions(ctx, count, name, id, pfx, del) +} + func (b *scriptBucket) listFileNames(ctx context.Context, count int, cont, pfx, del string) ([]b2FileInterface, string, error) { n := int(atomic.AddInt32(&b.r.calls, 1)) if b.r.names != nil { @@ -74,19 +85,55 @@ func newScriptRoot() *scriptRoot { // A backend that keeps returning an empty page with the same cursor must make // the iterator fail with an error, not hang or overflow the stack. func TestListFailsWhenAnEmptyPageDoesNotAdvance(t *testing.T) { - root := newScriptRoot() - root.names = func(int, string) ([]b2FileInterface, string, bool) { return nil, "a", true } - bucket, ctx := scriptedBucket(t, root) - - iter := bucket.List(ctx, ListPageSize(100)) - for iter.Next() { - } - err := iter.Err() - if err == nil || !strings.Contains(err.Error(), "not advancing") { - t.Fatalf("Err() = %v, want a 'not advancing' error", err) - } - if got := atomic.LoadInt32(&root.calls); got > 3 { - t.Errorf("backend was asked %d times, want the listing to stop within a few requests", got) + for _, mode := range []string{"names", "versions", "unfinished"} { + t.Run(mode, func(t *testing.T) { + root := newScriptRoot() + stalled := func(n int, _ string) ([]b2FileInterface, string, bool) { + if n > 2 { + return nil, "", true + } + return nil, "a", true + } + root.names, root.unfinished = stalled, stalled + root.versions = func(n int, _, id string) ([]b2FileInterface, string, string) { + // A new ID under the same name advances; only the third page stalls. + switch n { + case 1: + return nil, "a", "first" + case 2: + if id != "first" { + t.Errorf("second page ID = %q, want first", id) + } + return nil, "a", "second" + case 3: + if id != "second" { + t.Errorf("third page ID = %q, want second", id) + } + return nil, "a", "second" + } + return nil, "", "" + } + bucket, ctx := scriptedBucket(t, root) + opts := []ListOption{ListPageSize(100)} + wantCalls := int32(2) + switch mode { + case "versions": + opts = append(opts, ListHidden()) + wantCalls = 3 + case "unfinished": + opts = append(opts, ListUnfinished()) + } + iter := bucket.List(ctx, opts...) + if iter.Next() { + t.Fatal("empty pages yielded an object") + } + if err := iter.Err(); err == nil || !strings.Contains(err.Error(), "not advancing") { + t.Fatalf("Err() = %v, want a 'not advancing' error", err) + } + if got := atomic.LoadInt32(&root.calls); got != wantCalls { + t.Errorf("backend calls = %d, want %d", got, wantCalls) + } + }) } } @@ -94,8 +141,17 @@ func TestListFailsWhenAnEmptyPageDoesNotAdvance(t *testing.T) { // walked, without recursion. func TestListWalksManyConsecutiveEmptyPages(t *testing.T) { const empties = 5000 + firstDepth, maximumDepth := 0, 0 root := newScriptRoot() root.names = func(n int, cont string) ([]b2FileInterface, string, bool) { + var stack [128]uintptr + depth := runtime.Callers(0, stack[:]) + if n == 1 { + firstDepth = depth + } + if depth > maximumDepth { + maximumDepth = depth + } if n <= empties { return nil, fmt.Sprintf("%06d", n), true // digits sort before "a" } @@ -114,6 +170,12 @@ func TestListWalksManyConsecutiveEmptyPages(t *testing.T) { if len(got) != 3 { t.Fatalf("listed %v after %d empty pages, want [a b c]", got, empties) } + if maximumDepth > firstDepth+4 { + t.Errorf("stack depth grew from %d to %d, want constant depth", firstDepth, maximumDepth) + } + if calls := atomic.LoadInt32(&root.calls); calls != 5001 { + t.Errorf("listing calls = %d, want 5001", calls) + } } // listUnfinishedLargeFiles is the third lister the fix touches; it had no diff --git a/b2/replication_setup_test.go b/b2/replication_setup_test.go index 1edca72..0dfbfbf 100644 --- a/b2/replication_setup_test.go +++ b/b2/replication_setup_test.go @@ -263,6 +263,15 @@ func TestSetupReplicationIdempotency(t *testing.T) { if (err != nil) != tc.wantErr { t.Fatalf("SetupReplication() = %v, wantErr=%t", err, tc.wantErr) } + if tc.wantErr { + want := `setup replication: rule name "rule" is already in use with different options` + if tc.name == "different destination key mapping" { + want = "setup replication: destination already maps the source key to a different application key ID" + } + if err.Error() != want { + t.Fatalf("SetupReplication error = %q, want %q", err, want) + } + } if root.keyCalls != 0 || len(srcBackend.updates) != 0 || len(dstBackend.updates) != 0 { t.Fatalf("repeat changed state: key calls=%d source updates=%d destination updates=%d", root.keyCalls, len(srcBackend.updates), len(dstBackend.updates)) } @@ -472,8 +481,8 @@ func TestSetupReplicationRejectsConflictingOptions(t *testing.T) { func TestSetupReplicationRejectsEmptyKeyIDs(t *testing.T) { for _, tc := range []struct{ sourceKeyID, destinationKeyID string }{{"", "dst-key"}, {"src-key", ""}} { _, srcBackend, dstBackend, src, dst := replicationTestPair(t, nil, nil) - if err := SetupReplication(context.Background(), src, dst, tc.sourceKeyID, tc.destinationKeyID); err == nil { - t.Fatalf("SetupReplication(%q, %q) succeeded with empty key ID", tc.sourceKeyID, tc.destinationKeyID) + if err := SetupReplication(context.Background(), src, dst, tc.sourceKeyID, tc.destinationKeyID); err == nil || err.Error() != "setup replication: source and destination application key IDs are required" { + t.Fatalf("SetupReplication(%q, %q) = %v, want required-key-IDs error", tc.sourceKeyID, tc.destinationKeyID, err) } if len(srcBackend.updates) != 0 || len(dstBackend.updates) != 0 { t.Fatal("empty key ID caused an update") @@ -484,7 +493,7 @@ func TestSetupReplicationRejectsEmptyKeyIDs(t *testing.T) { func TestReplicationStatusRejectsUnreadableConfig(t *testing.T) { _, srcBackend, _, src, _ := replicationTestPair(t, nil, nil) srcBackend.unreadable = true - if status, err := ReplicationStatus(context.Background(), src); err == nil { - t.Fatalf("ReplicationStatus() = %+v, nil; want unreadable-config error", status) + if status, err := ReplicationStatus(context.Background(), src); err == nil || err.Error() != "replication status: cannot read existing replication config" { + t.Fatalf("ReplicationStatus() = %+v, %v; want unreadable-config error", status, err) } } diff --git a/b2/sdkharness_contract_test.go b/b2/sdkharness_contract_test.go index b28fc31..203a2c9 100644 --- a/b2/sdkharness_contract_test.go +++ b/b2/sdkharness_contract_test.go @@ -1,9 +1,14 @@ package b2 import ( + "fmt" + "go/ast" + "go/parser" + "go/token" "os" "path/filepath" "runtime" + "strconv" "strings" "testing" ) @@ -57,6 +62,9 @@ func TestSDKHarnessContract(t *testing.T) { if !strings.Contains(string(scenario), "replace github.com/Backblaze/blazer => $REPO_ROOT") { t.Fatalf("scenario does not select the checked-out module: %s", scenarioPath) } + if err := sdkHarnessScenarioChecks(string(scenario)); err != nil { + t.Fatalf("scenario %s: %v", scenarioPath, err) + } } for level, count := range wantCount { if counts[level] != count { @@ -73,3 +81,247 @@ func TestSDKHarnessContract(t *testing.T) { } } } + +// Inspect the embedded Go entry point, rather than accepting shell boilerplate +// as evidence of a scenario. This structural gate does not replace running it. +func sdkHarnessScenarioChecks(script string) error { + script = strings.ReplaceAll(script, "\r\n", "\n") + _, program, ok := strings.Cut(script, "\npackage main\n") + if !ok { + return fmt.Errorf("no embedded Go program") + } + program, _, _ = strings.Cut(program, "\n__EOF_GO__") + file, err := parser.ParseFile(token.NewFileSet(), "scenario.go", "package main\n"+program, 0) + if err != nil { + return err + } + functions := make(map[string]*ast.FuncDecl) + for _, declaration := range file.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok && function.Recv == nil { + functions[function.Name.Name] = function + } + } + entry := functions["runCheck"] + if entry == nil { + entry = functions["run"] + } + if entry == nil { + entry = functions["check"] + } + if entry == nil { + entry = functions["main"] // The inbound-webhook scenario runs directly in main. + } + if entry == nil || functions["main"] == nil { + return fmt.Errorf("no scenario entry point") + } + called, conditional, asserted := entry == functions["main"], false, false + ast.Inspect(functions["main"].Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + if name, ok := call.Fun.(*ast.Ident); ok && name.Name == entry.Name.Name { + called = true + } + } + return true + }) + seenAssertions := make(map[string]bool) + var inspect func(ast.Node) bool + inspect = func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + switch name := call.Fun.(type) { + case *ast.Ident: + asserted = asserted || name.Name == "failf" || name.Name == "fail" + case *ast.SelectorExpr: + if pkg, ok := name.X.(*ast.Ident); ok { + asserted = asserted || pkg.Name == "fmt" && name.Sel.Name == "Errorf" + } + } + if name, ok := call.Fun.(*ast.Ident); ok && (strings.HasPrefix(name.Name, "assert") || strings.HasPrefix(name.Name, "check")) && !seenAssertions[name.Name] { + seenAssertions[name.Name] = true + if helper := functions[name.Name]; helper != nil { + ast.Inspect(helper.Body, inspect) + } + } + } + if _, ok := node.(*ast.IfStmt); ok { + conditional = true + } + return true + } + ast.Inspect(entry.Body, inspect) + if !called || !conditional || !asserted || !sdkHarnessResultConsumed(functions["main"], entry) { + return fmt.Errorf("scenario must be called by main, consume its result, and contain conditions and failure assertions") + } + return nil +} + +func sdkHarnessResultConsumed(main, entry *ast.FuncDecl) bool { + if main == entry { + return true + } + resultNames := make(map[string]bool) + consumed := false + ast.Inspect(main.Body, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.AssignStmt: + for name := range sdkHarnessEntryResultNames(node, entry.Name.Name) { + resultNames[name] = true + } + case *ast.DeclStmt: + for name := range sdkHarnessEntryResultNames(node, entry.Name.Name) { + resultNames[name] = true + } + case *ast.IfStmt: + names := sdkHarnessEntryResultNames(node.Init, entry.Name.Name) + if sdkHarnessCallsEntry(node.Cond, entry.Name.Name) || sdkHarnessReferencesResult(node.Cond, names) { + consumed = consumed || sdkHarnessReportsFailure(node.Body, names) + } + if sdkHarnessReferencesResult(node.Cond, resultNames) { + consumed = consumed || sdkHarnessReportsFailure(node.Body, resultNames) + } + case *ast.SwitchStmt: + if sdkHarnessReferencesResult(node.Tag, resultNames) { + consumed = consumed || sdkHarnessReportsFailure(node.Body, resultNames) + } + case *ast.TypeSwitchStmt: + if sdkHarnessReferencesResult(node.Assign, resultNames) { + consumed = consumed || sdkHarnessReportsFailure(node.Body, resultNames) + } + } + return true + }) + return consumed +} + +func sdkHarnessEntryResultNames(node ast.Node, entry string) map[string]bool { + names := make(map[string]bool) + var lhs, rhs []ast.Expr + switch node := node.(type) { + case *ast.AssignStmt: + lhs, rhs = node.Lhs, node.Rhs + case *ast.DeclStmt: + declaration, ok := node.Decl.(*ast.GenDecl) + if !ok || declaration.Tok != token.VAR || len(declaration.Specs) != 1 { + return names + } + value, ok := declaration.Specs[0].(*ast.ValueSpec) + if !ok { + return names + } + lhs = make([]ast.Expr, len(value.Names)) + for i, name := range value.Names { + lhs[i] = name + } + rhs = value.Values + default: + return names + } + for i, value := range rhs { + if i >= len(lhs) || !sdkHarnessCallsEntry(value, entry) { + continue + } + if name, ok := lhs[i].(*ast.Ident); ok { + names[name.Name] = true + } + } + return names +} + +func sdkHarnessCallsEntry(node ast.Node, entry string) bool { + called := false + ast.Inspect(node, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + name, ok := call.Fun.(*ast.Ident) + if ok && name.Name == entry { + called = true + } + return true + }) + return called +} + +func sdkHarnessReferencesResult(node ast.Node, names map[string]bool) bool { + if node == nil || len(names) == 0 { + return false + } + referenced := false + ast.Inspect(node, func(node ast.Node) bool { + identifier, ok := node.(*ast.Ident) + if ok && names[identifier.Name] { + referenced = true + } + return true + }) + return referenced +} + +func sdkHarnessReportsFailure(body *ast.BlockStmt, errors map[string]bool) bool { + reported := false + ast.Inspect(body, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.CallExpr: + switch name := node.Fun.(type) { + case *ast.Ident: + reported = reported || name.Name == "fail" || name.Name == "failf" + case *ast.SelectorExpr: + pkg, ok := name.X.(*ast.Ident) + if ok && pkg.Name == "os" && name.Sel.Name == "Exit" && len(node.Args) == 1 { + reported = reported || sdkHarnessNonZero(node.Args[0]) + } + } + case *ast.ReturnStmt: + for _, result := range node.Results { + if len(errors) == 0 || sdkHarnessReferencesResult(result, errors) { + reported = true + } + } + } + return true + }) + return reported +} + +func sdkHarnessNonZero(expression ast.Expr) bool { + literal, ok := expression.(*ast.BasicLit) + if !ok || literal.Kind != token.INT { + return false + } + value, err := strconv.ParseInt(literal.Value, 0, 64) + return err == nil && value != 0 +} + +func TestSDKHarnessAcceptsScenario(t *testing.T) { + const script = "replace github.com/Backblaze/blazer => $REPO_ROOT\npackage main\n" + + `func runCheck() error { if false { return fmt.Errorf("failed") }; return nil } +func main() { if err := runCheck(); err != nil { os.Exit(1) } } +__EOF_GO__ +` + for _, tc := range []struct { + name, script string + }{ + {name: "LF", script: script}, + {name: "CRLF", script: strings.ReplaceAll(script, "\n", "\r\n")}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := sdkHarnessScenarioChecks(tc.script); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestSDKHarnessRejectsNoOpScenario(t *testing.T) { + for _, program := range []string{ + "func main() {}", + "func runCheck() error { return nil }; func main() { runCheck() }", + "func fail() {}; func runCheck() error { if true { fail() }; return nil }; func main() { runCheck() }", + `func runCheck() error { if false { return fmt.Errorf("failed") }; return nil }; func main() {}`, + `func runCheck() error { if false { return fmt.Errorf("failed") }; return nil }; func main() { if err := runCheck(); false { fmt.Println(err); os.Exit(1) } }`, + } { + if err := sdkHarnessScenarioChecks("replace github.com/Backblaze/blazer => $REPO_ROOT\npackage main\n" + program); err == nil { + t.Fatalf("accepted no-op scenario %q", program) + } + } +} diff --git a/b2/ssec_test.go b/b2/ssec_test.go index c9fda80..6f37300 100644 --- a/b2/ssec_test.go +++ b/b2/ssec_test.go @@ -146,7 +146,8 @@ func TestSSECKeyEveryFormatRedactsMaterial(t *testing.T) { } } -func TestSSECReaderMethodTypeChangeDocumented(t *testing.T) { +// This is a documentation gate, not a test of SSE-C request behavior. +func TestDocsSSECReaderMethodTypeChangeDocumented(t *testing.T) { // The note lives in a changie fragment until a release batches it into CHANGELOG.md, so // look in both: one line must carry the Breaking marker and every detail. files, err := filepath.Glob(filepath.Join("..", ".changes", "*", "*.yaml")) diff --git a/b2/ssec_wire_test.go b/b2/ssec_wire_test.go index ca196d4..43e012c 100644 --- a/b2/ssec_wire_test.go +++ b/b2/ssec_wire_test.go @@ -734,7 +734,7 @@ func TestDownloadFileByIDWithSSECWire(t *testing.T) { assertNoSSECWireHeaders(t, requests[1].headers) } -func TestSSECReadRejectsMissingAndWrongKeys(t *testing.T) { +func TestSSECReadSendsRequestedKeyAndPropagatesRejection(t *testing.T) { wire := newSSECPublicWire(t) bucket := wire.newBucket(t) ctx := context.Background() @@ -750,9 +750,11 @@ func TestSSECReadRejectsMissingAndWrongKeys(t *testing.T) { }{ {"missing", func() *Reader { return bucket.Object("encrypted").NewReader(ctx) }, "SSE-C key required"}, {"wrong", func() *Reader { return bucket.Object("encrypted").NewReader(ctx, wrong) }, "SSE-C key required"}, - {"nil", func() *Reader { return bucket.Object("encrypted").NewReader(ctx, nil) }, "SSE-C key is nil"}, } { t.Run(tc.name, func(t *testing.T) { + wire.mu.Lock() + before := len(wire.requests) + wire.mu.Unlock() reader := tc.newReader() _, err := io.ReadAll(reader) if closeErr := reader.Close(); err == nil { @@ -761,10 +763,46 @@ func TestSSECReadRejectsMissingAndWrongKeys(t *testing.T) { if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("read error = %v, want %q", err, tc.want) } + // The server rejects these keys; the SDK must send the requested + // headers and propagate that response, not implement its own key check. + wire.mu.Lock() + requests := append([]ssecPublicRequest(nil), wire.requests[before:]...) + wire.mu.Unlock() + if len(requests) != 1 || requests[0].method != http.MethodGet || requests[0].path != "/file/bucket/encrypted" { + t.Fatalf("download requests = %+v, want one GET /file/bucket/encrypted", requests) + } + if tc.name == "missing" { + assertNoSSECWireHeaders(t, requests[0].headers) + } else { + raw := []byte("abcdef0123456789abcdef0123456789") + digest := md5.Sum(raw) + assertSSECWireHeaders(t, requests[0].headers, base64.StdEncoding.EncodeToString(raw), base64.StdEncoding.EncodeToString(digest[:])) + } }) } } +func TestSSECReadRejectsExplicitNilKeyBeforeRequest(t *testing.T) { + wire := newSSECPublicWire(t) + bucket := wire.newBucket(t) + wire.mu.Lock() + before := len(wire.requests) + wire.mu.Unlock() + reader := bucket.Object("encrypted").NewReader(context.Background(), nil) + _, err := io.ReadAll(reader) + if err == nil || err.Error() != "b2: SSE-C key is nil" { + t.Fatalf("read error = %v, want explicit nil-key error", err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + wire.mu.Lock() + defer wire.mu.Unlock() + if got := len(wire.requests) - before; got != 0 { + t.Fatalf("nil key caused %d requests, want 0", got) + } +} + func TestSSECKeyCannotBeClientWriterDefault(t *testing.T) { wire := newSSECPublicWire(t) _, err := NewClient(context.Background(), "account", "key", APIBase(wire.srv.URL), DefaultWriterOptions(WithSSECKey(wire.key))) diff --git a/b2/update_cors_name_test.go b/b2/update_cors_name_test.go index b6d042f..225185c 100644 --- a/b2/update_cors_name_test.go +++ b/b2/update_cors_name_test.go @@ -2,12 +2,7 @@ package b2 import ( "context" - "encoding/json" - "fmt" - "io" "net/http" - "net/http/httptest" - "strings" "testing" "github.com/Backblaze/blazer/base" @@ -39,6 +34,7 @@ func TestUpdateBucketSendsCORSRuleName(t *testing.T) { Name: "browser", AllowedOrigins: []string{"https://example.com"}, AllowedOperations: []string{"b2_download_file_by_name"}, + MaxAgeSeconds: 60, }}}) if err != nil { t.Fatalf("updateBucket: %v", err) @@ -48,37 +44,21 @@ func TestUpdateBucketSendsCORSRuleName(t *testing.T) { "corsRuleName": "browser", "allowedOrigins": []string{"https://example.com"}, "allowedOperations": []string{"b2_download_file_by_name"}, + "maxAgeSeconds": 60, }}) } func TestReadModifyWriteKeepsCORSRuleName(t *testing.T) { const withCORS = `{"bucketId":"bid","bucketName":"n","bucketType":"allPrivate","bucketInfo":{},"lifecycleRules":[],"revision":2,` + - `"corsRules":[{"corsRuleName":"browser","allowedOrigins":["https://example.com"],"allowedOperations":["b2_download_file_by_name"]}],` + + `"corsRules":[{"corsRuleName":"browser","allowedOrigins":["https://example.com"],"allowedOperations":["b2_download_file_by_name"],"maxAgeSeconds":60}],` + `"fileLockConfiguration":{"isClientAuthorizedToRead":true,"value":{"isFileLockEnabled":false,"defaultRetention":{"mode":null,"period":null}}},` + `"replicationConfiguration":{"isClientAuthorizedToRead":true,"value":{"asReplicationSource":null,"asReplicationDestination":null}}}` - var srv *httptest.Server - var sent map[string]any - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/b2_authorize_account"): - fmt.Fprintf(w, updateAuthJSON, srv.URL) - case strings.HasSuffix(r.URL.Path, "/b2_create_bucket"): - fmt.Fprint(w, withCORS) - case strings.HasSuffix(r.URL.Path, "/b2_update_bucket"): - body, _ := io.ReadAll(r.Body) - if err := json.Unmarshal(body, &sent); err != nil { - t.Errorf("decode update body: %v", err) - } - fmt.Fprint(w, withCORS) - default: - t.Errorf("unexpected request path: %s", r.URL.Path) - http.NotFound(w, r) - } - })) - defer srv.Close() + srv := b2wiretest.New(t) + srv.Reply("b2_create_bucket", http.StatusOK, withCORS) + srv.Reply("b2_update_bucket", http.StatusOK, withCORS) ctx := context.Background() - root, err := base.AuthorizeAccount(ctx, "a", "k", base.SetAPIBase(srv.URL)) + root, err := base.AuthorizeAccount(ctx, "a", "k", base.SetAPIBase(srv.URL())) if err != nil { t.Fatal(err) } @@ -94,11 +74,8 @@ func TestReadModifyWriteKeepsCORSRuleName(t *testing.T) { if err := bucket.updateBucket(ctx, attrs); err != nil { t.Fatalf("updateBucket: %v", err) } - rules, ok := sent["corsRules"].([]any) - if !ok || len(rules) != 1 { - t.Fatalf("update request corsRules = %v, want the existing rule", sent["corsRules"]) - } - if name := rules[0].(map[string]any)["corsRuleName"]; name != "browser" { - t.Errorf("update request corsRuleName = %v, want browser", name) - } + b2wiretest.AssertJSON(t, srv.Last(t, "b2_update_bucket"), "corsRules", []map[string]any{{ + "corsRuleName": "browser", "allowedOrigins": []string{"https://example.com"}, + "allowedOperations": []string{"b2_download_file_by_name"}, "maxAgeSeconds": 60, + }}) } diff --git a/b2/webhook_test.go b/b2/webhook_test.go index d9fe88b..ea296f7 100644 --- a/b2/webhook_test.go +++ b/b2/webhook_test.go @@ -20,7 +20,6 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "os" "reflect" "strings" "testing" @@ -54,7 +53,7 @@ func TestVerifySignature(t *testing.T) { } } -func TestVerifySignatureLargeBodyAndConstantTimeComparison(t *testing.T) { +func TestVerifySignatureLargeBody(t *testing.T) { body := bytes.Repeat([]byte("raw webhook bytes\n"), 4096) header := webhookTestSignature(body, "large-body-secret") if err := VerifySignature(body, header, "large-body-secret"); err != nil { @@ -64,14 +63,6 @@ func TestVerifySignatureLargeBodyAndConstantTimeComparison(t *testing.T) { if err := VerifySignature(body, header, "large-body-secret"); err == nil { t.Fatal("VerifySignature() accepted a changed final byte") } - // Guard the security property that cannot be measured reliably by timing a test. - source, err := os.ReadFile("webhook.go") - if err != nil { - t.Fatal(err) - } - if !bytes.Contains(source, []byte("hmac.Equal(signature, mac.Sum(nil))")) || bytes.Contains(source, []byte("signature ==")) { - t.Fatal("VerifySignature must compare decoded HMAC bytes with hmac.Equal, not string equality") - } } func TestVerifySignatureKnownSHA256Vector(t *testing.T) { @@ -230,10 +221,20 @@ func TestParseEventKnownAndFutureEventTypes(t *testing.T) { for i, eventType := range types { want.Events[i] = Event{EventID: eventType, EventType: eventType} } - body, err := json.Marshal(want) - if err != nil { - t.Fatal(err) - } + body := []byte(`{"events":[ + {"eventId":"b2:ObjectCreated:Upload","eventType":"b2:ObjectCreated:Upload"}, + {"eventId":"b2:ObjectCreated:MultipartUpload","eventType":"b2:ObjectCreated:MultipartUpload"}, + {"eventId":"b2:ObjectCreated:Copy","eventType":"b2:ObjectCreated:Copy"}, + {"eventId":"b2:ObjectCreated:Replica","eventType":"b2:ObjectCreated:Replica"}, + {"eventId":"b2:ObjectCreated:MultipartReplica","eventType":"b2:ObjectCreated:MultipartReplica"}, + {"eventId":"b2:ObjectDeleted:Delete","eventType":"b2:ObjectDeleted:Delete"}, + {"eventId":"b2:ObjectDeleted:LifecycleRule","eventType":"b2:ObjectDeleted:LifecycleRule"}, + {"eventId":"b2:HideMarkerCreated:Hide","eventType":"b2:HideMarkerCreated:Hide"}, + {"eventId":"b2:HideMarkerCreated:LifecycleRule","eventType":"b2:HideMarkerCreated:LifecycleRule"}, + {"eventId":"b2:MultipartUploadCreated:LiveRead","eventType":"b2:MultipartUploadCreated:LiveRead"}, + {"eventId":"b2:TestEvent","eventType":"b2:TestEvent"}, + {"eventId":"b2:FutureCategory:FutureEvent","eventType":"b2:FutureCategory:FutureEvent"} + ]}`) got, err := ParseEvent(body) if err != nil { t.Fatal(err)