From 4cd28bb4155add82a0f9bef3923de26ca7a2d1e8 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:28:16 +0200 Subject: [PATCH] Restore S3 carver wiring and reject the unimplemented local carver --- cmd/api/carver_test.go | 57 +++++ cmd/api/main.go | 8 +- cmd/tls/carver_test.go | 57 +++++ cmd/tls/main.go | 9 +- deploy/config/api.yml | 6 +- deploy/config/tls.yml | 6 +- .../service-config/ServiceConfigPage.tsx | 4 +- pkg/carves/carver_test.go | 238 ++++++++++++++++++ pkg/carves/carves.go | 39 ++- pkg/carves/s3.go | 14 +- pkg/config/flags.go | 4 +- pkg/config/validation.go | 20 +- pkg/config/validation_test.go | 10 + 13 files changed, 437 insertions(+), 35 deletions(-) create mode 100644 cmd/api/carver_test.go create mode 100644 cmd/tls/carver_test.go create mode 100644 pkg/carves/carver_test.go diff --git a/cmd/api/carver_test.go b/cmd/api/carver_test.go new file mode 100644 index 000000000..39115aa90 --- /dev/null +++ b/cmd/api/carver_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/jmpsec/osctrl/pkg/carves" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// Startup builds the carver from the resolved carver section. These pin the +// YAML -> ServiceParameters -> carver wiring that once silently dropped S3. +func TestCarverWiringFromYAML(t *testing.T) { + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none")) + tests := []struct { + name string + carver string + wantS3 bool + wantErr string + }{ + {name: "db", carver: "carver:\n type: db\n"}, + {name: "s3", carver: "carver:\n type: s3\n s3:\n bucket: carves\n region: us-east-1\n", wantS3: true}, + {name: "s3 incomplete", carver: "carver:\n type: s3\n s3:\n bucket: carves\n", wantErr: "carver.s3.region"}, + {name: "local", carver: "carver:\n type: local\n local:\n carvesDir: /tmp\n", wantErr: `carver type "local" is not supported`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg, err := loadYAMLConfiguration(writeTempConfig(t, "service:\n auth: none\n"+tt.carver)) + if err == nil { + params := loadedYAMLToServiceParams(cfg, "api.yml") + db, derr := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if derr != nil { + t.Fatal(derr) + } + var c *carves.Carves + c, err = carves.CreateFileCarvesFromConfig(db, params.Carver) + if err == nil { + if c.Carver != tt.name { + t.Fatalf("carver = %q", c.Carver) + } + if (c.S3 != nil) != tt.wantS3 { + t.Fatalf("S3 carver initialized = %v, want %v", c.S3 != nil, tt.wantS3) + } + } + } + if tt.wantErr == "" && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Fatalf("error = %v, want %q", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 3ac7cd474..4341d4571 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -487,8 +487,6 @@ func osctrlAPIService() { } consolemgr.SetLogReader(logReader) fileexplorermgr.SetLogReader(logReader) - log.Info().Msg("Initialize carves") - filecarves = carves.CreateFileCarves(db.Conn, flagParams.Carver.Type, nil) log.Info().Msg("Loading service settings") if err := loadingSettings(settingsmgr, flagParams); err != nil { log.Fatal().Msgf("Error loading settings - %v", err) @@ -523,6 +521,12 @@ func osctrlAPIService() { if err := serviceConfigMgr.Resolve(config.ServiceAPI, flagParams, settings.NoEnvironmentID); err != nil { log.Fatal().Msgf("Error resolving service config - %v", err) } + // Carves are built from the resolved config so DB overrides apply. + log.Info().Msg("Initialize carves") + filecarves, err = carves.CreateFileCarvesFromConfig(db.Conn, flagParams.Carver) + if err != nil { + log.Fatal().Msgf("Error initializing carver - %v", err) + } // Report whether this process can write its own config file. Only this // process can know — osctrl-tls runs elsewhere and reports its own. if err := serviceConfigMgr.ReportFile(config.ServiceAPI, flagParams.ConfigFilePath()); err != nil { diff --git a/cmd/tls/carver_test.go b/cmd/tls/carver_test.go new file mode 100644 index 000000000..0ca5355e9 --- /dev/null +++ b/cmd/tls/carver_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/jmpsec/osctrl/pkg/carves" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// Startup builds the carver from the resolved carver section. These pin the +// YAML -> ServiceParameters -> carver wiring that once silently dropped S3. +func TestCarverWiringFromYAML(t *testing.T) { + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none")) + tests := []struct { + name string + carver string + wantS3 bool + wantErr string + }{ + {name: "db", carver: "carver:\n type: db\n"}, + {name: "s3", carver: "carver:\n type: s3\n s3:\n bucket: carves\n region: us-east-1\n", wantS3: true}, + {name: "s3 incomplete", carver: "carver:\n type: s3\n s3:\n bucket: carves\n", wantErr: "carver.s3.region"}, + {name: "local", carver: "carver:\n type: local\n local:\n carvesDir: /tmp\n", wantErr: `carver type "local" is not supported`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg, err := loadYAMLConfiguration(writeTempTLSConfig(t, "service:\n auth: none\n"+tt.carver)) + if err == nil { + params := loadedYAMLToServiceParams(cfg, "tls.yml") + db, derr := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if derr != nil { + t.Fatal(derr) + } + var c *carves.Carves + c, err = carves.CreateFileCarvesFromConfig(db, params.Carver) + if err == nil { + if c.Carver != tt.name { + t.Fatalf("carver = %q", c.Carver) + } + if (c.S3 != nil) != tt.wantS3 { + t.Fatalf("S3 carver initialized = %v, want %v", c.S3 != nil, tt.wantS3) + } + } + } + if tt.wantErr == "" && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Fatalf("error = %v, want %q", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/tls/main.go b/cmd/tls/main.go index e6653548a..2c36f609b 100644 --- a/cmd/tls/main.go +++ b/cmd/tls/main.go @@ -90,7 +90,6 @@ var ( logSinksMgr *logsinks.LogSinksManager handlersTLS *handlers.HandlersTLS tagsmgr *tags.TagManager - carvers3 *carves.CarverS3 app *cli.Command flags []cli.Flag serviceConfiguration config.TLSConfiguration @@ -328,8 +327,6 @@ func osctrlService() { tagsmgr = tags.CreateTagManager(db.Conn) log.Info().Msg("Initialize queries") queriesmgr = queries.CreateQueries(db.Conn) - log.Info().Msg("Initialize carves") - filecarves = carves.CreateFileCarves(db.Conn, flagParams.Carver.Type, carvers3) log.Info().Msg("Loading service settings") if err := loadingSettings(settingsmgr, flagParams); err != nil { log.Fatal().Msgf("Error loading settings - %v", err) @@ -344,6 +341,12 @@ func osctrlService() { if err := serviceConfigMgr.Resolve(config.ServiceTLS, flagParams, settings.NoEnvironmentID); err != nil { log.Fatal().Msgf("Error resolving service config - %v", err) } + // Carves are built from the resolved config so DB overrides apply. + log.Info().Msg("Initialize carves") + filecarves, err = carves.CreateFileCarvesFromConfig(db.Conn, flagParams.Carver) + if err != nil { + log.Fatal().Msgf("Error initializing carver - %v", err) + } var queryDispatchTTL time.Duration if flagParams.Osquery != nil { queryDispatchTTL = flagParams.Osquery.QueryDispatchTTL diff --git a/deploy/config/api.yml b/deploy/config/api.yml index ae7f77f98..3d8c31139 100644 --- a/deploy/config/api.yml +++ b/deploy/config/api.yml @@ -513,7 +513,7 @@ logger: # Carver configuration to handle file carves from osquery nodes carver: - # Valid values: "none", "local", "db", "s3" + # Valid values: "db" (blocks stored in the database), "s3" type: db # S3 destination for carved files when type is s3. s3: @@ -525,10 +525,6 @@ carver: accessKey: "" # Optional static secret key. Prefer instance/task roles in production. secretAccessKey: "" - # Local filesystem destination when type is local. - local: - # Directory where carved files are written. - carvesDir: ./carved_files/ # Debug configuration for dumping incoming HTTP requests. Use only temporarily. debug: diff --git a/deploy/config/tls.yml b/deploy/config/tls.yml index 947a2a905..3faa4bccc 100644 --- a/deploy/config/tls.yml +++ b/deploy/config/tls.yml @@ -392,7 +392,7 @@ logger: # Carver configuration to handle file carves from osquery nodes carver: - # Valid values: "none", "local", "db", "s3" + # Valid values: "db" (blocks stored in the database), "s3" type: db # S3 destination for carved files when type is s3. s3: @@ -404,10 +404,6 @@ carver: accessKey: "" # Optional static secret key. Prefer instance/task roles in production. secretAccessKey: "" - # Local filesystem destination when type is local. - local: - # Directory where carved files are written. - carvesDir: ./carved_files/ # Debug configuration for dumping incoming HTTP requests. Use only temporarily. debug: diff --git a/frontend/src/features/service-config/ServiceConfigPage.tsx b/frontend/src/features/service-config/ServiceConfigPage.tsx index b7e742178..ac1fea860 100644 --- a/frontend/src/features/service-config/ServiceConfigPage.tsx +++ b/frontend/src/features/service-config/ServiceConfigPage.tsx @@ -124,7 +124,7 @@ const FIELD_HELP: Record = { 'logger.Types': 'Optional multi-destination logging/export. Empty uses `type`.', 'logger.AlwaysLog': 'Also persist status/on-demand query logs in DB even with external exporters.', 'logger.DB': 'Separate DB destination for log records when loggerDBSame is false. Fields match the top-level db section.', - 'carver.Type': 'Valid values: "local", "db", "s3". ("none" is documented in the sample YAML but rejected at startup.)', + 'carver.Type': 'Valid values: "db", "s3".', 'debug.ShowBody': 'Include request bodies. May contain secrets or node data.', 'debug.TargetHostIdentifier': "When non-empty, only dump requests from the osquery node whose UUID (or enroll host_identifier) matches this value (case-insensitive). Empty dumps every request when enableHttp is true. Useful to isolate one host's traffic on a busy server.", // --- YAML-annotated: metrics / osctrld @@ -219,7 +219,7 @@ const FIELD_HELP: Record = { AccessKey: 'AWS access key ID.', SecretAccessKey: 'AWS secret access key.', // Local carver - CarvesDir: 'Local directory to store carved files.', + CarvesDir: 'Unused: the "local" carver is not supported.', // Kinesis Stream: 'Kinesis stream name.', Endpoint: 'Custom Kinesis endpoint URL (for localstack or VPC endpoints).', diff --git a/pkg/carves/carver_test.go b/pkg/carves/carver_test.go new file mode 100644 index 000000000..20b520f5a --- /dev/null +++ b/pkg/carves/carver_test.go @@ -0,0 +1,238 @@ +package carves + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jmpsec/osctrl/pkg/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// fakeS3 is a minimal path-style S3 endpoint covering the calls the carver +// makes: PutObject, CreateMultipartUpload, UploadPartCopy, +// CompleteMultipartUpload and GetObject. +type fakeS3 struct { + mu sync.Mutex + objects map[string][]byte // "bucket/key" -> body + parts map[string]map[int][]byte // uploadID -> part number -> body +} + +func newFakeS3(t *testing.T) (*fakeS3, *s3.Client) { + t.Helper() + f := &fakeS3{objects: map[string][]byte{}, parts: map[string]map[int][]byte{}} + srv := httptest.NewServer(f) + t.Cleanup(srv.Close) + client := s3.NewFromConfig(aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("AKID", "SECRET", ""), + }, func(o *s3.Options) { + o.BaseEndpoint = aws.String(srv.URL) + o.UsePathStyle = true + }) + return f, client +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + obj := strings.TrimPrefix(r.URL.Path, "/") + q := r.URL.Query() + switch { + case r.Method == http.MethodPut && r.Header.Get("x-amz-copy-source") != "": + src, err := url.QueryUnescape(r.Header.Get("x-amz-copy-source")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + data, ok := f.objects[strings.TrimPrefix(src, "/")] + if !ok { + http.Error(w, "no such copy source "+src, http.StatusNotFound) + return + } + var part int + if _, err := fmt.Sscanf(q.Get("partNumber"), "%d", &part); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + f.parts[q.Get("uploadId")][part] = data + fmt.Fprintf(w, `"etag-%d"`, part) + case r.Method == http.MethodPut: + body, _ := io.ReadAll(r.Body) + f.objects[obj] = body + w.Header().Set("ETag", `"etag"`) + case r.Method == http.MethodPost && q.Has("uploads"): + id := fmt.Sprintf("upload-%d", len(f.parts)+1) + f.parts[id] = map[int][]byte{} + fmt.Fprintf(w, `%s`, id) + case r.Method == http.MethodPost && q.Has("uploadId"): + parts := f.parts[q.Get("uploadId")] + var buf bytes.Buffer + for i := 1; i <= len(parts); i++ { + buf.Write(parts[i]) + } + f.objects[obj] = buf.Bytes() + fmt.Fprintf(w, `%s`, obj) + case r.Method == http.MethodGet: + data, ok := f.objects[obj] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + _, _ = w.Write(data) + default: + http.Error(w, "unsupported "+r.Method+" "+r.URL.String(), http.StatusNotImplemented) + } +} + +func testCarvesDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + return db +} + +// carveBlocks creates a carve plus its blocks through the same calls the TLS +// carve handlers make, and returns the original file contents. +func carveBlocks(t *testing.T, c *Carves, chunks ...string) []byte { + t.Helper() + require.NoError(t, c.CreateCarve(CarvedFile{ + CarveID: "carve-1", RequestID: "req-1", SessionID: "sess-1", UUID: "NODE-1", + Environment: "dev", Path: "/etc/hosts", TotalBlocks: len(chunks), EnvironmentID: 1, + })) + var all []byte + for i, chunk := range chunks { + data := base64.StdEncoding.EncodeToString([]byte(chunk)) + block := c.InitateBlock("dev", "NODE-1", "req-1", "sess-1", data, i, 1) + require.NoError(t, c.CreateBlock(block, "NODE-1", data)) + all = append(all, chunk...) + } + return all +} + +func TestCarverDBStoresBlocksAndArchivesLocally(t *testing.T) { + c := CreateFileCarves(testCarvesDB(t), config.CarverDB, nil) + want := carveBlocks(t, c, "hello ", "world") + + blocks, err := c.GetBlocks("sess-1") + require.NoError(t, err) + require.Len(t, blocks, 2) + assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("hello ")), blocks[0].Data) + + dir := t.TempDir() + res, err := c.Archive("sess-1", dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, GenerateArchiveName(CarvedFile{UUID: "NODE-1", SessionID: "sess-1", Path: "/etc/hosts"})), res.File) + got, err := os.ReadFile(res.File) + require.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestCarverS3UploadsBlocksAndArchivesToBucket(t *testing.T) { + fake, client := newFakeS3(t) + s3c := &CarverS3{S3Config: config.S3Carver{Bucket: "carves"}, Client: client} + c := CreateFileCarves(testCarvesDB(t), config.CarverS3, s3c) + want := carveBlocks(t, c, "hello ", "world") + + // The DB keeps only a pointer; the decoded bytes live in the bucket. + blocks, err := c.GetBlocks("sess-1") + require.NoError(t, err) + require.Len(t, blocks, 2) + assert.Equal(t, "s3://carves/dev:NODE-1:sess-1:0", blocks[0].Data) + assert.Equal(t, []byte("hello "), fake.objects["carves/dev:NODE-1:sess-1:0"]) + + res, err := c.Archive("sess-1", "") + require.NoError(t, err) + assert.Equal(t, "s3://carves/dev:NODE-1:sess-1:/etc/hosts.tar", res.File) + assert.Equal(t, want, fake.objects["carves/dev:NODE-1:sess-1:/etc/hosts.tar"]) + + // The API serves the archive from the same bucket. + _, err = s3c.Download(CarvedFile{ArchivePath: res.File}) + require.NoError(t, err) +} + +func TestCarverS3WithoutClientErrorsInsteadOfPanicking(t *testing.T) { + c := CreateFileCarves(testCarvesDB(t), config.CarverS3, nil) + require.NoError(t, c.CreateCarve(CarvedFile{CarveID: "c", SessionID: "sess-1", Environment: "dev"})) + + block := c.InitateBlock("dev", "NODE-1", "req-1", "sess-1", "aGk=", 0, 1) + assert.ErrorContains(t, c.CreateBlock(block, "NODE-1", "aGk="), "s3 carver not initialized") + _, err := c.Archive("sess-1", "") + assert.ErrorContains(t, err, "s3 carver not initialized") +} + +func TestCreateFileCarvesFromConfig(t *testing.T) { + // Keep the AWS SDK from reading the developer's ~/.aws files. + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none")) + s3ok := config.S3Carver{Bucket: "carves", Region: "us-east-1"} + tests := []struct { + name string + cfg *config.YAMLConfigurationCarver + want string + wantS3 bool + wantErr string + }{ + {name: "nil section defaults to db", cfg: nil, want: config.CarverDB}, + {name: "db", cfg: &config.YAMLConfigurationCarver{Type: config.CarverDB}, want: config.CarverDB}, + {name: "s3 with role credentials", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3, S3: &s3ok}, want: config.CarverS3, wantS3: true}, + {name: "s3 with static keys", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3, S3: &config.S3Carver{Bucket: "b", Region: "r", AccessKey: "k", SecretAccessKey: "s"}}, want: config.CarverS3, wantS3: true}, + {name: "s3 without section", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3}, wantErr: "carver.s3.bucket"}, + {name: "s3 without bucket", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3, S3: &config.S3Carver{Region: "r"}}, wantErr: "carver.s3.bucket"}, + {name: "s3 without region", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3, S3: &config.S3Carver{Bucket: "b"}}, wantErr: "carver.s3.region"}, + {name: "s3 key without secret", cfg: &config.YAMLConfigurationCarver{Type: config.CarverS3, S3: &config.S3Carver{Bucket: "b", Region: "r", AccessKey: "k"}}, wantErr: "accessKey and secretAccessKey"}, + {name: "local is rejected", cfg: &config.YAMLConfigurationCarver{Type: config.CarverLocal, Local: &config.LocalCarver{CarvesDir: "/tmp"}}, wantErr: `carver type "local" is not supported`}, + {name: "unknown is rejected", cfg: &config.YAMLConfigurationCarver{Type: "ftp"}, wantErr: "invalid carver method: ftp"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := CreateFileCarvesFromConfig(testCarvesDB(t), tt.cfg) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, c.Carver) + assert.Equal(t, tt.wantS3, c.S3 != nil) + if tt.wantS3 { + assert.Equal(t, tt.cfg.S3.Bucket, c.S3.S3Config.Bucket) + } + }) + } +} + +func TestCreateCarverS3UsesStaticKeysOnlyWhenConfigured(t *testing.T) { + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none")) + t.Setenv("AWS_ACCESS_KEY_ID", "ENVKEY") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ENVSECRET") + + static, err := CreateCarverS3(config.S3Carver{Bucket: "b", Region: "r", AccessKey: "YAMLKEY", SecretAccessKey: "YAMLSECRET"}) + require.NoError(t, err) + creds, err := static.AWSConfig.Credentials.Retrieve(t.Context()) + require.NoError(t, err) + assert.Equal(t, "YAMLKEY", creds.AccessKeyID) + + // Without static keys the default chain (env, shared files, instance or + // task role) must be used rather than empty static credentials. + chain, err := CreateCarverS3(config.S3Carver{Bucket: "b", Region: "r"}) + require.NoError(t, err) + creds, err = chain.AWSConfig.Credentials.Retrieve(t.Context()) + require.NoError(t, err) + assert.Equal(t, "ENVKEY", creds.AccessKeyID) +} diff --git a/pkg/carves/carves.go b/pkg/carves/carves.go index 4642e140f..ad59732f7 100644 --- a/pkg/carves/carves.go +++ b/pkg/carves/carves.go @@ -81,6 +81,38 @@ func CreateFileCarves(backend *gorm.DB, carverType string, s3 *CarverS3) *Carves return c } +// CreateFileCarvesFromConfig validates the carver configuration, initializes +// the S3 carver when configured and creates the carves manager. A nil section +// means the default database carver. +func CreateFileCarvesFromConfig(backend *gorm.DB, cfg *config.YAMLConfigurationCarver) (*Carves, error) { + if cfg == nil { + return CreateFileCarves(backend, config.CarverDB, nil), nil + } + if err := config.ValidateCarverType(cfg.Type); err != nil { + return nil, err + } + var carverS3 *CarverS3 + if cfg.Type == config.CarverS3 { + s3Cfg := config.S3Carver{} + if cfg.S3 != nil { + s3Cfg = *cfg.S3 + } + switch { + case s3Cfg.Bucket == "": + return nil, fmt.Errorf("carver type s3 requires carver.s3.bucket (--carver-s3-bucket)") + case s3Cfg.Region == "": + return nil, fmt.Errorf("carver type s3 requires carver.s3.region (--carver-s3-region)") + case (s3Cfg.AccessKey == "") != (s3Cfg.SecretAccessKey == ""): + return nil, fmt.Errorf("carver type s3 requires both carver.s3.accessKey and secretAccessKey, or neither to use the default AWS credential chain") + } + var err error + if carverS3, err = CreateCarverS3(s3Cfg); err != nil { + return nil, fmt.Errorf("failed to initialize s3 carver - %w", err) + } + } + return CreateFileCarves(backend, cfg.Type, carverS3), nil +} + // CreateCarve to create a new carved file for a node func (c *Carves) CreateCarve(carve CarvedFile) error { return c.DB.Create(&carve).Error // can be nil or err @@ -132,7 +164,7 @@ func (c *Carves) GetCheckCarve(sessionid, requestid string) (CarvedFile, error) // InitateBlock to initiate a block based on the configured carver func (c *Carves) InitateBlock(env, uuid, requestid, sessionid, data string, blockid int, envid uint) CarvedBlock { var cData string - if c.Carver != config.CarverS3 { + if c.Carver != config.CarverS3 || c.S3 == nil { cData = data } else { cData = GenerateS3Data(c.S3.S3Config.Bucket, env, uuid, sessionid, blockid) @@ -357,11 +389,12 @@ func (c *Carves) Archive(sessionid, destPath string) (*CarveResult, error) { return nil, fmt.Errorf("error getting blocks - %w", err) } switch c.Carver { - case config.CarverLocal: - return c.ArchiveLocal(destPath, carve, blocks) case config.CarverDB: return c.ArchiveLocal(destPath, carve, blocks) case config.CarverS3: + if c.S3 == nil { + return nil, fmt.Errorf("s3 carver not initialized") + } return c.S3.Archive(carve, blocks) } return nil, fmt.Errorf("unknown carver - %s", c.Carver) diff --git a/pkg/carves/s3.go b/pkg/carves/s3.go index 934fa5a15..22aa2ed14 100644 --- a/pkg/carves/s3.go +++ b/pkg/carves/s3.go @@ -41,12 +41,14 @@ type CarverS3 struct { // CreateCarverS3 to initialize the carver func CreateCarverS3(s3Config osctrl_config.S3Carver) (*CarverS3, error) { - ctx := context.Background() - creds := credentials.NewStaticCredentialsProvider(s3Config.AccessKey, s3Config.SecretAccessKey, "") - cfg, err := config.LoadDefaultConfig( - ctx, - config.WithCredentialsProvider(creds), config.WithRegion(s3Config.Region), - ) + opts := []func(*config.LoadOptions) error{config.WithRegion(s3Config.Region)} + // Static keys are optional; without them the default chain (env, shared + // files, instance or task role) is used. + if s3Config.AccessKey != "" { + creds := credentials.NewStaticCredentialsProvider(s3Config.AccessKey, s3Config.SecretAccessKey, "") + opts = append(opts, config.WithCredentialsProvider(creds)) + } + cfg, err := config.LoadDefaultConfig(context.Background(), opts...) if err != nil { return nil, err } diff --git a/pkg/config/flags.go b/pkg/config/flags.go index f1cbab464..93c0c8500 100644 --- a/pkg/config/flags.go +++ b/pkg/config/flags.go @@ -730,7 +730,7 @@ func initCarverFlags(params *ServiceParameters) []cli.Flag { &cli.StringFlag{ Name: "carver-type", Value: CarverDB, - Usage: "Carver to be used to receive files extracted from nodes", + Usage: "Carver to be used to receive files extracted from nodes (db or s3)", Sources: cli.EnvVars("CARVER_TYPE"), Destination: ¶ms.Carver.Type, }, @@ -765,7 +765,7 @@ func initCarverFlags(params *ServiceParameters) []cli.Flag { &cli.StringFlag{ Name: "carver-local-dir", Value: defCarvedFolder, - Usage: "Local directory to store carved files", + Usage: "Unused: the local carver is not supported", Sources: cli.EnvVars("CARVER_LOCAL_DIR"), Destination: ¶ms.Carver.Local.CarvesDir, }, diff --git a/pkg/config/validation.go b/pkg/config/validation.go index 57fec0380..2a830afba 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -26,11 +26,17 @@ var validLogging = map[string]bool{ LoggingHTTP: true, } -// Valid values for carver in configuration -var validCarver = map[string]bool{ - CarverDB: true, - CarverLocal: true, - CarverS3: true, +// ValidateCarverType checks the carver type is one osctrl can run with. +// "local" was accepted for years but never implemented: blocks were never +// written to carvesDir, so it is rejected instead of failing every carve. +func ValidateCarverType(carverType string) error { + switch carverType { + case CarverDB, CarverS3: + return nil + case CarverLocal: + return fmt.Errorf("carver type %q is not supported; use %q (blocks stored in the database) or %q", CarverLocal, CarverDB, CarverS3) + } + return fmt.Errorf("invalid carver method: %s", carverType) } const maxRateLimitBurst = 100000 @@ -132,8 +138,8 @@ func ValidateTLSConfigValues(cfg TLSConfiguration) error { } } if cfg.Carver != nil { - if !validCarver[cfg.Carver.Type] { - return fmt.Errorf("invalid carver method: %s", cfg.Carver.Type) + if err := ValidateCarverType(cfg.Carver.Type); err != nil { + return err } } if cfg.RateLimits != nil { diff --git a/pkg/config/validation_test.go b/pkg/config/validation_test.go index f848330b4..449e50f3d 100644 --- a/pkg/config/validation_test.go +++ b/pkg/config/validation_test.go @@ -63,3 +63,13 @@ func TestValidateRateLimitsAcceptsDefaults(t *testing.T) { t.Fatalf("default rate limits should validate: %v", err) } } + +func TestValidateTLSConfigValuesRejectsLocalCarver(t *testing.T) { + err := ValidateTLSConfigValues(TLSConfiguration{ + Service: YAMLConfigurationService{Auth: AuthNone}, + Carver: &YAMLConfigurationCarver{Type: CarverLocal}, + }) + if err == nil || !strings.Contains(err.Error(), `carver type "local" is not supported`) { + t.Fatalf("expected local carver to be rejected with a clear message, got %v", err) + } +}