Restore S3 carver wiring and reject the unimplemented local carver

cmd/api/carver_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/jmpsec/osctrl/pkg/carves"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+// Startup builds the carver from the resolved carver section. These pin the
+// YAML -> ServiceParameters -> carver wiring that once silently dropped S3.
+func TestCarverWiringFromYAML(t *testing.T) {
+	t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none"))
+	t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none"))
+	tests := []struct {
+		name    string
+		carver  string
+		wantS3  bool
+		wantErr string
+	}{
+		{name: "db", carver: "carver:\n  type: db\n"},
+		{name: "s3", carver: "carver:\n  type: s3\n  s3:\n    bucket: carves\n    region: us-east-1\n", wantS3: true},
+		{name: "s3 incomplete", carver: "carver:\n  type: s3\n  s3:\n    bucket: carves\n", wantErr: "carver.s3.region"},
+		{name: "local", carver: "carver:\n  type: local\n  local:\n    carvesDir: /tmp\n", wantErr: `carver type "local" is not supported`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, err := loadYAMLConfiguration(writeTempConfig(t, "service:\n  auth: none\n"+tt.carver))
+			if err == nil {
+				params := loadedYAMLToServiceParams(cfg, "api.yml")
+				db, derr := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+				if derr != nil {
+					t.Fatal(derr)
+				}
+				var c *carves.Carves
+				c, err = carves.CreateFileCarvesFromConfig(db, params.Carver)
+				if err == nil {
+					if c.Carver != tt.name {
+						t.Fatalf("carver = %q", c.Carver)
+					}
+					if (c.S3 != nil) != tt.wantS3 {
+						t.Fatalf("S3 carver initialized = %v, want %v", c.S3 != nil, tt.wantS3)
+					}
+				}
+			}
+			if tt.wantErr == "" && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) {
+				t.Fatalf("error = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

cmd/api/main.go

@@ -487,8 +487,6 @@ func osctrlAPIService() {
 	}
 	consolemgr.SetLogReader(logReader)
 	fileexplorermgr.SetLogReader(logReader)
-	log.Info().Msg("Initialize carves")
-	filecarves = carves.CreateFileCarves(db.Conn, flagParams.Carver.Type, nil)
 	log.Info().Msg("Loading service settings")
 	if err := loadingSettings(settingsmgr, flagParams); err != nil {
 		log.Fatal().Msgf("Error loading settings - %v", err)
@@ -523,6 +521,12 @@ func osctrlAPIService() {
 	if err := serviceConfigMgr.Resolve(config.ServiceAPI, flagParams, settings.NoEnvironmentID); err != nil {
 		log.Fatal().Msgf("Error resolving service config - %v", err)
 	}
+	// Carves are built from the resolved config so DB overrides apply.
+	log.Info().Msg("Initialize carves")
+	filecarves, err = carves.CreateFileCarvesFromConfig(db.Conn, flagParams.Carver)
+	if err != nil {
+		log.Fatal().Msgf("Error initializing carver - %v", err)
+	}
 	// Report whether this process can write its own config file. Only this
 	// process can know — osctrl-tls runs elsewhere and reports its own.
 	if err := serviceConfigMgr.ReportFile(config.ServiceAPI, flagParams.ConfigFilePath()); err != nil {

cmd/tls/carver_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/jmpsec/osctrl/pkg/carves"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+// Startup builds the carver from the resolved carver section. These pin the
+// YAML -> ServiceParameters -> carver wiring that once silently dropped S3.
+func TestCarverWiringFromYAML(t *testing.T) {
+	t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "none"))
+	t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "none"))
+	tests := []struct {
+		name    string
+		carver  string
+		wantS3  bool
+		wantErr string
+	}{
+		{name: "db", carver: "carver:\n  type: db\n"},
+		{name: "s3", carver: "carver:\n  type: s3\n  s3:\n    bucket: carves\n    region: us-east-1\n", wantS3: true},
+		{name: "s3 incomplete", carver: "carver:\n  type: s3\n  s3:\n    bucket: carves\n", wantErr: "carver.s3.region"},
+		{name: "local", carver: "carver:\n  type: local\n  local:\n    carvesDir: /tmp\n", wantErr: `carver type "local" is not supported`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, err := loadYAMLConfiguration(writeTempTLSConfig(t, "service:\n  auth: none\n"+tt.carver))
+			if err == nil {
+				params := loadedYAMLToServiceParams(cfg, "tls.yml")
+				db, derr := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+				if derr != nil {
+					t.Fatal(derr)
+				}
+				var c *carves.Carves
+				c, err = carves.CreateFileCarvesFromConfig(db, params.Carver)
+				if err == nil {
+					if c.Carver != tt.name {
+						t.Fatalf("carver = %q", c.Carver)
+					}
+					if (c.S3 != nil) != tt.wantS3 {
+						t.Fatalf("S3 carver initialized = %v, want %v", c.S3 != nil, tt.wantS3)
+					}
+				}
+			}
+			if tt.wantErr == "" && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) {
+				t.Fatalf("error = %v, want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

cmd/tls/main.go

@@ -90,7 +90,6 @@ var (
 	logSinksMgr          *logsinks.LogSinksManager
 	handlersTLS          *handlers.HandlersTLS
 	tagsmgr              *tags.TagManager
-	carvers3             *carves.CarverS3
 	app                  *cli.Command
 	flags                []cli.Flag
 	serviceConfiguration config.TLSConfiguration
@@ -328,8 +327,6 @@ func osctrlService() {
 	tagsmgr = tags.CreateTagManager(db.Conn)
 	log.Info().Msg("Initialize queries")
 	queriesmgr = queries.CreateQueries(db.Conn)
-	log.Info().Msg("Initialize carves")
-	filecarves = carves.CreateFileCarves(db.Conn, flagParams.Carver.Type, carvers3)
 	log.Info().Msg("Loading service settings")
 	if err := loadingSettings(settingsmgr, flagParams); err != nil {
 		log.Fatal().Msgf("Error loading settings - %v", err)
@@ -344,6 +341,12 @@ func osctrlService() {
 	if err := serviceConfigMgr.Resolve(config.ServiceTLS, flagParams, settings.NoEnvironmentID); err != nil {
 		log.Fatal().Msgf("Error resolving service config - %v", err)
 	}
+	// Carves are built from the resolved config so DB overrides apply.
+	log.Info().Msg("Initialize carves")
+	filecarves, err = carves.CreateFileCarvesFromConfig(db.Conn, flagParams.Carver)
+	if err != nil {
+		log.Fatal().Msgf("Error initializing carver - %v", err)
+	}
 	var queryDispatchTTL time.Duration
 	if flagParams.Osquery != nil {
 		queryDispatchTTL = flagParams.Osquery.QueryDispatchTTL

deploy/config/api.yml

@@ -513,7 +513,7 @@ logger:
 
 # Carver configuration to handle file carves from osquery nodes
 carver:
-  # Valid values: "none", "local", "db", "s3"
+  # Valid values: "db" (blocks stored in the database), "s3"
   type: db
   # S3 destination for carved files when type is s3.
   s3:
@@ -525,10 +525,6 @@ carver:
     accessKey: ""
     # Optional static secret key. Prefer instance/task roles in production.
     secretAccessKey: ""
-  # Local filesystem destination when type is local.
-  local:
-    # Directory where carved files are written.
-    carvesDir: ./carved_files/
 
 # Debug configuration for dumping incoming HTTP requests. Use only temporarily.
 debug:

deploy/config/tls.yml

@@ -392,7 +392,7 @@ logger:
 
 # Carver configuration to handle file carves from osquery nodes
 carver:
-  # Valid values: "none", "local", "db", "s3"
+  # Valid values: "db" (blocks stored in the database), "s3"
   type: db
   # S3 destination for carved files when type is s3.
   s3:
@@ -404,10 +404,6 @@ carver:
     accessKey: ""
     # Optional static secret key. Prefer instance/task roles in production.
     secretAccessKey: ""
-  # Local filesystem destination when type is local.
-  local:
-    # Directory where carved files are written.
-    carvesDir: ./carved_files/
 
 # Debug configuration for dumping incoming HTTP requests. Use only temporarily.
 debug:

frontend/src/features/service-config/ServiceConfigPage.tsx

@@ -124,7 +124,7 @@ const FIELD_HELP: Record<string, string> = {
   'logger.Types': 'Optional multi-destination logging/export. Empty uses `type`.',
   'logger.AlwaysLog': 'Also persist status/on-demand query logs in DB even with external exporters.',
   'logger.DB': 'Separate DB destination for log records when loggerDBSame is false. Fields match the top-level db section.',
-  'carver.Type': 'Valid values: "local", "db", "s3". ("none" is documented in the sample YAML but rejected at startup.)',
+  'carver.Type': 'Valid values: "db", "s3".',
   'debug.ShowBody': 'Include request bodies. May contain secrets or node data.',
   'debug.TargetHostIdentifier': "When non-empty, only dump requests from the osquery node whose UUID (or enroll host_identifier) matches this value (case-insensitive). Empty dumps every request when enableHttp is true. Useful to isolate one host's traffic on a busy server.",
   // --- YAML-annotated: metrics / osctrld
@@ -219,7 +219,7 @@ const FIELD_HELP: Record<string, string> = {
   AccessKey: 'AWS access key ID.',
   SecretAccessKey: 'AWS secret access key.',
   // Local carver
-  CarvesDir: 'Local directory to store carved files.',
+  CarvesDir: 'Unused: the "local" carver is not supported.',
   // Kinesis
   Stream: 'Kinesis stream name.',
   Endpoint: 'Custom Kinesis endpoint URL (for localstack or VPC endpoints).',