From 38e2614f18138fc4138a0fe35256f2fceb2dfa34 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:11:56 +0200 Subject: [PATCH] Per-node flags overrides: host identity and a rollback guard for osquery restarts --- README.md | 10 + cmd/osctrld/actions.go | 19 +- cmd/osctrld/actions_helpers.go | 4 +- cmd/osctrld/actions_helpers_test.go | 2 +- cmd/osctrld/flagsguard.go | 262 +++++++++++++ cmd/osctrld/flagsguard_test.go | 569 ++++++++++++++++++++++++++++ cmd/osctrld/hostid.go | 74 ++++ cmd/osctrld/hostid_test.go | 108 ++++++ cmd/osctrld/service.go | 18 +- 9 files changed, 1057 insertions(+), 9 deletions(-) create mode 100644 cmd/osctrld/flagsguard.go create mode 100644 cmd/osctrld/flagsguard_test.go create mode 100644 cmd/osctrld/hostid.go create mode 100644 cmd/osctrld/hostid_test.go diff --git a/README.md b/README.md index 5ff6fb9..1ed017d 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,16 @@ In daemon mode, osctrld will: 3. Restart osquery through the OS service manager when changes are detected. 4. Shut down gracefully on `SIGINT` or `SIGTERM`. +Flag requests carry the host UUID (read with `osqueryd -S`), so osctrl can serve a +per-node flagfile, plus the outcome of the last flags change. osquery exits on an +unknown flag or bad value, so after a restart caused by new flags osctrld waits up to +60 seconds for `osqueryd` to stay running. If it does not, osctrld writes the +environment flags back, restarts osquery once, reports the flags as `rejected` and +does not apply that same content again until osctrl serves something different. A +check interrupted by a stop, crash or reboot is finished on the next sync. This state, +along with the last known host UUID (used when the lookup fails), is kept next to the +flagfile, in `.osctrld-state.json`. + ## Native enrollment `osctrld install` does everything the osctrl quick-add script does, without a shell: diff --git a/cmd/osctrld/actions.go b/cmd/osctrld/actions.go index 1607130..4ca1b89 100644 --- a/cmd/osctrld/actions.go +++ b/cmd/osctrld/actions.go @@ -5,6 +5,7 @@ import ( "fmt" "runtime" "strings" + "time" "github.com/rs/zerolog/log" "github.com/shirou/gopsutil/v3/process" @@ -35,9 +36,19 @@ var ( // FlagsRequest to retrieve flags type FlagsRequest struct { - OsctrlSecret string `json:"secret"` - OsquerySecretFile string `json:"secretFile"` - OsqueryCertFile string `json:"certFile"` + OsctrlSecret string `json:"secret"` + OsquerySecretFile string `json:"secretFile"` + OsqueryCertFile string `json:"certFile"` + UUID string `json:"uuid,omitempty"` + Apply *ApplyStatus `json:"apply,omitempty"` +} + +// ApplyStatus reports the outcome of the last flags apply; Result is "ok" or "rejected" +type ApplyStatus struct { + Result string `json:"result"` + SHA256 string `json:"sha256"` + Error string `json:"error,omitempty"` + At time.Time `json:"at"` } // CertRequest to retrieve certificate @@ -87,7 +98,7 @@ func enrollNode(ctx context.Context, cmd *cli.Command) error { func getFlags(ctx context.Context, cmd *cli.Command) (bool, error) { log.Debug().Str("url", osctrlURLs.Flags).Msg("getting flags") flags, err := spin("retrieving flags from osctrl", func() (string, error) { - return retrieveFlags(appConfig.OsctrlSecret, appConfig.OsquerySecretFile, appConfig.OsqueryCertFile) + return retrieveFlags(appConfig.OsctrlSecret, appConfig.OsquerySecretFile, appConfig.OsqueryCertFile, "", nil) }) if err != nil { return false, fmt.Errorf("error retrieving flags - %v", err) diff --git a/cmd/osctrld/actions_helpers.go b/cmd/osctrld/actions_helpers.go index bf23a60..fb46229 100644 --- a/cmd/osctrld/actions_helpers.go +++ b/cmd/osctrld/actions_helpers.go @@ -14,11 +14,13 @@ import ( ) // Helper function to retrieve flags -func retrieveFlags(osctrlSecret, osquerySecretFile, osqueryCertFile string) (string, error) { +func retrieveFlags(osctrlSecret, osquerySecretFile, osqueryCertFile, uuid string, apply *ApplyStatus) (string, error) { flagsData := FlagsRequest{ OsctrlSecret: osctrlSecret, OsquerySecretFile: osquerySecretFile, OsqueryCertFile: osqueryCertFile, + UUID: uuid, + Apply: apply, } jsonReq, err := json.Marshal(flagsData) if err != nil { diff --git a/cmd/osctrld/actions_helpers_test.go b/cmd/osctrld/actions_helpers_test.go index 59389fa..adc57d8 100644 --- a/cmd/osctrld/actions_helpers_test.go +++ b/cmd/osctrld/actions_helpers_test.go @@ -200,7 +200,7 @@ func TestRetrieveFlags(t *testing.T) { osctrlURLs.Flags = server.URL + "/env/osctrld-flags" appConfig.Insecure = false - flags, err := retrieveFlags("test-secret", "/tmp/secret", "/tmp/cert") + flags, err := retrieveFlags("test-secret", "/tmp/secret", "/tmp/cert", "", nil) assert.NoError(t, err) assert.Contains(t, flags, "tls_hostname") } diff --git a/cmd/osctrld/flagsguard.go b/cmd/osctrld/flagsguard.go new file mode 100644 index 0000000..8baf2bc --- /dev/null +++ b/cmd/osctrld/flagsguard.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/rs/zerolog/log" + "github.com/shirou/gopsutil/v3/process" +) + +const ( + applyOK = "ok" + applyRejected = "rejected" + // runningChecks is how many consecutive polls must see osqueryd before new + // flags count as accepted: a service manager respawning a crashing osqueryd + // makes it visible for a moment, which must not read as success + runningChecks = 3 +) + +// rejectedError is reported to osctrl when osqueryd did not survive new flags +func rejectedError() string { + return fmt.Sprintf("osqueryd was not running %gs after restart", restartCheckTimeout.Seconds()) +} + +var ( + // osquerydRunning is a seam so the restart guard can be tested without osqueryd + osquerydRunning = isOsquerydRunning + // processNames is a seam over the process list + processNames = listProcessNames + restartCheckTimeout = 60 * time.Second + restartCheckInterval = 2 * time.Second +) + +// flagsState is what osctrld remembers between syncs about applying flags +type flagsState struct { + Last *ApplyStatus `json:"last,omitempty"` + RejectedSHA256 string `json:"rejected_sha256,omitempty"` + // Pending is the SHA-256 of flags written but not verified yet + Pending string `json:"pending,omitempty"` + // HostUUID is the last good host UUID, used when the lookup fails + HostUUID string `json:"host_uuid,omitempty"` +} + +// flagsStatePath is where flagsState lives, next to the flagfile it describes +func flagsStatePath() string { + return appConfig.OsqueryFlagFile + ".osctrld-state.json" +} + +// loadFlagsState reads the state, falling back to the zero value on any problem: +// worst case a rejected override is tried (and rolled back) once more +func loadFlagsState(path string) flagsState { + var s flagsState + b, err := os.ReadFile(path) + if err != nil { + log.Debug().Err(err).Str("path", path).Msg("no flags state") + return flagsState{} + } + if err := json.Unmarshal(b, &s); err != nil { + log.Debug().Err(err).Str("path", path).Msg("ignoring corrupt flags state") + return flagsState{} + } + return s +} + +func saveFlagsState(path string, s flagsState) error { + b, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("error encoding flags state - %v", err) + } + if err := os.WriteFile(path, b, 0600); err != nil { + return fmt.Errorf("error writing flags state to %s - %v", path, err) + } + // WriteFile only applies the mode when it creates the file + if err := os.Chmod(path, 0600); err != nil { + return fmt.Errorf("error setting mode on %s - %v", path, err) + } + return nil +} + +// sha256Hex hashes flags exactly as they are written to the flagfile +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func listProcessNames() ([]string, error) { + ps, err := process.Processes() + if err != nil { + return nil, err + } + names := make([]string, 0, len(ps)) + for _, p := range ps { + if n, err := p.Name(); err == nil { + names = append(names, n) + } + } + return names, nil +} + +// isOsquerydRunning matches the process name, which works on every OS +func isOsquerydRunning() (bool, error) { + names, err := processNames() + if err != nil { + return false, fmt.Errorf("error getting processes - %v", err) + } + for _, n := range names { + if strings.EqualFold(n, "osqueryd") || strings.EqualFold(n, "osqueryd.exe") { + return true, nil + } + } + return false, nil +} + +func fetchFlags(uuid string, apply *ApplyStatus) (string, error) { + return retrieveFlags(appConfig.OsctrlSecret, appConfig.OsquerySecretFile, appConfig.OsqueryCertFile, uuid, apply) +} + +// syncFlags retrieves the flags for this host, sending its identity and the last +// apply status, and writes them unless they were already rejected. Changed flags +// are marked pending in the state before they reach the disk, so a check that +// never completes is resumed by resumeFlagsCheck. It returns whether the flagfile +// changed, the flags written and the uuid that was sent. +func syncFlags(ctx context.Context) (changed bool, attempted string, uuid string, err error) { + path := flagsStatePath() + st := loadFlagsState(path) + dirty := false + if uuid, err = hostUUIDReader(); err != nil { + // A cached uuid keeps an override applied through a flaky lookup instead + // of flapping between override and environment flags every sync + uuid = st.HostUUID + log.Warn().Err(err).Str("cached_uuid", uuid).Msg("host uuid lookup failed, using the last known one") + } else if uuid != st.HostUUID { + st.HostUUID = uuid + dirty = true + } + flags, err := fetchFlags(uuid, st.Last) + if err != nil { + return false, "", uuid, fmt.Errorf("error retrieving flags - %v", err) + } + if st.RejectedSHA256 != "" && sha256Hex(flags) == st.RejectedSHA256 { + log.Warn().Str("sha256", st.RejectedSHA256).Msg("served flags were rejected before, using environment flags") + if flags, err = fetchFlags("", st.Last); err != nil { + return false, "", uuid, fmt.Errorf("error retrieving environment flags - %v", err) + } + } + if !checkFileExist(appConfig.OsqueryFlagFile) || !checkFileContent(appConfig.OsqueryFlagFile, flags) { + st.Pending = sha256Hex(flags) + dirty = true + } + if dirty { + // Unverifiable flags are never written: no state, no write + if err := saveFlagsState(path, st); err != nil { + return false, "", uuid, err + } + } + changed, err = writeContentExists(appConfig.OsqueryFlagFile, flags, "flags", true, 0700) + if err != nil { + return false, "", uuid, err + } + return changed, flags, uuid, nil +} + +// resumeFlagsCheck finishes the check of flags written by an earlier sync whose +// verification never completed (osctrld stopped or crashed, host rebooted). It +// does not restart osquery first. A pending hash that no longer matches the +// flagfile (edited since) is dropped. +func resumeFlagsCheck(ctx context.Context) error { + path := flagsStatePath() + st := loadFlagsState(path) + if st.Pending == "" { + return nil + } + b, err := os.ReadFile(appConfig.OsqueryFlagFile) + current := strings.TrimSpace(string(b)) + if err != nil || sha256Hex(current) != st.Pending { + log.Warn().Str("sha256", st.Pending).Msg("flagfile changed since it was written, dropping its pending check") + st.Pending = "" + return saveFlagsState(path, st) + } + log.Info().Str("sha256", st.Pending).Msg("resuming check of unverified flags") + return verifyFlagsAfterRestart(ctx, current, st.HostUUID) +} + +// waitForOsqueryd polls until osqueryd has been seen on runningChecks +// consecutive polls, or restartCheckTimeout passes +func waitForOsqueryd(ctx context.Context) (bool, error) { + deadline := time.After(restartCheckTimeout) + seen := 0 + for { + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-deadline: + return false, nil + case <-time.After(restartCheckInterval): + } + running, err := osquerydRunning() + if err != nil { + log.Debug().Err(err).Msg("error checking osqueryd") + } + if !running { + seen = 0 + continue + } + if seen++; seen >= runningChecks { + return true, nil + } + } +} + +// verifyFlagsAfterRestart checks osqueryd survived the attempted flags and +// clears their pending mark. If it did not, it writes the environment flags back, restarts osquery once and +// remembers the attempted content so it is not applied again. The outcome is +// reported to osctrl on the next flags request. +func verifyFlagsAfterRestart(ctx context.Context, attempted string, uuid string) error { + path := flagsStatePath() + sum := sha256Hex(attempted) + running, err := waitForOsqueryd(ctx) + if err != nil { + // Pending stays set, the next sync resumes the check + return err + } + st := loadFlagsState(path) + // Every outcome below completes the check. The rollback's own restart is + // deliberately not verified, so no new Pending is set for the env flags. + st.Pending = "" + if running { + log.Info().Str("sha256", sum).Msg("osqueryd is running with the new flags") + st.Last = &ApplyStatus{Result: applyOK, SHA256: sum, At: time.Now().UTC()} + return saveFlagsState(path, st) + } + log.Error().Str("sha256", sum).Str("uuid", uuid).Msg("osqueryd is not running after flags change, rolling back") + st.Last = &ApplyStatus{Result: applyRejected, SHA256: sum, Error: rejectedError(), At: time.Now().UTC()} + env, fetchErr := fetchFlags("", nil) + if fetchErr != nil { + // Remember the rejection anyway: the next sync then writes the environment flags + st.RejectedSHA256 = sum + if err := saveFlagsState(path, st); err != nil { + log.Error().Err(err).Msg("failed to save flags state") + } + return fmt.Errorf("error retrieving environment flags for rollback - %v", fetchErr) + } + if env == attempted { + // Not an override that failed, nothing to roll back to + log.Error().Msg("environment flags do not keep osqueryd running") + return saveFlagsState(path, st) + } + st.RejectedSHA256 = sum + if err := saveFlagsState(path, st); err != nil { + return err + } + if _, err := writeContentExists(appConfig.OsqueryFlagFile, env, "flags", true, 0700); err != nil { + return err + } + return restartOsquery() +} diff --git a/cmd/osctrld/flagsguard_test.go b/cmd/osctrld/flagsguard_test.go new file mode 100644 index 0000000..ea8f00a --- /dev/null +++ b/cmd/osctrld/flagsguard_test.go @@ -0,0 +1,569 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testEnvFlags = "--tls_hostname=osctrl.example.com" + testOverrideFlags = "--tls_hostname=osctrl.example.com\n--bad_flag=1" +) + +// flagsServer serves override when the request carries a uuid and env otherwise, +// recording every request it receives +type flagsServer struct { + mu sync.Mutex + override string + env string + requests []FlagsRequest +} + +func (s *flagsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var req FlagsRequest + _ = json.NewDecoder(r.Body).Decode(&req) + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, req) + if req.UUID != "" { + _, _ = w.Write([]byte(s.override)) + return + } + _, _ = w.Write([]byte(s.env)) +} + +func (s *flagsServer) recorded() []FlagsRequest { + s.mu.Lock() + defer s.mu.Unlock() + return append([]FlagsRequest(nil), s.requests...) +} + +// setupGuard wires a fake osctrl, host uuid, process check and service manager +func setupGuard(t *testing.T, override string, running func() (bool, error)) (*flagsServer, *[][]string) { + t.Helper() + fs := &flagsServer{override: override, env: testEnvFlags} + server := httptest.NewServer(fs) + t.Cleanup(server.Close) + + prevConfig, prevURLs := appConfig, osctrlURLs + prevUUID, prevRunning := hostUUIDReader, osquerydRunning + prevTimeout, prevInterval := restartCheckTimeout, restartCheckInterval + t.Cleanup(func() { + appConfig, osctrlURLs = prevConfig, prevURLs + hostUUIDReader, osquerydRunning = prevUUID, prevRunning + restartCheckTimeout, restartCheckInterval = prevTimeout, prevInterval + }) + + dir := t.TempDir() + appConfig = Configuration{ + OsctrlSecret: "test-secret", + OsquerySecretFile: filepath.Join(dir, "osquery.secret"), + OsqueryFlagFile: filepath.Join(dir, "osquery.flags"), + OsqueryCertFile: filepath.Join(dir, "osctrl.crt"), + Force: true, + } + osctrlURLs = genURLs(server.URL, "env", false) + osctrlURLs.Flags = server.URL + "/flags" + hostUUIDReader = func() (string, error) { return testHostUUID, nil } + if running == nil { + running = func() (bool, error) { return true, nil } + } + osquerydRunning = running + restartCheckTimeout = 50 * time.Millisecond + restartCheckInterval = 5 * time.Millisecond + + recorded := fakeExec(t) + fakeLookPath(t, "launchctl", "systemctl") + return fs, recorded +} + +func readFlagFile(t *testing.T) string { + t.Helper() + b, err := os.ReadFile(appConfig.OsqueryFlagFile) + require.NoError(t, err) + return string(b) +} + +func neverRunning() (bool, error) { return false, nil } + +func TestSyncFlagsSendsUUIDAndLastStatus(t *testing.T) { + fs, _ := setupGuard(t, testOverrideFlags, nil) + at := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{ + Last: &ApplyStatus{Result: "ok", SHA256: sha256Hex(testEnvFlags), At: at}, + })) + + changed, attempted, uuid, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, testOverrideFlags, attempted) + assert.Equal(t, testHostUUID, uuid) + + reqs := fs.recorded() + require.Len(t, reqs, 1) + assert.Equal(t, testHostUUID, reqs[0].UUID) + require.NotNil(t, reqs[0].Apply) + assert.Equal(t, "ok", reqs[0].Apply.Result) + assert.Equal(t, sha256Hex(testEnvFlags), reqs[0].Apply.SHA256) + assert.Equal(t, testOverrideFlags, readFlagFile(t)) +} + +func TestSyncFlagsWithoutUUIDWhenLookupFails(t *testing.T) { + fs, _ := setupGuard(t, testOverrideFlags, nil) + hostUUIDReader = func() (string, error) { return "", errors.New("osqueryd missing") } + + changed, _, uuid, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.True(t, changed) + assert.Empty(t, uuid) + + reqs := fs.recorded() + require.Len(t, reqs, 1) + assert.Empty(t, reqs[0].UUID) + assert.Nil(t, reqs[0].Apply, "no state file yet, so no status to report") + assert.Equal(t, testEnvFlags, readFlagFile(t)) +} + +func TestSyncFlagsUnchanged(t *testing.T) { + setupGuard(t, testOverrideFlags, nil) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags+"\n"), 0700)) + + changed, _, _, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.False(t, changed) +} + +func TestVerifyAfterRestartOK(t *testing.T) { + fs, recorded := setupGuard(t, testOverrideFlags, nil) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testOverrideFlags, testHostUUID)) + + st := loadFlagsState(flagsStatePath()) + require.NotNil(t, st.Last) + assert.Equal(t, "ok", st.Last.Result) + assert.Equal(t, sha256Hex(testOverrideFlags), st.Last.SHA256) + assert.Empty(t, st.Last.Error) + assert.Empty(t, st.RejectedSHA256) + assert.Empty(t, fs.recorded(), "no rollback request") + assert.Empty(t, *recorded, "no rollback restart") + assert.Equal(t, testOverrideFlags, readFlagFile(t)) +} + +func TestVerifyAfterRestartKeepsRejectedHashOnOK(t *testing.T) { + setupGuard(t, testOverrideFlags, nil) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{RejectedSHA256: "old"})) + + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testEnvFlags, testHostUUID)) + assert.Equal(t, "old", loadFlagsState(flagsStatePath()).RejectedSHA256) +} + +func TestVerifyAfterRestartRollsBack(t *testing.T) { + fs, recorded := setupGuard(t, testOverrideFlags, neverRunning) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testOverrideFlags, testHostUUID)) + + assert.Equal(t, testEnvFlags, readFlagFile(t), "environment flags written back") + assert.Len(t, *recorded, 1, "one rollback restart") + reqs := fs.recorded() + require.Len(t, reqs, 1) + assert.Empty(t, reqs[0].UUID, "rollback asks for the environment flags") + assert.Nil(t, reqs[0].Apply) + + st := loadFlagsState(flagsStatePath()) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) + assert.Equal(t, sha256Hex(testOverrideFlags), st.Last.SHA256) + assert.Equal(t, rejectedErrorFor(restartCheckTimeout), st.Last.Error) +} + +// A service manager respawning a crashing osqueryd makes it visible briefly; +// that must not count as a successful apply +func TestVerifyAfterRestartFlappingIsRejected(t *testing.T) { + var mu sync.Mutex + n := 0 + setupGuard(t, testOverrideFlags, func() (bool, error) { + mu.Lock() + defer mu.Unlock() + n++ + return n%2 == 0, nil + }) + + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testOverrideFlags, testHostUUID)) + st := loadFlagsState(flagsStatePath()) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256) +} + +func TestVerifyAfterRestartRejectsWhenRollbackFetchFails(t *testing.T) { + setupGuard(t, testOverrideFlags, neverRunning) + osctrlURLs.Flags = "http://127.0.0.1:1/flags" + + err := verifyFlagsAfterRestart(context.Background(), testOverrideFlags, testHostUUID) + assert.Error(t, err) + st := loadFlagsState(flagsStatePath()) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256, + "rejection is remembered so the next sync writes the environment flags") + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) +} + +func TestVerifyAfterRestartStopsOnCancel(t *testing.T) { + setupGuard(t, testOverrideFlags, neverRunning) + restartCheckTimeout = time.Minute + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := verifyFlagsAfterRestart(ctx, testOverrideFlags, testHostUUID) + assert.ErrorIs(t, err, context.Canceled) + assert.Nil(t, loadFlagsState(flagsStatePath()).Last) +} + +func TestSyncFlagsSkipsRejectedContent(t *testing.T) { + fs, _ := setupGuard(t, testOverrideFlags, nil) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{RejectedSHA256: sha256Hex(testOverrideFlags)})) + + changed, attempted, _, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, testEnvFlags, attempted) + assert.Equal(t, testEnvFlags, readFlagFile(t)) + + reqs := fs.recorded() + require.Len(t, reqs, 2) + assert.Equal(t, testHostUUID, reqs[0].UUID) + assert.Empty(t, reqs[1].UUID) + + // Already holding the environment flags: nothing to do + changed, _, _, err = syncFlags(context.Background()) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, testEnvFlags, readFlagFile(t)) +} + +func TestSyncFlagsRetriesWhenContentChanges(t *testing.T) { + newOverride := "--tls_hostname=osctrl.example.com\n--host_identifier=uuid" + fs, _ := setupGuard(t, newOverride, nil) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{RejectedSHA256: sha256Hex(testOverrideFlags)})) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testEnvFlags), 0700)) + + changed, attempted, _, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, newOverride, attempted) + assert.Equal(t, newOverride, readFlagFile(t)) + assert.Len(t, fs.recorded(), 1) +} + +func TestVerifyAfterRestartEnvFlagsFailToo(t *testing.T) { + _, recorded := setupGuard(t, testEnvFlags, neverRunning) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testEnvFlags), 0700)) + + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testEnvFlags, testHostUUID)) + + assert.Empty(t, *recorded, "nothing to roll back to, no restart") + st := loadFlagsState(flagsStatePath()) + assert.Empty(t, st.RejectedSHA256) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) + assert.Equal(t, sha256Hex(testEnvFlags), st.Last.SHA256) + assert.Equal(t, rejectedErrorFor(restartCheckTimeout), st.Last.Error) +} + +func TestLoadFlagsStateCorrupt(t *testing.T) { + dir := t.TempDir() + corrupt := filepath.Join(dir, "corrupt.json") + require.NoError(t, os.WriteFile(corrupt, []byte("{not json"), 0600)) + + for name, path := range map[string]string{ + "corrupt": corrupt, + "missing": filepath.Join(dir, "missing.json"), + "directory": dir, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, flagsState{}, loadFlagsState(path)) + }) + } +} + +func TestSaveFlagsStateRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + at := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + want := flagsState{ + Last: &ApplyStatus{Result: "rejected", SHA256: "ab", Error: "e", At: at}, + RejectedSHA256: "ab", + } + require.NoError(t, saveFlagsState(path, want)) + got := loadFlagsState(path) + assert.Equal(t, want.RejectedSHA256, got.RejectedSHA256) + require.NotNil(t, got.Last) + assert.True(t, at.Equal(got.Last.At)) + assert.Equal(t, "rejected", got.Last.Result) +} + +func TestSaveFlagsStateMode(t *testing.T) { + if runtime.GOOS == WindowsOS { + t.Skip("no unix modes on windows") + } + path := filepath.Join(t.TempDir(), "state.json") + // A pre-existing, too-open file is tightened too + require.NoError(t, os.WriteFile(path, []byte("{}"), 0644)) + require.NoError(t, saveFlagsState(path, flagsState{RejectedSHA256: "ab"})) + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestFlagsStatePath(t *testing.T) { + prev := appConfig + t.Cleanup(func() { appConfig = prev }) + appConfig.OsqueryFlagFile = "/etc/osquery/osquery.flags" + assert.Equal(t, "/etc/osquery/osquery.flags.osctrld-state.json", flagsStatePath()) +} + +func TestIsOsquerydRunningMatchesName(t *testing.T) { + prev := processNames + t.Cleanup(func() { processNames = prev }) + + tests := []struct { + name string + want bool + }{ + {"osqueryd", true}, + {"osqueryd.exe", true}, + {"OSQUERYD.EXE", true}, + {"osqueryi", false}, + {"not-osqueryd-helper", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + processNames = func() ([]string, error) { return []string{"launchd", tt.name}, nil } + got, err := isOsquerydRunning() + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + processNames = func() ([]string, error) { return nil, errors.New("boom") } + _, err := isOsquerydRunning() + assert.Error(t, err) +} + +// syncOnceGuard points cert and extensions at a server that 404s, so only flags change +func syncOnceGuard(t *testing.T, running func() (bool, error)) *[][]string { + t.Helper() + _, recorded := setupGuard(t, testOverrideFlags, running) + notFound := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(notFound.Close) + osctrlURLs.Cert = notFound.URL + osctrlURLs.Extensions = notFound.URL + return recorded +} + +func TestSyncOnceVerifiesChangedFlags(t *testing.T) { + recorded := syncOnceGuard(t, neverRunning) + + syncOnce(context.Background(), newTestCLICommand()) + + assert.Len(t, *recorded, 2, "restart for the new flags, then the rollback restart") + assert.Equal(t, testEnvFlags, readFlagFile(t)) + st := loadFlagsState(flagsStatePath()) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256) + + // The next cycle keeps the environment flags: no restart, no new check + *recorded = nil + syncOnce(context.Background(), newTestCLICommand()) + assert.Empty(t, *recorded) + assert.Equal(t, testEnvFlags, readFlagFile(t)) +} + +func TestSyncOnceSkipsVerifyWhenFlagsUnchanged(t *testing.T) { + checks := 0 + recorded := syncOnceGuard(t, func() (bool, error) { checks++; return false, nil }) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + + syncOnce(context.Background(), newTestCLICommand()) + + assert.Empty(t, *recorded) + assert.Zero(t, checks) + assert.Equal(t, testOverrideFlags, readFlagFile(t)) +} + +func TestSyncFlagsSetsPending(t *testing.T) { + setupGuard(t, testOverrideFlags, nil) + + changed, _, _, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, sha256Hex(testOverrideFlags), loadFlagsState(flagsStatePath()).Pending) + + // Unchanged content leaves Pending to the verification, it never sets a new one + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{})) + changed, _, _, err = syncFlags(context.Background()) + require.NoError(t, err) + assert.False(t, changed) + assert.Empty(t, loadFlagsState(flagsStatePath()).Pending) +} + +func TestVerifyAfterRestartClearsPending(t *testing.T) { + for name, running := range map[string]func() (bool, error){ + "ok": nil, + "rejected": neverRunning, + } { + t.Run(name, func(t *testing.T) { + setupGuard(t, testOverrideFlags, running) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{Pending: sha256Hex(testOverrideFlags)})) + require.NoError(t, verifyFlagsAfterRestart(context.Background(), testOverrideFlags, testHostUUID)) + st := loadFlagsState(flagsStatePath()) + assert.Empty(t, st.Pending) + require.NotNil(t, st.Last) + assert.Equal(t, name, st.Last.Result) + }) + } +} + +func TestSyncOnceRestartErrorStillVerifies(t *testing.T) { + syncOnceGuard(t, neverRunning) + restarts := 0 + execCommand = func(name string, args ...string) *exec.Cmd { + restarts++ + return exec.Command("false") + } + + syncOnce(context.Background(), newTestCLICommand()) + + assert.Equal(t, 2, restarts, "failed restart, then the rollback restart") + assert.Equal(t, testEnvFlags, readFlagFile(t)) + st := loadFlagsState(flagsStatePath()) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256) + assert.Empty(t, st.Pending) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) +} + +func TestSyncOnceResumesInterruptedCheck(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // osctrld is stopped while waiting for osqueryd + recorded := syncOnceGuard(t, func() (bool, error) { cancel(); return false, nil }) + + syncOnce(ctx, newTestCLICommand()) + assert.Len(t, *recorded, 1, "restart for the new flags") + assert.Equal(t, testOverrideFlags, readFlagFile(t)) + st := loadFlagsState(flagsStatePath()) + assert.Equal(t, sha256Hex(testOverrideFlags), st.Pending, "unverified flags stay pending") + assert.Nil(t, st.Last) + + // Next start: flags are unchanged on disk, the check resumes without a + // restart and rolls back + *recorded = nil + osquerydRunning = neverRunning + syncOnce(context.Background(), newTestCLICommand()) + assert.Len(t, *recorded, 1, "only the rollback restart") + assert.Equal(t, testEnvFlags, readFlagFile(t)) + st = loadFlagsState(flagsStatePath()) + assert.Empty(t, st.Pending) + assert.Equal(t, sha256Hex(testOverrideFlags), st.RejectedSHA256) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) + + // After the rollback nothing re-verifies the environment flags: the + // rejection stays reported + *recorded = nil + osquerydRunning = func() (bool, error) { return true, nil } + syncOnce(context.Background(), newTestCLICommand()) + assert.Empty(t, *recorded) + st = loadFlagsState(flagsStatePath()) + assert.Empty(t, st.Pending) + require.NotNil(t, st.Last) + assert.Equal(t, "rejected", st.Last.Result) +} + +func TestSyncOnceResumedCheckOK(t *testing.T) { + recorded := syncOnceGuard(t, nil) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{Pending: sha256Hex(testOverrideFlags)})) + + syncOnce(context.Background(), newTestCLICommand()) + + assert.Empty(t, *recorded, "a resumed check does not restart") + st := loadFlagsState(flagsStatePath()) + assert.Empty(t, st.Pending) + require.NotNil(t, st.Last) + assert.Equal(t, "ok", st.Last.Result) +} + +func TestSyncOnceClearsStalePending(t *testing.T) { + checks := 0 + recorded := syncOnceGuard(t, func() (bool, error) { checks++; return false, nil }) + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{Pending: sha256Hex("edited since")})) + + syncOnce(context.Background(), newTestCLICommand()) + + assert.Zero(t, checks) + assert.Empty(t, *recorded) + st := loadFlagsState(flagsStatePath()) + assert.Empty(t, st.Pending) + assert.Nil(t, st.Last) +} + +func TestSyncFlagsUsesCachedUUID(t *testing.T) { + fs, _ := setupGuard(t, testOverrideFlags, nil) + cached := "11111111-2222-3333-4444-555555555555" + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{HostUUID: cached})) + hostUUIDReader = func() (string, error) { return "", errors.New("osqueryd busy") } + + _, attempted, uuid, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.Equal(t, cached, uuid) + assert.Equal(t, testOverrideFlags, attempted, "no flap back to environment flags") + reqs := fs.recorded() + require.Len(t, reqs, 1) + assert.Equal(t, cached, reqs[0].UUID) + assert.Equal(t, cached, loadFlagsState(flagsStatePath()).HostUUID) +} + +func TestSyncFlagsCachesUUID(t *testing.T) { + setupGuard(t, testOverrideFlags, nil) + require.NoError(t, saveFlagsState(flagsStatePath(), flagsState{HostUUID: "11111111-2222-3333-4444-555555555555"})) + // Unchanged flags still refresh the cache + require.NoError(t, os.WriteFile(appConfig.OsqueryFlagFile, []byte(testOverrideFlags), 0700)) + + changed, _, _, err := syncFlags(context.Background()) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, testHostUUID, loadFlagsState(flagsStatePath()).HostUUID) +} + +// The reported text names the configured timeout, and keeps its wording for the default. +func TestRejectedErrorNamesTimeout(t *testing.T) { + prev := restartCheckTimeout + t.Cleanup(func() { restartCheckTimeout = prev }) + + restartCheckTimeout = 60 * time.Second + assert.Equal(t, "osqueryd was not running 60s after restart", rejectedError()) + restartCheckTimeout = 90 * time.Second + assert.Equal(t, "osqueryd was not running 90s after restart", rejectedError()) +} + +func rejectedErrorFor(d time.Duration) string { + return fmt.Sprintf("osqueryd was not running %gs after restart", d.Seconds()) +} diff --git a/cmd/osctrld/hostid.go b/cmd/osctrld/hostid.go new file mode 100644 index 0000000..3c07ff3 --- /dev/null +++ b/cmd/osctrld/hostid.go @@ -0,0 +1,74 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "runtime" + "strings" + "time" +) + +var ( + // hostUUIDReader is a seam so callers can be tested without osqueryd + hostUUIDReader = readHostUUID + // hostUUIDTimeout bounds osqueryd -S so a hung binary can never block the service loop + hostUUIDTimeout = 15 * time.Second +) + +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// readHostUUID returns the UUID osquery enrolls with (system_info.uuid) +func readHostUUID() (string, error) { + var osquerydBin string + switch runtime.GOOS { + case DarwinOS: + osquerydBin = OsqueryDarwin[1] + case LinuxOS: + osquerydBin = OsqueryLinux[1] + case WindowsOS: + osquerydBin = OsqueryWindows[1] + } + out, err := outputWithTimeout(osquerydBin, hostUUIDTimeout, "-S", "--json", "select uuid from system_info;") + if err != nil { + return "", fmt.Errorf("error running osqueryd - %v", err) + } + var rows []struct { + UUID string `json:"uuid"` + } + if err := json.Unmarshal(out, &rows); err != nil { + return "", fmt.Errorf("error parsing osqueryd output - %v", err) + } + if len(rows) == 0 { + return "", fmt.Errorf("osqueryd returned no system_info row") + } + uuid := strings.TrimSpace(rows[0].UUID) + if !uuidPattern.MatchString(uuid) { + return "", fmt.Errorf("invalid host uuid %q", uuid) + } + return uuid, nil +} + +// outputWithTimeout runs a command through the execCommand seam and returns its +// stdout, killing it once timeout passes. WaitDelay bounds Wait too, in case a +// child process keeps the output pipe open after the kill. +func outputWithTimeout(name string, timeout time.Duration, args ...string) ([]byte, error) { + cmd := execCommand(name, args...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.WaitDelay = time.Second + if err := cmd.Start(); err != nil { + return nil, err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + return out.Bytes(), err + case <-time.After(timeout): + _ = cmd.Process.Kill() + <-done + return nil, fmt.Errorf("timed out after %s", timeout) + } +} diff --git a/cmd/osctrld/hostid_test.go b/cmd/osctrld/hostid_test.go new file mode 100644 index 0000000..087c6d2 --- /dev/null +++ b/cmd/osctrld/hostid_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testHostUUID = "4C4C4544-0042-3510-8051-B4C04F4E4B32" + +// fakeExecOutput makes execCommand run a shell that prints out (or fails) +func fakeExecOutput(t *testing.T, out string, fail bool) { + t.Helper() + prev := execCommand + t.Cleanup(func() { execCommand = prev }) + execCommand = func(name string, args ...string) *exec.Cmd { + if fail { + return exec.Command("false") + } + return exec.Command("printf", "%s", out) + } +} + +func TestReadHostUUID(t *testing.T) { + fakeExecOutput(t, `[{"uuid":"`+testHostUUID+`"}]`+"\n", false) + got, err := readHostUUID() + require.NoError(t, err) + assert.Equal(t, testHostUUID, got) +} + +func TestReadHostUUIDFailures(t *testing.T) { + tests := []struct { + name string + out string + fail bool + }{ + {"command error", "", true}, + {"empty array", "[]", false}, + {"not json", "not json", false}, + {"not a uuid", `[{"uuid":"abc"}]`, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeExecOutput(t, tt.out, tt.fail) + _, err := readHostUUID() + assert.Error(t, err) + }) + } +} + +// A hung osqueryd -S must never block the service loop +func TestReadHostUUIDTimeout(t *testing.T) { + prevExec, prevTimeout := execCommand, hostUUIDTimeout + t.Cleanup(func() { execCommand, hostUUIDTimeout = prevExec, prevTimeout }) + execCommand = func(name string, args ...string) *exec.Cmd { return exec.Command("sleep", "10") } + hostUUIDTimeout = 50 * time.Millisecond + + start := time.Now() + _, err := readHostUUID() + require.Error(t, err) + assert.Contains(t, err.Error(), "timed out") + assert.Less(t, time.Since(start), 5*time.Second) +} + +func TestRetrieveFlagsSendsIdentity(t *testing.T) { + var got FlagsRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&got)) + _, _ = w.Write([]byte("--flag=1")) + })) + defer server.Close() + osctrlURLs.Flags = server.URL + appConfig.Insecure = false + + at := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + _, err := retrieveFlags("s", "/sf", "/cf", testHostUUID, + &ApplyStatus{Result: "rejected", SHA256: "ab", Error: "e", At: at}) + require.NoError(t, err) + assert.Equal(t, testHostUUID, got.UUID) + require.NotNil(t, got.Apply) + assert.Equal(t, "rejected", got.Apply.Result) + assert.Equal(t, "ab", got.Apply.SHA256) + assert.Equal(t, "e", got.Apply.Error) + assert.True(t, at.Equal(got.Apply.At)) +} + +func TestRetrieveFlagsOmitsEmptyIdentity(t *testing.T) { + var raw []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ = io.ReadAll(r.Body) + _, _ = w.Write([]byte("--flag=1")) + })) + defer server.Close() + osctrlURLs.Flags = server.URL + appConfig.Insecure = false + + _, err := retrieveFlags("s", "/sf", "/cf", "", nil) + require.NoError(t, err) + assert.NotContains(t, string(raw), `"uuid"`) + assert.NotContains(t, string(raw), `"apply"`) +} diff --git a/cmd/osctrld/service.go b/cmd/osctrld/service.go index efad659..eed897b 100644 --- a/cmd/osctrld/service.go +++ b/cmd/osctrld/service.go @@ -25,11 +25,17 @@ func syncOnce(ctx context.Context, cmd *cli.Command) { var flagsChanged, certChanged, extensionsChanged bool + if err := resumeFlagsCheck(ctx); err != nil { + log.Error().Err(err).Msg("failed to resume flags check") + if ctx.Err() != nil { + return + } + } + log.Info().Msg("syncing flags") - if changed, err := getFlags(ctx, cmd); err != nil { + flagsChanged, attempted, uuid, err := syncFlags(ctx) + if err != nil { log.Error().Err(err).Msg("failed to sync flags") - } else { - flagsChanged = changed } log.Info().Msg("syncing cert") @@ -52,8 +58,14 @@ func syncOnce(ctx context.Context, cmd *cli.Command) { Bool("extensions_changed", extensionsChanged). Msg("configuration changed, restarting osquery") if err := restartOsquery(); err != nil { + // Still verify: osqueryd may be down with the new flags either way log.Error().Err(err).Msg("failed to restart osquery") } + if flagsChanged { + if err := verifyFlagsAfterRestart(ctx, attempted, uuid); err != nil { + log.Error().Err(err).Msg("failed to verify flags after restart") + } + } } }