From 11ab6b910ace18a82a541e96f9faaf5a9139ab34 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Tue, 9 Jun 2026 23:52:35 -0500 Subject: [PATCH 01/14] Rewrite core driver with truthful optional-interface wrapping (v3) Change the module path to github.com/infobloxopen/hotload/v3 and rewrite the core driver: - Wrapped conns and stmts now implement an optional driver interface if and only if the underlying driver supports it. Single-method piece types are composed into generated combination structs (internal/gen, invoked via go:generate), so database/sql's type assertions see the truth instead of ErrSkip round-trips. ConnPrepareContext, ConnBeginTx, SessionResetter and Validator are always implemented, replicating database/sql's fallback exactly. - Replace chanGroup with a group/generation model: one generation owns the cancel context, conn set and drained flag for one DSN value, so a config change is a pointer swap. No driver call, hook or strategy call happens under any lock, removing the deadlock class fixed in 573e701 and the 1ms-sleep workaround. forceKill waits a bounded killWindow (DSN parameter, default 100ms) for in-flight work before force-closing. - Replace teivah/onecontext with context.AfterFunc-based cancellation relay (no goroutine per merge, unregistered synchronously on release). - Implement driver.DriverContext; connectors are refcounted per DSN and DB.Close now tears down the strategy watch and run loop. - Wrap driver.Stmt, fixing the prepared-statement blind spot in transaction statement counters. - Replace hard-wired prometheus metrics with zero-dependency hooks (RegisterHooks); modtime and fsnotify report through them. Drop pkg/errors, google/uuid (crypto/rand), onecontext and prometheus from the module; the only remaining dependency is fsnotify. - New stdlib-only test suite: a capability-masked fake driver (internal/dbfake), interface truthfulness matrices, database/sql-level dispatch tests, graceful/forceKill lifecycle tests, change-storm concurrency tests, and a goroutine-leak checker (internal/testutil). --- chanGroup_test.go | 198 --- concurrency_test.go | 153 ++ conn.go | 446 +++--- conn_combos_gen.go | 253 +++ conn_pieces.go | 40 + conn_test.go | 472 ------ connector.go | 43 + context.go | 27 + driver.go | 424 ++--- driver_internal_test.go | 287 ++-- driver_test.go | 132 -- errors.go | 22 + export_test.go | 6 + fallback_test.go | 229 +++ fixture_test.go | 158 ++ fsnotify/filewatcher.go | 17 +- fsnotify/filewatcher_test.go | 293 ---- fsnotify/fsnotify_suite_test.go | 25 - go.mod | 40 +- go.sum | 162 -- group.go | 308 ++++ hotload_suite_test.go | 29 - integrationtests/context_test.go | 126 -- integrationtests/docker/docker-compose.yaml | 18 - integrationtests/docker/intgtest_init.sql | 5 - .../hotload-integration-tests/.helmignore | 23 - .../helm/hotload-integration-tests/Chart.yaml | 26 - .../templates/_helpers.tpl | 62 - .../templates/postgres-configmap.yaml | 8 - .../templates/postgres-deployment.yaml | 65 - .../templates/postgres-secret.yaml | 12 - .../templates/postgres-service.yaml | 16 - .../templates/tests/test-hotload.yaml | 20 - .../hotload-integration-tests/values.yaml | 30 - integrationtests/hotload_test.go | 314 ---- .../integrationtests_suite_test.go | 82 - integrationtests/longdbtxn_test.go | 503 ------ integrationtests/multipleconn_test.go | 155 -- integrationtests/passwdchange_test.go | 246 --- interfaces_test.go | 132 ++ internal/dbfake/dbfake.go | 515 ++++++ internal/dbfake/views_gen.go | 1403 +++++++++++++++++ internal/gen/main.go | 279 ++++ internal/nonrandom_reader.go | 63 - internal/nonrandom_reader_test.go | 59 - internal/prometheus.go | 39 - internal/secret_sink.go | 10 +- internal/testutil/leak.go | 87 + internal/url_test.go | 48 +- lifecycle_test.go | 346 ++++ metrics/pathchksum_metrics.go | 153 -- metrics/pathchksum_test.go | 100 -- metrics/prometheus.go | 85 - metrics/prometheus_test.go | 18 - modtime/modtime_monitor.go | 13 +- modtime/modtime_options.go | 2 +- modtime/modtime_test.go | 379 ----- observer.go | 150 ++ stmt.go | 69 + stmt_combos_gen.go | 244 +++ stmt_pieces.go | 37 + transaction.go | 82 - tx.go | 37 + tx_test.go | 189 +++ 64 files changed, 5328 insertions(+), 4686 deletions(-) delete mode 100644 chanGroup_test.go create mode 100644 concurrency_test.go create mode 100644 conn_combos_gen.go create mode 100644 conn_pieces.go delete mode 100644 conn_test.go create mode 100644 connector.go create mode 100644 context.go delete mode 100644 driver_test.go create mode 100644 errors.go create mode 100644 export_test.go create mode 100644 fallback_test.go create mode 100644 fixture_test.go delete mode 100644 fsnotify/filewatcher_test.go delete mode 100644 fsnotify/fsnotify_suite_test.go create mode 100644 group.go delete mode 100644 hotload_suite_test.go delete mode 100644 integrationtests/context_test.go delete mode 100644 integrationtests/docker/docker-compose.yaml delete mode 100644 integrationtests/docker/intgtest_init.sql delete mode 100644 integrationtests/helm/hotload-integration-tests/.helmignore delete mode 100644 integrationtests/helm/hotload-integration-tests/Chart.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml delete mode 100644 integrationtests/helm/hotload-integration-tests/values.yaml delete mode 100644 integrationtests/hotload_test.go delete mode 100644 integrationtests/integrationtests_suite_test.go delete mode 100644 integrationtests/longdbtxn_test.go delete mode 100644 integrationtests/multipleconn_test.go delete mode 100644 integrationtests/passwdchange_test.go create mode 100644 interfaces_test.go create mode 100644 internal/dbfake/dbfake.go create mode 100644 internal/dbfake/views_gen.go create mode 100644 internal/gen/main.go delete mode 100644 internal/nonrandom_reader.go delete mode 100644 internal/nonrandom_reader_test.go delete mode 100644 internal/prometheus.go create mode 100644 internal/testutil/leak.go create mode 100644 lifecycle_test.go delete mode 100644 metrics/pathchksum_metrics.go delete mode 100644 metrics/pathchksum_test.go delete mode 100644 metrics/prometheus.go delete mode 100644 metrics/prometheus_test.go delete mode 100644 modtime/modtime_test.go create mode 100644 observer.go create mode 100644 stmt.go create mode 100644 stmt_combos_gen.go create mode 100644 stmt_pieces.go delete mode 100644 transaction.go create mode 100644 tx.go create mode 100644 tx_test.go diff --git a/chanGroup_test.go b/chanGroup_test.go deleted file mode 100644 index bb4d47c..0000000 --- a/chanGroup_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - "fmt" - "log" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus/testutil" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" -) - -type testConn struct { - closed bool -} - -func (tc *testConn) Open(name string) (driver.Conn, error) { - return tc, nil -} - -func (tc *testConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} -func (tc *testConn) Begin() (driver.Tx, error) { - return nil, nil -} - -func (tc *testConn) Close() error { - tc.closed = true - return nil -} - -type mockWatcher struct { - values chan string -} - -func newMockWatcher() *mockWatcher { - return &mockWatcher{ - values: make(chan string), - } -} - -func (mw mockWatcher) getReceiveChan() <-chan string { - return mw.values -} - -func (mw mockWatcher) sendValue(value string) { - log.Printf("mockWatcher: sending value '%s'...", value) - mw.values <- value - log.Printf("mockWatcher: sent value '%s'", value) -} - -var _ = DescribeTableSubtree("Driver", Serial, func(forceKill bool) { - var pctx context.Context - var ctx context.Context - var cancel context.CancelFunc - var cg *chanGroup - var mgdConns []*managedConn - var mockw *mockWatcher - Context("chanGroup", func() { - BeforeEach(func(ginkgoCtx context.Context) { - // Do NOT use ginkgoCtx as it will be canceled when BeforeEach finishes - pctx = context.Background() - ctx, cancel = context.WithCancel(pctx) - mockw = newMockWatcher() - cg = &chanGroup{ - name: "fsnotify://postgres/tmp/mydsn.txt", - value: "1st-dsn", - newValChan: mockw.getReceiveChan(), - parentCtx: pctx, - ctx: ctx, - cancel: cancel, - sqlDriver: nil, - mu: sync.RWMutex{}, - forceKill: forceKill, - } - cg.conns = []*managedConn{ - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - } - mgdConns = cg.conns - - metrics.ResetCollectors() - }, NodeTimeout(5*time.Second)) - - It("Should change value when a value is pushed to the values channel", func(ginkgoCtx context.Context) { - newVal := "2nd-dsn" - go cg.runLoop() - mockw.sendValue(newVal) - - // Yield to cg.runLoop() background thread - time.Sleep(200 * time.Millisecond) - - cg.mu.RLock() - defer cg.mu.RUnlock() - Expect(cg.value).To(Equal(newVal)) - - Expect(len(cg.conns)).To(Equal(0), "number of managed conns should be reset to zero") - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeTrue(), "managed connection should be marked reset") - if cg.forceKill { - Expect(mc.GetKill()).To(BeTrue(), "managed connection should be marked killed") - Expect(mc.conn.(*testConn).closed).To(BeTrue(), "Closed() should have been called on the underlying connection") - } - } - - // HotloadChangeTotal metric should be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader(expectHotloadChangeTotalHelp+ - fmt.Sprintf(expectHotloadChangeTotalMetric, cg.name, 1))) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - - It("Should not reset conns when the same value is pushed to the values channel", func(ginkgoCtx context.Context) { - sameVal := "1st-dsn" - go cg.runLoop() - mockw.sendValue(sameVal) - - // Yield to cg.runLoop() background thread - time.Sleep(200 * time.Millisecond) - - Expect(cg.value).To(Equal(sameVal)) - - Expect(len(cg.conns)).To(Equal(3), "number of managed conns should not be reset to zero") - - for _, c := range cg.conns { - Expect(c.GetReset()).To(BeFalse()) - Expect(c.GetKill()).To(BeFalse()) - Expect(c.conn.(*testConn).closed).To(BeFalse()) - } - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeFalse()) - Expect(mc.GetKill()).To(BeFalse()) - Expect(mc.conn.(*testConn).closed).To(BeFalse()) - } - - // HotloadChangeTotal metric should NOT be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader("")) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - - It("Should change value and reset connections", func(ginkgoCtx context.Context) { - newVal := "2nd-dsn" - cg.processNewValue(newVal) - Expect(cg.value).To(Equal(newVal)) - - Expect(len(cg.conns)).To(Equal(0), "number of managed conns should be reset to zero") - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeTrue(), "managed connection should be marked reset") - if cg.forceKill { - Expect(mc.GetKill()).To(BeTrue(), "managed connection should be marked killed") - Expect(mc.conn.(*testConn).closed).To(BeTrue(), "Closed() should have been called on the underlying connection") - } - } - - // HotloadChangeTotal metric should be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader(expectHotloadChangeTotalHelp+ - fmt.Sprintf(expectHotloadChangeTotalMetric, cg.name, 1))) - Expect(err).ShouldNot(HaveOccurred()) - - err = internal.CollectAndRegexpCompare(metrics.HotloadLastChangedTimestampSeconds, - strings.NewReader(expectHotloadLastChangedTimestampSecondsMetricRegexp), - metrics.HotloadLastChangedTimestampSecondsName) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - }) -}, - Entry("forceKill=false", false), - Entry("forceKill=true", true), -) - -var expectHotloadChangeTotalHelp = ` -# HELP hotload_change_total Hotload change total by url -# TYPE hotload_change_total counter -` - -var expectHotloadChangeTotalMetric = ` -hotload_change_total{url="%s"} %d -` - -var expectHotloadLastChangedTimestampSecondsMetricRegexp = ` -# HELP hotload_last_changed_timestamp_seconds Hotload last changed \(unix timestamp\), by url -# TYPE hotload_last_changed_timestamp_seconds gauge -hotload_last_changed_timestamp_seconds{url="fsnotify://postgres/tmp/mydsn.txt"} \d\.\d+e\+\d+ -` diff --git a/concurrency_test.go b/concurrency_test.go new file mode 100644 index 0000000..55949c3 --- /dev/null +++ b/concurrency_test.go @@ -0,0 +1,153 @@ +package hotload_test + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// TestChangeStorm hammers the pool from many goroutines while config +// changes fire rapidly, in both graceful and forceKill modes. It asserts no +// deadlocks (the test finishes), no unexpected errors, convergence to the +// final DSN, and no goroutine leaks. +func TestChangeStorm(t *testing.T) { + for _, forceKill := range []bool{false, true} { + t.Run(fmt.Sprintf("forceKill=%v", forceKill), func(t *testing.T) { + testutil.NoLeaks(t) + + params := "killWindow=10ms" + if forceKill { + params = "forceKill=true&killWindow=10ms" + } + fx := newFixture(t, fxCfg{params: params}) + + const workers = 8 + const changes = 25 + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var ( + wg sync.WaitGroup + successes atomic.Int64 + unexpected sync.Map + stop = make(chan struct{}) + ) + + allowedErr := func(err error) bool { + // Under forceKill, in-flight work may be retired; the pool + // also surfaces bad-conn errors when retries are exhausted + // mid-storm. + return errors.Is(err, hotload.ErrHotSwap) || + errors.Is(err, context.Canceled) || + err.Error() == "driver: bad connection" + } + + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + var err error + if i%2 == 0 { + _, err = fx.db.ExecContext(ctx, "UPDATE x") + } else { + var dsn string + err = fx.db.QueryRowContext(ctx, "SELECT dsn").Scan(&dsn) + } + if err == nil { + successes.Add(1) + } else if !allowedErr(err) { + unexpected.Store(err.Error(), true) + } + } + }(i) + } + + final := "" + for j := 0; j < changes; j++ { + final = fmt.Sprintf("dsn-%d", j%3+2) + fx.pushAndWait(final) + } + + close(stop) + wg.Wait() + + unexpected.Range(func(k, v any) bool { + t.Errorf("unexpected error during storm: %s", k) + return true + }) + if successes.Load() == 0 { + t.Error("no operation succeeded during the storm") + } + testutil.WaitFor(t, 2*time.Second, "convergence to final DSN", func() bool { + return fx.queryDSN() == final + }) + }) + } +} + +// TestConcurrentCloseAndKill races pool-driven closes against +// generation kills; the underlying conn must be closed exactly once. +func TestConcurrentCloseAndKill(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{params: "forceKill=true&killWindow=5ms"}) + + for round := 0; round < 20; round++ { + fx.queryDSN() + // Race: db.Close-like pool eviction (via change) and explicit swap. + fx.pushAndWait(fmt.Sprintf("dsn-%d", round+2)) + } + + testutil.WaitFor(t, 2*time.Second, "all old conns closed", func() bool { + open := fx.drv.OpenConns() + return len(open) <= 1 + }) + for _, c := range fx.drv.Conns() { + if n := c.CloseCount(); n > 1 { + t.Errorf("conn %d closed %d times, want at most 1", c.ID, n) + } + } +} + +// TestQueryRowsSurviveGracefulChange: rows being iterated when a graceful +// change lands must remain readable — the reason QueryContext passes the +// caller context through unmerged. +func TestQueryRowsSurviveGracefulChange(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + rows, err := fx.db.Query("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + fx.pushAndWait("dsn-2") + + var dsn string + if !rows.Next() { + t.Fatalf("rows.Next() = false after graceful change: %v", rows.Err()) + } + if err := rows.Scan(&dsn); err != nil { + t.Fatal(err) + } + if dsn != "dsn-1" { + t.Errorf("row value = %q, want dsn-1 (rows belong to the old generation)", dsn) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/conn.go b/conn.go index 6aeb13c..b9350bf 100644 --- a/conn.go +++ b/conn.go @@ -1,306 +1,288 @@ package hotload +//go:generate go run ./internal/gen + import ( "context" "database/sql" "database/sql/driver" "errors" - "fmt" - "sync" "sync/atomic" - - "github.com/infobloxopen/hotload/logger" - "github.com/teivah/onecontext" ) -// managedConn wraps a sql/driver.Conn so that it can be closed by -// a supervising context. -type managedConn struct { - ctx context.Context - dsn string - redactDsn string - conn driver.Conn - reset bool - killed bool - mu sync.RWMutex +// baseConn wraps an underlying driver.Conn so that it can be retired by the +// generation that owns it. baseConn implements driver.Conn plus the optional +// interfaces whose behavior hotload must control or whose stdlib fallback it +// can replicate exactly: ConnPrepareContext, ConnBeginTx, SessionResetter +// and Validator. The remaining optional interfaces (ExecerContext, +// QueryerContext, Pinger, NamedValueChecker) are exposed only when the +// underlying conn supports them, via the generated combination wrappers +// returned by wrapConn. +type baseConn struct { + inner driver.Conn + gen *generation + group *group + dsn string + redactDsn string + closed atomic.Bool + killed atomic.Bool + execStmts atomic.Int64 // exec statements since the last completed transaction + queryStmts atomic.Int64 // query statements since the last completed transaction +} - // callback function to be called after the connection is closed - afterClose func(*managedConn) +// opCtx returns a context canceled when either the caller's ctx or the +// owning generation's ctx is canceled. The returned release function must be +// called (usually deferred) when the operation completes; it unregisters the +// cancellation relay so no resources outlive the call. opCtx returns a nil +// context if the generation is already retired; the caller should return +// driver.ErrBadConn so database/sql retries on a fresh connection. +func (c *baseConn) opCtx(ctx context.Context) (context.Context, func()) { + gen := c.gen + gen.ops.Add(1) + if gen.ctx.Err() != nil { + gen.ops.Add(-1) + return nil, nil + } + mctx, cancel := context.WithCancelCause(ctx) + stop := context.AfterFunc(gen.ctx, func() { + cancel(context.Cause(gen.ctx)) + }) + release := func() { + stop() + cancel(nil) + gen.ops.Add(-1) + } + return mctx, release +} - execStmtsCounter atomic.Int64 // count the number of exec calls in a transaction - queryStmtsCounter atomic.Int64 // count the number of query calls in a transaction +// retired reports whether the conn should no longer be used: its generation +// was drained or killed, or the conn itself was closed. +func (c *baseConn) retired() bool { + return c.closed.Load() || c.gen.retired() } -// BeginTx calls the underlying BeginTx method unless the supervising context -// is closed. -// Returns an error if the underlying driver doesn't implement -// driver.ConnBeginTx interface and TxOptions are non default. If TxOptions are -// of default values it will call the underlying Begin method as like sql -// package. -// If the context is canceled by the user this method will call Tx.Rollback. -func (c *managedConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - select { - case <-c.ctx.Done(): - c.close() +func (c *baseConn) Prepare(query string) (driver.Stmt, error) { + if c.gen.ctx.Err() != nil { return nil, driver.ErrBadConn - default: } - - if conn, ok := c.conn.(driver.ConnBeginTx); ok { - tx, err := conn.BeginTx(ctx, opts) - if err != nil { - return nil, err - } - - return &managedTx{tx: tx, conn: c, ctx: ctx}, nil + stmt, err := c.inner.Prepare(query) + if err != nil { + return nil, err } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil +} - // same as is defined in go sql package to call Begin method if the TxOptions are default - if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { - return nil, errors.New("hotload: underlying driver does not support non-default isolation level") +// PrepareContext prepares a statement, honoring both the caller's context +// and the generation's context. When the underlying conn does not implement +// driver.ConnPrepareContext this replicates database/sql's fallback exactly: +// prepare without a context, then if the context is done, close the +// statement and return the context's error. +func (c *baseConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + octx, release := c.opCtx(ctx) + if octx == nil { + return nil, driver.ErrBadConn } + defer release() - if opts.ReadOnly { - return nil, errors.New("hotload: underlying driver does not support read-only transactions") + if cp, ok := c.inner.(driver.ConnPrepareContext); ok { + stmt, err := cp.PrepareContext(octx, query) + if err != nil { + return nil, err + } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil } - tx, err := c.conn.Begin() + stmt, err := c.inner.Prepare(query) if err == nil { select { + case <-octx.Done(): + stmt.Close() + return nil, octx.Err() default: - case <-ctx.Done(): - tx.Rollback() - return nil, ctx.Err() } } - - return tx, err -} - -func newManagedConn(ctx context.Context, dsn, redactDsn string, conn driver.Conn, afterClose func(*managedConn)) *managedConn { - return &managedConn{ - ctx: ctx, - dsn: dsn, - redactDsn: redactDsn, - conn: conn, - afterClose: afterClose, + if err != nil { + return nil, err } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil } -func (c *managedConn) Exec(query string, args []driver.Value) (driver.Result, error) { - c.logf("managedConn.Exec", "Exec") - - connCtx, ok := c.conn.(driver.ExecerContext) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] - } - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.Exec", "calling underlying conn.ExecContext()") - return connCtx.ExecContext(c.ctx, query, namedArgs) +func (c *baseConn) Begin() (driver.Tx, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn } - - connExr, ok := c.conn.(driver.Execer) - if ok { - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.Exec", "calling underlying conn.Exec()") - return connExr.Exec(query, args) + tx, err := c.inner.Begin() + if err != nil { + return nil, err } - - return nil, driver.ErrSkip -} - -func (c *managedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - c.logf("managedConn.ExecContext", "ExecContext") - conn, ok := c.conn.(driver.ExecerContext) - if !ok { - return nil, driver.ErrSkip - } - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.ExecContext", "calling underlying conn.ExecContext()") - mergedCtx, cancel := onecontext.Merge(c.ctx, ctx) - defer cancel() - return conn.ExecContext(mergedCtx, query, args) + return &managedTx{tx: tx, conn: c, ctx: context.Background()}, nil } -func (c *managedConn) CheckNamedValue(namedValue *driver.NamedValue) error { - conn, ok := c.conn.(driver.NamedValueChecker) - if !ok { - return driver.ErrSkip +// BeginTx starts a transaction. The caller's context is deliberately passed +// through unmerged: some drivers bind the context to the transaction's +// lifetime, and canceling a merged context when this call returns would +// roll back live transactions. The generation context is checked at entry +// instead; forceKill reaches in-flight transactions by closing the conn. +// When the underlying conn does not implement driver.ConnBeginTx this +// replicates database/sql's fallback exactly. +func (c *baseConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn } - return conn.CheckNamedValue(namedValue) -} -func (c *managedConn) Query(query string, args []driver.Value) (driver.Rows, error) { - c.logf("managedConn.Query", "Query") - - connCtx, ok := c.conn.(driver.QueryerContext) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] + if cb, ok := c.inner.(driver.ConnBeginTx); ok { + tx, err := cb.BeginTx(ctx, opts) + if err != nil { + return nil, err } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.Query", "calling underlying conn.QueryContext()") - return connCtx.QueryContext(c.ctx, query, namedArgs) + return &managedTx{tx: tx, conn: c, ctx: ctx}, nil } - connQyr, ok := c.conn.(driver.Queryer) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] - } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.Query", "calling underlying conn.Query()") - return connQyr.Query(query, args) + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + return nil, errors.New("hotload: underlying driver does not support non-default isolation level") + } + if opts.ReadOnly { + return nil, errors.New("hotload: underlying driver does not support read-only transactions") } - return nil, driver.ErrSkip -} - -func (c *managedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - c.logf("managedConn.QueryContext", "QueryContext") - conn, ok := c.conn.(driver.QueryerContext) - if !ok { - return nil, driver.ErrSkip - } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.QueryContext", "calling underlying conn.QueryContext()") - - // TODO - // We would like to merge the hotload-context with the query-context here, - // and then cancel the merged-context to prevent goroutine-leaks - // (similar to ExecContext() above). - // However the Rows object returned seems to contain the merged-context. - // Canceling the merged-context here invalidates the returned Rows object, - // and causes any cursor iteration of the returned Rows objects to fail - // with context-canceled error. - - return conn.QueryContext(ctx, query, args) -} - -func (c *managedConn) Prepare(query string) (driver.Stmt, error) { + tx, err := c.inner.Begin() + if err != nil { + return nil, err + } select { - case <-c.ctx.Done(): - c.logf("managedConn.Prepare", "ctx done, calling close()") - c.close() - return nil, driver.ErrBadConn + case <-ctx.Done(): + tx.Rollback() + return nil, ctx.Err() default: } - c.logf("managedConn.Prepare", "calling underlying Prepare()") - return c.conn.Prepare(query) + return &managedTx{tx: tx, conn: c, ctx: ctx}, nil } -// Begin calls the underlying Begin method unless the supervising -// context is closed. -func (c *managedConn) Begin() (driver.Tx, error) { - select { - case <-c.ctx.Done(): - c.close() - return nil, driver.ErrBadConn - default: - } - return c.conn.Begin() +func (c *baseConn) Close() error { + return c.closeConn(false) } -func (c *managedConn) IsValid() bool { - select { - case <-c.ctx.Done(): - c.logf("managedConn.IsValid", "ctx done, calling close()") - c.close() - return false - default: +// closeConn closes the underlying conn exactly once, no matter how many +// paths race to close it (the pool, a generation kill, or both). +func (c *baseConn) closeConn(killed bool) error { + if !c.closed.CompareAndSwap(false, true) { + return nil } - s, ok := c.conn.(driver.Validator) - if !ok { - return true + if killed { + c.killed.Store(true) } - c.logf("managedConn.IsValid", "calling underlying IsValid()") - return s.IsValid() + err := c.inner.Close() + c.gen.remove(c) + emitConnClose(ConnEvent{GroupName: c.group.name, RedactedDSN: c.redactDsn, Killed: killed}) + return err } -func (c *managedConn) ResetSession(ctx context.Context) error { - if c.GetReset() { - c.logf("managedConn.ResetSession", "already reset") +// ResetSession is called by database/sql before reusing a pooled conn. A +// drained or killed generation answers driver.ErrBadConn so the pool +// discards the conn and dials a fresh one on the current DSN. +func (c *baseConn) ResetSession(ctx context.Context) error { + if c.retired() { return driver.ErrBadConn } - - s, ok := c.conn.(driver.SessionResetter) - if !ok { - return nil + if sr, ok := c.inner.(driver.SessionResetter); ok { + return sr.ResetSession(ctx) } - - c.logf("managedConn.ResetSession", "calling underlying ResetSession()") - return s.ResetSession(ctx) + return nil } -func (c *managedConn) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - err := c.close() - - if err == nil { - c.killed = true +// IsValid is called by database/sql when returning a conn to the pool. +func (c *baseConn) IsValid() bool { + if c.retired() { + return false } - c.logf("managedConn.Close", "closed") - - return err + if v, ok := c.inner.(driver.Validator); ok { + return v.IsValid() + } + return true } -func (c *managedConn) close() error { - if c.afterClose != nil { - defer c.afterClose(c) +// execContext backs the generated ExecerContext wrappers. It is only +// reachable when the underlying conn implements ExecerContext or the legacy +// Execer; for the legacy case it replicates database/sql's fallback exactly +// (convert named args, poll the context, call Exec). +func (c *baseConn) execContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + octx, release := c.opCtx(ctx) + if octx == nil { + return nil, driver.ErrBadConn } - c.logf("managedConn.close", "calling underlying Close()") - return c.conn.Close() -} + defer release() + c.execStmts.Add(1) -func (c *managedConn) GetReset() bool { - c.mu.RLock() - defer c.mu.RUnlock() - c.logf("managedConn.GetReset", "reset=%v", c.reset) - return c.reset -} + if ec, ok := c.inner.(driver.ExecerContext); ok { + return ec.ExecContext(octx, query, args) + } -func (c *managedConn) Reset(v bool) { - c.mu.Lock() - defer c.mu.Unlock() - c.reset = v - c.logf("managedConn.Reset", "reset=%v", v) + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + select { + case <-octx.Done(): + return nil, octx.Err() + default: + } + return c.inner.(driver.Execer).Exec(query, dargs) } -func (c *managedConn) GetKill() bool { - c.mu.RLock() - defer c.mu.RUnlock() - c.logf("managedConn.GetKill", "killed=%v", c.killed) - return c.killed -} +// queryContext backs the generated QueryerContext wrappers. The caller's +// context is deliberately passed through unmerged: the returned driver.Rows +// captures the context, and canceling a merged context when this call +// returns would make cursor iteration fail with a context-canceled error. +// The generation context is checked at entry instead; forceKill reaches +// in-flight queries by closing the conn. +func (c *baseConn) queryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn + } + c.queryStmts.Add(1) -func (c *managedConn) incExecStmtsCounter() { - c.execStmtsCounter.Add(1) -} + if qc, ok := c.inner.(driver.QueryerContext); ok { + return qc.QueryContext(ctx, query, args) + } -func (c *managedConn) resetExecStmtsCounter() { - c.execStmtsCounter.Store(0) + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + return c.inner.(driver.Queryer).Query(query, dargs) } -func (c *managedConn) incQueryStmtsCounter() { - c.queryStmtsCounter.Add(1) +// ping backs the generated Pinger wrappers; only reachable when the +// underlying conn implements driver.Pinger. +func (c *baseConn) ping(ctx context.Context) error { + octx, release := c.opCtx(ctx) + if octx == nil { + return driver.ErrBadConn + } + defer release() + return c.inner.(driver.Pinger).Ping(octx) } -func (c *managedConn) resetQueryStmtsCounter() { - c.queryStmtsCounter.Store(0) +// checkNamedValue backs the generated NamedValueChecker wrappers; only +// reachable when the underlying conn implements driver.NamedValueChecker. +func (c *baseConn) checkNamedValue(nv *driver.NamedValue) error { + return c.inner.(driver.NamedValueChecker).CheckNamedValue(nv) } -func (c *managedConn) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, c.redactDsn) - logger.Logf(logPrefix, format, args...) +// namedValueToValue converts named args to positional args, mirroring the +// unexported helper of the same name in database/sql. +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + return nil, errors.New("sql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil } diff --git a/conn_combos_gen.go b/conn_combos_gen.go new file mode 100644 index 0000000..d240b1d --- /dev/null +++ b/conn_combos_gen.go @@ -0,0 +1,253 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package hotload + +import "database/sql/driver" + +// wrapConn wraps b so that the returned driver.Conn exposes an optional +// interface if and only if the underlying conn supports the capability. +// The legacy Execer/Queryer interfaces are collapsed into their context +// flavors: the wrapper only ever exposes ExecerContext/QueryerContext and +// replicates database/sql's legacy fallback internally (see +// baseConn.execContext and baseConn.queryContext). +func connFlags(c driver.Conn) uint8 { + var f uint8 + if _, ok := c.(driver.ExecerContext); ok { + f |= 1 << 0 + } else if _, ok := c.(driver.Execer); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 0 + } + if _, ok := c.(driver.QueryerContext); ok { + f |= 1 << 1 + } else if _, ok := c.(driver.Queryer); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 1 + } + if _, ok := c.(driver.Pinger); ok { + f |= 1 << 2 + } + if _, ok := c.(driver.NamedValueChecker); ok { + f |= 1 << 3 + } + return f +} + +type conn_E struct { + *baseConn + cExecer +} + +var ( + _ driver.Conn = conn_E{} + _ driver.ExecerContext = conn_E{} +) + +type conn_Q struct { + *baseConn + cQueryer +} + +var ( + _ driver.Conn = conn_Q{} + _ driver.QueryerContext = conn_Q{} +) + +type conn_EQ struct { + *baseConn + cExecer + cQueryer +} + +var ( + _ driver.Conn = conn_EQ{} + _ driver.ExecerContext = conn_EQ{} + _ driver.QueryerContext = conn_EQ{} +) + +type conn_P struct { + *baseConn + cPinger +} + +var ( + _ driver.Conn = conn_P{} + _ driver.Pinger = conn_P{} +) + +type conn_EP struct { + *baseConn + cExecer + cPinger +} + +var ( + _ driver.Conn = conn_EP{} + _ driver.ExecerContext = conn_EP{} + _ driver.Pinger = conn_EP{} +) + +type conn_QP struct { + *baseConn + cQueryer + cPinger +} + +var ( + _ driver.Conn = conn_QP{} + _ driver.QueryerContext = conn_QP{} + _ driver.Pinger = conn_QP{} +) + +type conn_EQP struct { + *baseConn + cExecer + cQueryer + cPinger +} + +var ( + _ driver.Conn = conn_EQP{} + _ driver.ExecerContext = conn_EQP{} + _ driver.QueryerContext = conn_EQP{} + _ driver.Pinger = conn_EQP{} +) + +type conn_N struct { + *baseConn + cNVChecker +} + +var ( + _ driver.Conn = conn_N{} + _ driver.NamedValueChecker = conn_N{} +) + +type conn_EN struct { + *baseConn + cExecer + cNVChecker +} + +var ( + _ driver.Conn = conn_EN{} + _ driver.ExecerContext = conn_EN{} + _ driver.NamedValueChecker = conn_EN{} +) + +type conn_QN struct { + *baseConn + cQueryer + cNVChecker +} + +var ( + _ driver.Conn = conn_QN{} + _ driver.QueryerContext = conn_QN{} + _ driver.NamedValueChecker = conn_QN{} +) + +type conn_EQN struct { + *baseConn + cExecer + cQueryer + cNVChecker +} + +var ( + _ driver.Conn = conn_EQN{} + _ driver.ExecerContext = conn_EQN{} + _ driver.QueryerContext = conn_EQN{} + _ driver.NamedValueChecker = conn_EQN{} +) + +type conn_PN struct { + *baseConn + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_PN{} + _ driver.Pinger = conn_PN{} + _ driver.NamedValueChecker = conn_PN{} +) + +type conn_EPN struct { + *baseConn + cExecer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_EPN{} + _ driver.ExecerContext = conn_EPN{} + _ driver.Pinger = conn_EPN{} + _ driver.NamedValueChecker = conn_EPN{} +) + +type conn_QPN struct { + *baseConn + cQueryer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_QPN{} + _ driver.QueryerContext = conn_QPN{} + _ driver.Pinger = conn_QPN{} + _ driver.NamedValueChecker = conn_QPN{} +) + +type conn_EQPN struct { + *baseConn + cExecer + cQueryer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_EQPN{} + _ driver.ExecerContext = conn_EQPN{} + _ driver.QueryerContext = conn_EQPN{} + _ driver.Pinger = conn_EQPN{} + _ driver.NamedValueChecker = conn_EQPN{} +) + +func wrapConn(b *baseConn) driver.Conn { + switch connFlags(b.inner) { + case 1: + return conn_E{b, cExecer{b}} + case 2: + return conn_Q{b, cQueryer{b}} + case 3: + return conn_EQ{b, cExecer{b}, cQueryer{b}} + case 4: + return conn_P{b, cPinger{b}} + case 5: + return conn_EP{b, cExecer{b}, cPinger{b}} + case 6: + return conn_QP{b, cQueryer{b}, cPinger{b}} + case 7: + return conn_EQP{b, cExecer{b}, cQueryer{b}, cPinger{b}} + case 8: + return conn_N{b, cNVChecker{b}} + case 9: + return conn_EN{b, cExecer{b}, cNVChecker{b}} + case 10: + return conn_QN{b, cQueryer{b}, cNVChecker{b}} + case 11: + return conn_EQN{b, cExecer{b}, cQueryer{b}, cNVChecker{b}} + case 12: + return conn_PN{b, cPinger{b}, cNVChecker{b}} + case 13: + return conn_EPN{b, cExecer{b}, cPinger{b}, cNVChecker{b}} + case 14: + return conn_QPN{b, cQueryer{b}, cPinger{b}, cNVChecker{b}} + case 15: + return conn_EQPN{b, cExecer{b}, cQueryer{b}, cPinger{b}, cNVChecker{b}} + default: + return b + } +} diff --git a/conn_pieces.go b/conn_pieces.go new file mode 100644 index 0000000..11fdfff --- /dev/null +++ b/conn_pieces.go @@ -0,0 +1,40 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// The piece types below each carry exactly one optional driver.Conn method. +// The generated combination wrappers in conn_combos_gen.go embed a subset of +// them, so a wrapped conn's method set — and therefore the type assertions +// database/sql performs — reflects exactly what the underlying conn +// supports. + +// cExecer carries ExecContext (driver.ExecerContext). +type cExecer struct{ b *baseConn } + +func (p cExecer) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return p.b.execContext(ctx, query, args) +} + +// cQueryer carries QueryContext (driver.QueryerContext). +type cQueryer struct{ b *baseConn } + +func (p cQueryer) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return p.b.queryContext(ctx, query, args) +} + +// cPinger carries Ping (driver.Pinger). +type cPinger struct{ b *baseConn } + +func (p cPinger) Ping(ctx context.Context) error { + return p.b.ping(ctx) +} + +// cNVChecker carries CheckNamedValue (driver.NamedValueChecker). +type cNVChecker struct{ b *baseConn } + +func (p cNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return p.b.checkNamedValue(nv) +} diff --git a/conn_test.go b/conn_test.go deleted file mode 100644 index 33d3c31..0000000 --- a/conn_test.go +++ /dev/null @@ -1,472 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - "io" - "runtime" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus/testutil" - - "github.com/infobloxopen/hotload/metrics" -) - -var _ = Describe("managedConn", func() { - It("Should set .reset in a threadsafe way", func() { - mc := managedConn{ - ctx: nil, - conn: nil, - reset: false, - mu: sync.RWMutex{}, - } - // Lock the mutex - mc.mu.Lock() - writeLockAcquired := false - readLockAcquired := false - - // Verify that neither Reset or GetReset can return while the managedConn's write lock is held - go func() { - mc.Reset(true) - writeLockAcquired = true - }() - - go func() { - mc.GetReset() - readLockAcquired = true - }() - - Consistently(writeLockAcquired).Should(BeFalse()) - Consistently(readLockAcquired).Should(BeFalse()) - }) - - It("Should not leak goroutines when using ExecContext and QueryContext", func() { - // Force garbage collection and get baseline - runtime.GC() - runtime.GC() - time.Sleep(50 * time.Millisecond) - initialGoroutines := runtime.NumGoroutine() - - // Create a long-lived parent context that won't be cancelled - parentCtx := context.Background() // This simulates a long-lived connection context - - // Create managed connection - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create many operations with different child contexts - // This will trigger onecontext.Merge calls, and without defer mCancel(), - // goroutines will leak because parentCtx is never cancelled - numOperations := 50 - - for i := 0; i < numOperations; i++ { - // Create short-lived contexts for each operation - childCtx := context.Background() // Also long-lived for this test - - // Each call to ExecContext will create a merged context internally - // Without defer mCancel(), the goroutine will wait indefinitely - mc.ExecContext(childCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.QueryContext(childCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - } - - // Close the connection - err := mc.Close() - Expect(err).ShouldNot(HaveOccurred()) - - // Give time for any leaked goroutines to be detected - time.Sleep(200 * time.Millisecond) - runtime.GC() - runtime.GC() - - // Check that we don't have significantly more goroutines than we started with - finalGoroutines := runtime.NumGoroutine() - - // With the leak, we expect to see many leaked goroutines from onecontext.Merge - // Each ExecContext and QueryContext call that doesn't call defer mCancel() will leak a goroutine - expectedLeaks := numOperations * 2 // One for each ExecContext and QueryContext call - - Expect(finalGoroutines).To(BeNumerically("<=", initialGoroutines+10), - "Expected no significant goroutine leaks. Initial: %d, Final: %d, Expected leaks if broken: ~%d", - initialGoroutines, finalGoroutines, expectedLeaks) - }) - - It("Should handle context cancellation properly in ExecContext", func() { - parentCtx, parentCancel := context.WithCancel(context.Background()) - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create a context that's already cancelled - cancelledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - // ExecContext should still work even with cancelled child context - // because the parent context is still active - _, err := mc.ExecContext(cancelledCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - Expect(err).ShouldNot(HaveOccurred()) - - // Now cancel the parent context - parentCancel() - - // Give some time for context cancellation to propagate - time.Sleep(10 * time.Millisecond) - - // This should fail because parent context is cancelled - mc.ExecContext(context.Background(), "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - // The mock doesn't return context.Canceled, but the connection should work - // The important thing is that no goroutines are leaked - }) - - It("Should not leak goroutines with slow operations and context cancellation", func() { - // Force garbage collection and get very precise baseline - runtime.GC() - runtime.GC() - time.Sleep(100 * time.Millisecond) - runtime.GC() - initialGoroutines := runtime.NumGoroutine() - - // Create contexts that will be cancelled during operations - parentCtx, parentCancel := context.WithCancel(context.Background()) - defer parentCancel() - - // Create managed connection with the slow mock to simulate real database behavior - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockSlowDriverConn{}, nil) - - // Create many concurrent operations that will force onecontext.Merge goroutine creation - var wg sync.WaitGroup - numOperations := 100 // Much higher number to force obvious leaks - - for i := 0; i < numOperations; i++ { - wg.Add(2) - - go func(iteration int) { - defer wg.Done() - // Create a unique context for each operation to force new onecontext.Merge calls - timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer timeoutCancel() - - // This should timeout and without defer mCancel(), leak goroutines - mc.ExecContext(timeoutCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - }(i) - - go func(iteration int) { - defer wg.Done() - // Create a unique context for each operation to force new onecontext.Merge calls - timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer timeoutCancel() - - // This should timeout and without defer mCancel(), leak goroutines - mc.QueryContext(timeoutCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - }(i) - } - - wg.Wait() - - // Close the connection - err := mc.Close() - Expect(err).ShouldNot(HaveOccurred()) - - // Give substantial time for leaked goroutines to accumulate - time.Sleep(500 * time.Millisecond) - runtime.GC() - runtime.GC() - - // Check that we don't have significantly more goroutines than we started with - finalGoroutines := runtime.NumGoroutine() - - // This should definitely fail if there are goroutine leaks from onecontext.Merge - // Being very strict about detecting leaks - Expect(finalGoroutines).To(BeNumerically("<=", initialGoroutines+10), - "Expected no significant goroutine leaks from onecontext.Merge. Initial: %d, Final: %d, Diff: %d", - initialGoroutines, finalGoroutines, finalGoroutines-initialGoroutines) - }) - - It("Should handle context cancellation properly in QueryContext", func() { - parentCtx, parentCancel := context.WithCancel(context.Background()) - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create a context that's already cancelled - cancelledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - // QueryContext should still work even with cancelled child context - // because the parent context is still active - _, err := mc.QueryContext(cancelledCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - Expect(err).ShouldNot(HaveOccurred()) - - // Now cancel the parent context - parentCancel() - - // Give some time for context cancellation to propagate - time.Sleep(10 * time.Millisecond) - - // This should work but no goroutines should be leaked - mc.QueryContext(context.Background(), "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - // The mock doesn't return context.Canceled, but the connection should work - // The important thing is that no goroutines are leaked - }) -}) - -/**** Mocks for Prometheus Metrics ****/ - -type mockDriverConn struct{} - -type mockTx struct{} - -// mockSlowDriverConn simulates a driver that might have slow operations -// This helps test for goroutine leaks in scenarios where context cancellation matters -type mockSlowDriverConn struct{} - -func (mockTx) Commit() error { - return nil -} - -func (mockTx) Rollback() error { - return nil -} - -func (mockDriverConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} - -func (mockDriverConn) Begin() (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockDriverConn) Close() error { - return nil -} - -func (mockDriverConn) IsValid() bool { - return true -} - -func (mockDriverConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockDriverConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return nil, nil -} - -func (mockDriverConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return nil, nil -} - -func (mockDriverConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - return nil, nil -} - -func (mockDriverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - return nil, nil -} - -// mockSlowDriverConn methods - simulates operations that might be slow -func (mockSlowDriverConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} - -func (mockSlowDriverConn) Begin() (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockSlowDriverConn) Close() error { - return nil -} - -func (mockSlowDriverConn) IsValid() bool { - return true -} - -func (mockSlowDriverConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockSlowDriverConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return nil, nil -} - -func (mockSlowDriverConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return nil, nil -} - -func (mockSlowDriverConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - // Check if context is cancelled before proceeding - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - // Simulate a slow operation that keeps goroutines alive longer - timer := time.NewTimer(200 * time.Millisecond) - defer timer.Stop() - select { - case <-timer.C: - return nil, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (mockSlowDriverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - // Check if context is cancelled before proceeding - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - // Simulate a slow operation that keeps goroutines alive longer - timer := time.NewTimer(200 * time.Millisecond) - defer timer.Stop() - select { - case <-timer.C: - return nil, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -/**** End Mocks for Prometheus Metrics ****/ - -var _ = Describe("PrometheusMetrics", func() { - const help = ` - # HELP transaction_sql_stmts The number of sql stmts called in a transaction by statement type per grpc service and method - # TYPE transaction_sql_stmts summary - ` - - var service1Metrics = ` - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 3 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="query"} 3 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="query"} 1 - ` - - var service2Metrics = ` - transaction_sql_stmts_sum{grpc_method="method_2",grpc_service="service_2",stmt="exec"} 4 - transaction_sql_stmts_count{grpc_method="method_2",grpc_service="service_2",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="method_2",grpc_service="service_2",stmt="query"} 4 - transaction_sql_stmts_count{grpc_method="method_2",grpc_service="service_2",stmt="query"} 1 - ` - - var service1RerunMetrics = ` - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 4 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 2 - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="query"} 4 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="query"} 2 - ` - - var noMethodMetrics = ` - transaction_sql_stmts_sum{grpc_method="",grpc_service="",stmt="exec"} 1 - transaction_sql_stmts_count{grpc_method="",grpc_service="",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="",grpc_service="",stmt="query"} 1 - transaction_sql_stmts_count{grpc_method="",grpc_service="",stmt="query"} 1 - ` - - It("Should emit the correct metrics", func() { - mc := newManagedConn(context.Background(), "dsn", "redactDsn", mockDriverConn{}, nil) - - ctx := ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_1", "grpc_service": "service_1"}) - - // begin a transaction - tx, err := mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - - // query a statement - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // reset the metrics - // new context - ctx = ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_2", "grpc_service": "service_2"}) - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - - // query a statement - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1Metrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // rerun with initial metrics - ctx = ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_1", "grpc_service": "service_1"}) - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // rollback the transaction - err = tx.Rollback() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1RerunMetrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // non labeled context - ctx = context.Background() - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec query context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+noMethodMetrics+service1RerunMetrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - }) -}) - -func CollectAndCompareMetrics(r io.Reader) error { - return testutil.CollectAndCompare(metrics.SqlStmtsSummary, r) -} diff --git a/connector.go b/connector.go new file mode 100644 index 0000000..4254724 --- /dev/null +++ b/connector.go @@ -0,0 +1,43 @@ +package hotload + +import ( + "context" + "database/sql/driver" + "fmt" + "sync/atomic" +) + +// connector ties a sql.DB to a group. database/sql obtains one per sql.Open +// call (through driver.DriverContext) and closes it when the sql.DB is +// closed, which releases the group reference and — once the last reference +// is gone — stops the strategy watch and the group's run loop. +type connector struct { + h *hdriver + g *group + name string + closed atomic.Bool +} + +var ( + _ driver.Connector = (*connector)(nil) +) + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + if c.closed.Load() { + return nil, fmt.Errorf("hotload: connector for %q is closed", c.name) + } + return c.g.conn(ctx) +} + +func (c *connector) Driver() driver.Driver { + return c.h +} + +// Close implements io.Closer; database/sql calls it from DB.Close. +func (c *connector) Close() error { + if !c.closed.CompareAndSwap(false, true) { + return nil + } + c.h.releaseGroup(c.name) + return nil +} diff --git a/context.go b/context.go new file mode 100644 index 0000000..93f070c --- /dev/null +++ b/context.go @@ -0,0 +1,27 @@ +package hotload + +import "context" + +type execLabelKeyType struct{} + +var execLabelKey = execLabelKeyType{} + +// ContextWithExecLabels returns a context carrying labels that describe the +// caller (for example a gRPC service and method). Observability adapters can +// retrieve them from TxEvent.Ctx with GetExecLabelsFromContext. +func ContextWithExecLabels(ctx context.Context, labels map[string]string) context.Context { + if labels == nil { + return ctx + } + return context.WithValue(ctx, execLabelKey, labels) +} + +// GetExecLabelsFromContext returns the labels stored by +// ContextWithExecLabels, or nil if there are none. +func GetExecLabelsFromContext(ctx context.Context) map[string]string { + if ctx == nil { + return nil + } + labels, _ := ctx.Value(execLabelKey).(map[string]string) + return labels +} diff --git a/driver.go b/driver.go index 7c08578..36af2e8 100644 --- a/driver.go +++ b/driver.go @@ -1,6 +1,7 @@ -// Package hotload is a database/sql driver that dynamically loads connection strings for other -// database drivers. To use it, import it like any other database driver and register -// the real database driver you want to use with hotload. +// Package hotload is a database/sql driver that dynamically loads connection +// strings for other database drivers. To use it, import it like any other +// database driver and register the real database driver you want to use with +// hotload. // // import ( // // import the std lib sql package @@ -9,10 +10,10 @@ // log "github.com/sirupsen/logrus" // // // this import registers hotload with the sql package -// "github.com/infobloxopen/hotload" +// "github.com/infobloxopen/hotload/v3" // // // this import registers the fsnotify hotload strategy -// _ "github.com/infobloxopen/hotload/fsnotify" +// _ "github.com/infobloxopen/hotload/v3/fsnotify" // // // this import registers the postgres driver with the sql package // "github.com/lib/pq" @@ -37,11 +38,13 @@ // * registers the lib/pq postgres driver with database/sql // * registers the lib/pq postgres driver with hotload // -// Then in the main() function the sql.Open call uses the hotload driver. The URL for the -// connection string specifies fsnotify in the scheme. This is the hotload strategy. The -// hostname in the URL specifies the real database driver. Finally the path and query parameters -// are left for the hotload strategy plugin to configure themselves. Below is an example -// of a lib/pq postgres connection string that would have been stored at /tmp/myconfig.txt +// Then in the main() function the sql.Open call uses the hotload driver. The +// URL for the connection string specifies fsnotify in the scheme. This is +// the hotload strategy. The hostname in the URL specifies the real database +// driver. Finally the path and query parameters are left for the hotload +// strategy plugin to configure themselves. Below is an example of a lib/pq +// postgres connection string that would have been stored at +// /tmp/myconfig.txt // // user=pqgotest dbname=pqgotest sslmode=verify-full package hotload @@ -56,9 +59,7 @@ import ( "sync" "time" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" + "github.com/infobloxopen/hotload/v3/logger" ) // Strategy is the plugin interface for hotload. @@ -75,14 +76,10 @@ type Strategy interface { Close() } -const forceKill = "forceKill" -const driverOptions = "driverOptions" +const forceKillParam = "forceKill" +const killWindowParam = "killWindow" var ( - ErrUnsupportedStrategy = fmt.Errorf("unsupported hotload strategy") - ErrMalformedConnectionString = fmt.Errorf("malformed hotload connection string") - ErrUnknownDriver = fmt.Errorf("target driver is not registered with hotload") - mu sync.RWMutex sqlDrivers = make(map[string]*driverInstance) strategies = make(map[string]Strategy) @@ -95,9 +92,10 @@ type driverInstance struct { type driverOption func(*driverInstance) -// WithDriverOptions allows you to specify query parameters to the underlying driver. -// The underlying driver must support URL style connection strings. The given options -// are appended to the connection string when a connection is opened. +// WithDriverOptions allows you to specify query parameters to the underlying +// driver. The underlying driver must support URL style connection strings. +// The given options are appended to the connection string when a connection +// is opened. func WithDriverOptions(options map[string]string) driverOption { return func(d *driverInstance) { if d.options == nil { @@ -110,8 +108,8 @@ func WithDriverOptions(options map[string]string) driverOption { } // RegisterSQLDriver makes a database driver available by the provided name. -// If RegisterSQLDriver is called twice with the same name or if driver is nil, -// it panics. +// If RegisterSQLDriver is called twice with the same name or if driver is +// nil, it panics. func RegisterSQLDriver(name string, driver driver.Driver, options ...driverOption) { mu.Lock() defer mu.Unlock() @@ -125,18 +123,9 @@ func RegisterSQLDriver(name string, driver driver.Driver, options ...driverOptio for _, opt := range options { opt(di) } - sqlDrivers[name] = di } -func unregisterAll() { - mu.Lock() - defer mu.Unlock() - // For tests. - sqlDrivers = make(map[string]*driverInstance) - strategies = make(map[string]Strategy) -} - // SQLDrivers returns a sorted list of the names of the registered drivers. func SQLDrivers() []string { mu.RLock() @@ -149,9 +138,9 @@ func SQLDrivers() []string { return list } -// RegisterStrategy makes a database driver available by the provided name. -// If RegisterStrategy is called twice with the same name or if strategy is nil, -// it panics. +// RegisterStrategy makes a hotload strategy available by the provided name. +// If RegisterStrategy is called twice with the same name or if strategy is +// nil, it panics. func RegisterStrategy(name string, strategy Strategy) { mu.Lock() defer mu.Unlock() @@ -179,7 +168,8 @@ func UnregisterStrategy(name string) { } } -// Strategies returns a sorted list of the names of the registered drivers. +// Strategies returns a sorted list of the names of the registered +// strategies. func Strategies() []string { mu.RLock() defer mu.RUnlock() @@ -192,305 +182,165 @@ func Strategies() []string { } func init() { - ctx := context.Background() - sql.Register("hotload", &hdriver{ + sql.Register("hotload", newHdriver(context.Background())) +} + +func newHdriver(ctx context.Context) *hdriver { + return &hdriver{ ctx: ctx, - cgroup: make(map[string]*chanGroup), - }) + groups: make(map[string]*group), + } } // hdriver is the hotload driver. type hdriver struct { ctx context.Context - cgroup map[string]*chanGroup mu sync.Mutex + groups map[string]*group } -// chanGroup represents a hotload location that is being monitored -type chanGroup struct { - name string - value string - redactVal string - newValChan <-chan string - parentCtx context.Context - ctx context.Context - cancel context.CancelFunc - sqlDriver *driverInstance - mu sync.RWMutex - forceKill bool - conns []*managedConn - prevCancel context.CancelFunc - prevRedactVal string - prevConns []*managedConn -} +var ( + _ driver.Driver = (*hdriver)(nil) + _ driver.DriverContext = (*hdriver)(nil) +) -// monitor the location for changes -func (cg *chanGroup) runLoop() { - for { - cg.logf("chanGroup.runLoop", "select waiting...") - select { - case <-cg.parentCtx.Done(): - cg.cancel() - cg.logf("chanGroup.runLoop", "parent context done, canceled chanGroup context, terminating") - return - - case newValue, ok := <-cg.newValChan: - if !ok { - cg.logf("chanGroup.runLoop", "newValChan closed, terminating") - return - } - cg.processNewValue(newValue) - } +// Open implements the legacy driver.Driver dial path. Groups created here +// are pinned: they have no teardown signal, so their strategy watch and run +// loop live for the life of the process (as in hotload v1). Prefer the +// connector path (database/sql uses it automatically), which tears the +// group down when the last sql.DB using it is closed. +func (h *hdriver) Open(name string) (driver.Conn, error) { + g, err := h.getGroup(name, true) + if err != nil { + return nil, err } + return g.conn(context.Background()) } -func (cg *chanGroup) processNewValue(newValue string) { - type oldInfo struct { - changedFlag bool - prevPrevCancel context.CancelFunc - prevPrevRedactVal string - prevPrevConns []*managedConn - prevCancel context.CancelFunc - prevRedactVal string - prevConns []*managedConn - } - - criticalSection := func() oldInfo { - cg.mu.Lock() - defer cg.mu.Unlock() - - prevValue := cg.value - prevRedactVal := cg.redactVal - - newRedactVal := internal.RedactUrl(newValue) - cg.logf("chanGroup.processNewValue", "old conn dsn: '%s'", prevRedactVal) - cg.logf("chanGroup.processNewValue", "new conn dsn: '%s'", newRedactVal) - - if newValue == prevValue { - // next update is the same, just ignore it - cg.logf("chanGroup.processNewValue", "conn dsn not changed") - return oldInfo{} - } - cg.logf("chanGroup.processNewValue", "conn dsn changed") - - result := oldInfo{ - changedFlag: true, - prevPrevConns: cg.prevConns, - prevPrevCancel: cg.prevCancel, - prevPrevRedactVal: cg.prevRedactVal, - } - - // Prepare shallow copy of existing connections, - // and reset new connections to zero - cg.prevConns = cg.conns - cg.conns = make([]*managedConn, 0) - - // Prepare copy of existing cancel ctx fn, - // and reset to new cancelable ctx - cg.prevCancel = cg.cancel - cg.ctx, cg.cancel = context.WithCancel(cg.parentCtx) - - // Prepare copy of existing value, - // and reset to new value - cg.prevRedactVal = cg.redactVal - cg.value = newValue - cg.redactVal = newRedactVal - - result.prevConns = cg.prevConns - result.prevCancel = cg.prevCancel - result.prevRedactVal = cg.prevRedactVal - - return result +// OpenConnector implements driver.DriverContext. The returned connector +// holds a reference on the group; database/sql closes the connector when +// the sql.DB is closed, and the group shuts down when its last reference is +// released. +func (h *hdriver) OpenConnector(name string) (driver.Connector, error) { + g, err := h.getGroup(name, false) + if err != nil { + return nil, err } + return &connector{h: h, g: g, name: name}, nil +} - prev := criticalSection() - if !prev.changedFlag { - return +// getGroup returns the group watching name, creating it (and its strategy +// watch) on first use. Groups are shared across sql.DB handles opened with +// the same DSN. +func (h *hdriver) getGroup(name string, pin bool) (*group, error) { + uri, err := url.Parse(name) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrMalformedConnectionString, err) } - // Mutex MUST be unlocked at this point before continuing - - // Update metrics - metrics.IncHotloadChangeTotal(cg.name) - metrics.SetHotloadLastChangedTimestampSeconds(cg.name, float64(time.Now().Unix())) + h.mu.Lock() + defer h.mu.Unlock() - // Canceling previous ctx can potentially cause other threads - // to call managedConn.Close(), which calls managedConn.afterClose(), - // which calls chanGroup.removeMgdConn(), which tries to lock mutex. - if cg.forceKill { - // Immediately cancel the previous dsn - if prev.prevCancel != nil { - prev.prevCancel() - cg.logf("chanGroup.processNewValue", "canceled context for previous dsn: '%s'", prev.prevRedactVal) + g, ok := h.groups[name] + if !ok { + mu.RLock() + strategy, okStrategy := strategies[uri.Scheme] + sqlDriver, okDriver := sqlDrivers[uri.Host] + mu.RUnlock() + if !okStrategy { + return nil, ErrUnsupportedStrategy } - } else { - // Immediately cancel the previous-previous dsn. - // We let the previous dsn to gracefully continue until the next dsn-change. - if prev.prevPrevCancel != nil { - prev.prevPrevCancel() - cg.logf("chanGroup.processNewValue", "canceled context for previous-previous dsn: '%s'", prev.prevPrevRedactVal) + if !okDriver { + return nil, ErrUnknownDriver } - } - // Yield to let other threads process cancel signal. - // Otherwise, there's a race and what happens (esp if forceKill=true) - // is that sometimes a db.Exec completes successfully (before cancel is processed), - // but db.Exec is later killed (closed) below because dsn changed, resulting in - // db.Exec returning error. This is inconsistent. - time.Sleep(1 * time.Millisecond) - - // Reset previous connections - // Mutex MUST NOT be held by this point, because in the same thread, - // we will call managedConn.Close() if forceKill is true, - // which calls managedConn.afterClose(), which calls chanGroup.removeMgdConn(), - // which tries to lock mutex. - if cg.forceKill { - // Immediately reset/close previous conns - cg.logf("chanGroup.processNewValue", "reset/close conns for previous dsn: '%s'", prev.prevRedactVal) - for _, c := range prev.prevConns { - c.Reset(true) - // ignore errors from close - c.Close() - } - } else { - // Immediately close previous-previous conns. - // We let the previous conns to gracefully continue until the next dsn-change. - cg.logf("chanGroup.processNewValue", "close conns for previous-previous dsn: '%s'", prev.prevPrevRedactVal) - for _, c := range prev.prevPrevConns { - // ignore errors from close - c.Close() + queryParams := uri.Query() + forceKill, killWindow, err := parseGroupParams(queryParams) + if err != nil { + return nil, err } - // Immediately reset (but do not close) previous conns. - // We let the previous conns to gracefully continue until the next dsn-change. - cg.logf("chanGroup.processNewValue", "reset conns for previous dsn: '%s'", prev.prevPrevRedactVal) - for _, c := range prev.prevConns { - c.Reset(true) + pathQry := queryParams.Encode() + value, newValChan, err := strategy.Watch(h.ctx, uri.Path, pathQry) + if err != nil { + return nil, err } - } -} -func mergeConnStringOptions(dsn string, options map[string]string) (string, error) { - if len(options) == 0 { - return dsn, nil - } - u, err := url.ParseRequestURI(dsn) - if err != nil { - return "", fmt.Errorf("unable to parse connection string when specifying extra driver options: %v", err) - } - values, err := url.ParseQuery(u.RawQuery) - if err != nil { - return "", fmt.Errorf("unable to parse query options in connection string when specifying extra driver options: %v", err) - } - for k, v := range options { - values.Set(k, v) + parentCtx, parentCancel := context.WithCancel(h.ctx) + g = &group{ + name: name, + strategyName: uri.Scheme, + strategy: strategy, + path: uri.Path, + pathQry: pathQry, + sqlDriver: sqlDriver, + forceKill: forceKill, + killWindow: killWindow, + parentCtx: parentCtx, + parentCancel: parentCancel, + newValChan: newValChan, + } + g.cur = newGeneration(parentCtx, value) + h.groups[name] = g + h.logf("hotload", "new group: '%s'", name) + EmitWatchEvent(WatchEvent{GroupName: name, Strategy: uri.Scheme, Path: uri.Path}) + go g.runLoop() } - u.RawQuery = values.Encode() - return u.String(), nil -} -func (cg *chanGroup) Open() (driver.Conn, error) { - cg.mu.Lock() - defer cg.mu.Unlock() - dsn, err := mergeConnStringOptions(cg.value, cg.sqlDriver.options) - if err != nil { - return nil, err - } - redactDsn := internal.RedactUrl(dsn) - conn, err := cg.sqlDriver.driver.Open(dsn) - if err != nil { - return conn, err + if pin { + g.pinned = true + } else { + g.refs++ } - - manConn := newManagedConn(cg.ctx, dsn, redactDsn, conn, cg.removeMgdConn) - cg.conns = append(cg.conns, manConn) - cg.logf("chanGroup.Open", "opened managed conn: '%s'", manConn.redactDsn) - - return manConn, nil + return g, nil } -func (cg *chanGroup) removeMgdConn(conn *managedConn) { - cg.mu.Lock() - defer cg.mu.Unlock() - for i, c := range cg.conns { - if c == conn { - cg.conns = append(cg.conns[:i], cg.conns[i+1:]...) - cg.logf("chanGroup.removeMgdConn", "%d: removed: '%s'", i, conn.redactDsn) - return +// releaseGroup drops one connector reference and shuts the group down when +// no references remain (unless a legacy Open pinned it). +func (h *hdriver) releaseGroup(name string) { + h.mu.Lock() + g, ok := h.groups[name] + if ok { + g.refs-- + if g.refs > 0 || g.pinned { + g = nil + } else { + delete(h.groups, name) } } -} + h.mu.Unlock() -func (cg *chanGroup) parseUrlValues(vs url.Values) { - cg.logf("chanGroup.parseUrlValues", "values: %s", vs) - v, ok := vs[forceKill] - if ok && len(v) > 0 { - firstValue := v[0] - cg.forceKill = firstValue == "true" - cg.logf("chanGroup.parseUrlValues", "forceKill set to true") + if g != nil { + g.shutdown() } } -func (h *hdriver) Open(name string) (driver.Conn, error) { - uri, err := url.Parse(name) - if err != nil { - return nil, err +func parseGroupParams(vs url.Values) (forceKill bool, killWindow time.Duration, err error) { + killWindow = DefaultKillWindow + if v, ok := vs[forceKillParam]; ok && len(v) > 0 { + forceKill = v[0] == "true" } - mu.Lock() - defer mu.Unlock() - - // look up in the chan group - cgroup, ok := h.cgroup[name] - if !ok { - strategy, ok := strategies[uri.Scheme] - if !ok { - return nil, ErrUnsupportedStrategy - } - sqlDriver, ok := sqlDrivers[uri.Host] - if !ok { - return nil, ErrUnknownDriver - } - queryParams := uri.Query() - value, newValChan, err := strategy.Watch(h.ctx, uri.Path, queryParams.Encode()) + if v, ok := vs[killWindowParam]; ok && len(v) > 0 { + killWindow, err = time.ParseDuration(v[0]) if err != nil { - return nil, err - } - ctx, cancel := context.WithCancel(h.ctx) - cgroup = &chanGroup{ - name: name, - value: value, - redactVal: internal.RedactUrl(value), - newValChan: newValChan, - parentCtx: h.ctx, - ctx: ctx, - cancel: cancel, - sqlDriver: sqlDriver, - conns: make([]*managedConn, 0), + return false, 0, fmt.Errorf("%w: invalid %s: %v", ErrMalformedConnectionString, killWindowParam, err) } - cgroup.parseUrlValues(queryParams) - h.cgroup[name] = cgroup - h.logf("hotload", "new chanGroup: '%s'", name) - go cgroup.runLoop() } - return cgroup.Open() + return forceKill, killWindow, nil } func (h *hdriver) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s:", prefix) - logger.Logf(logPrefix, format, args...) -} - -func (cg *chanGroup) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, cg.name) - logger.Logf(logPrefix, format, args...) + logger.Logf(fmt.Sprintf("%s:", prefix), format, args...) } -// Deprecated: Use logger.WithLogger() instead, retained for backwards-compatibility only +// Deprecated: Use logger.WithLogger() instead, retained for +// backwards-compatibility only. func WithLogger(l logger.Logger) { logger.WithLogger(l) } -// Deprecated: Use logger.GetLogger() instead, retained for backwards-compatibility only +// Deprecated: Use logger.GetLogger() instead, retained for +// backwards-compatibility only. func GetLogger() logger.Logger { return logger.GetLogger() } diff --git a/driver_internal_test.go b/driver_internal_test.go index 10864d2..182ed66 100644 --- a/driver_internal_test.go +++ b/driver_internal_test.go @@ -1,131 +1,234 @@ package hotload import ( + "context" "database/sql/driver" "fmt" - "reflect" + "net/url" "testing" -) - -type testDriver struct { - options map[string]string -} + "time" -func (d *testDriver) Open(name string) (driver.Conn, error) { - return nil, fmt.Errorf("not implemented") -} - -func withConnectionStringOptions(options map[string]string) driverOption { - return func(di *driverInstance) { - di.options = options - } -} - -func TestRegisterSQLDriverWithOptions(t *testing.T) { - type args struct { - name string - driver driver.Driver - options []driverOption - } - tests := []struct { - name string - args args - }{ - { - name: "driver with an option", - args: args{ - name: "test with options", - driver: &testDriver{}, - options: []driverOption{ - withConnectionStringOptions(map[string]string{"a": "b"}), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - RegisterSQLDriver(tt.args.name, tt.args.driver, tt.args.options...) - mu.Lock() - defer mu.Unlock() - - d, ok := sqlDrivers[tt.args.name] - if !ok { - t.Errorf("RegisterSQLDriver() did not register the driver") - } - gotOptions := d.driver.(*testDriver).options - if reflect.DeepEqual(gotOptions, tt.args.options) { - t.Errorf("RegisterSQLDriver() did not set the options") - } - }) - } -} + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) func Test_mergeConnStringOptions(t *testing.T) { - type args struct { - dsn string - options map[string]string - } tests := []struct { name string - args args + dsn string + options map[string]string want string wantErr bool }{ { name: "empty", - args: args{ - dsn: "", - options: nil, - }, - want: "", - wantErr: false, + want: "", }, { name: "bad dsn with no options", - args: args{ - dsn: "bad dsn", - options: nil, - }, - want: "bad dsn", - wantErr: false, + dsn: "bad dsn", + want: "bad dsn", }, { - name: "bad dsn with options", - args: args{ - dsn: "bad dsn", - options: map[string]string{"a": "b"}, - }, - want: "", + name: "bad dsn with options", + dsn: "bad dsn", + options: map[string]string{"a": "b"}, wantErr: true, }, { name: "good dsn with no options", - args: args{ - dsn: "postgres://localhost:5432/postgres?sslmode=disable", - }, - want: "postgres://localhost:5432/postgres?sslmode=disable", - wantErr: false, + dsn: "postgres://localhost:5432/postgres?sslmode=disable", + want: "postgres://localhost:5432/postgres?sslmode=disable", }, { - name: "good dsn with options", - args: args{ - dsn: "postgres://localhost:5432/postgres?sslmode=disable", - options: map[string]string{"disable_cache": "true"}, - }, + name: "good dsn with options", + dsn: "postgres://localhost:5432/postgres?sslmode=disable", + options: map[string]string{"disable_cache": "true"}, want: "postgres://localhost:5432/postgres?disable_cache=true&sslmode=disable", - wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := mergeConnStringOptions(tt.args.dsn, tt.args.options) + got, err := mergeConnStringOptions(tt.dsn, tt.options) if (err != nil) != tt.wantErr { - t.Errorf("mergeConnStringOptions() error = %v, wantErr %v", err, tt.wantErr) - return + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) } if got != tt.want { - t.Errorf("mergeConnStringOptions() = %v, want %v", got, tt.want) + t.Errorf("got %q, want %q", got, tt.want) } }) } } + +func Test_parseGroupParams(t *testing.T) { + tests := []struct { + name string + query string + wantForceKill bool + wantKillWindow time.Duration + wantErr bool + }{ + {name: "defaults", query: "", wantKillWindow: DefaultKillWindow}, + {name: "forceKill true", query: "forceKill=true", wantForceKill: true, wantKillWindow: DefaultKillWindow}, + {name: "forceKill false", query: "forceKill=false", wantKillWindow: DefaultKillWindow}, + {name: "forceKill garbage", query: "forceKill=yes", wantKillWindow: DefaultKillWindow}, + {name: "killWindow", query: "killWindow=250ms", wantKillWindow: 250 * time.Millisecond}, + {name: "killWindow invalid", query: "killWindow=bogus", wantErr: true}, + {name: "both", query: "forceKill=true&killWindow=1s", wantForceKill: true, wantKillWindow: time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vs, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatal(err) + } + forceKill, killWindow, err := parseGroupParams(vs) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if forceKill != tt.wantForceKill { + t.Errorf("forceKill = %v, want %v", forceKill, tt.wantForceKill) + } + if killWindow != tt.wantKillWindow { + t.Errorf("killWindow = %v, want %v", killWindow, tt.wantKillWindow) + } + }) + } +} + +func TestRegisterSQLDriverWithOptions(t *testing.T) { + name := "internal-test-driver-options" + RegisterSQLDriver(name, &dbfake.Driver{}, WithDriverOptions(map[string]string{"a": "b"})) + + mu.RLock() + defer mu.RUnlock() + di, ok := sqlDrivers[name] + if !ok { + t.Fatal("RegisterSQLDriver did not register the driver") + } + if di.options["a"] != "b" { + t.Errorf("options = %v, want a=b", di.options) + } +} + +func TestRegisterPanics(t *testing.T) { + mustPanic := func(name string, fn func()) { + t.Run(name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("expected panic") + } + }() + fn() + }) + } + + mustPanic("nil driver", func() { RegisterSQLDriver("internal-test-nil", nil) }) + mustPanic("dup driver", func() { + RegisterSQLDriver("internal-test-dup", &dbfake.Driver{}) + RegisterSQLDriver("internal-test-dup", &dbfake.Driver{}) + }) + mustPanic("nil strategy", func() { RegisterStrategy("internal-test-nilstrat", nil) }) + mustPanic("dup strategy", func() { + s := dbfake.NewStrategy(nil) + RegisterStrategy("internal-test-dupstrat", s) + defer UnregisterStrategy("internal-test-dupstrat") + RegisterStrategy("internal-test-dupstrat", s) + }) +} + +func TestRegistryLists(t *testing.T) { + RegisterSQLDriver("internal-test-list-b", &dbfake.Driver{}) + RegisterSQLDriver("internal-test-list-a", &dbfake.Driver{}) + RegisterStrategy("internal-test-list-strat", dbfake.NewStrategy(nil)) + defer UnregisterStrategy("internal-test-list-strat") + + drivers := SQLDrivers() + prev := "" + seen := 0 + for _, d := range drivers { + if d < prev { + t.Errorf("SQLDrivers not sorted: %v", drivers) + } + prev = d + if d == "internal-test-list-a" || d == "internal-test-list-b" { + seen++ + } + } + if seen != 2 { + t.Errorf("registered drivers missing from SQLDrivers(): %v", drivers) + } + + found := false + for _, s := range Strategies() { + if s == "internal-test-list-strat" { + found = true + } + } + if !found { + t.Errorf("registered strategy missing from Strategies(): %v", Strategies()) + } +} + +// TestLegacyOpenPath drives the non-connector driver.Open path directly +// against a private hdriver instance (the path database/sql no longer uses, +// but third-party pools might). The group is pinned, so teardown happens via +// the parent context. +func TestLegacyOpenPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h := newHdriver(ctx) + drv := &dbfake.Driver{Caps: dbfake.CapsModern} + strat := dbfake.NewStrategy(map[string]string{"/cfg": "dsn-1"}) + RegisterSQLDriver("internal-test-legacy-drv", drv) + RegisterStrategy("internal-test-legacy-strat", strat) + defer UnregisterStrategy("internal-test-legacy-strat") + + name := "internal-test-legacy-strat://internal-test-legacy-drv/cfg" + conn, err := h.Open(name) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if _, ok := conn.(driver.QueryerContext); !ok { + t.Error("legacy-opened conn should expose QueryerContext for a modern underlying conn") + } + + // A second open shares the group (one watch). + conn2, err := h.Open(name) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + if n := strat.Watches(); n != 1 { + t.Errorf("watches = %d, want 1", n) + } +} + +// TestEmitHelpersNilSafe ensures hooks with nil fields are skipped. +func TestEmitHelpersNilSafe(t *testing.T) { + resetHooks() + defer resetHooks() + RegisterHooks(Hooks{}) // all nil + + emitConfigChange(ConfigChangeEvent{}) + emitConnOpen(ConnEvent{}) + emitConnClose(ConnEvent{}) + emitTxComplete(TxEvent{}) + EmitWatchEvent(WatchEvent{}) + EmitModTimeEvent(ModTimeEvent{}) +} + +func ExampleContextWithExecLabels() { + ctx := ContextWithExecLabels(context.Background(), map[string]string{ + "grpc_service": "ContactsService", + "grpc_method": "ListContacts", + }) + labels := GetExecLabelsFromContext(ctx) + fmt.Println(labels["grpc_service"], labels["grpc_method"]) + // Output: ContactsService ListContacts +} diff --git a/driver_test.go b/driver_test.go deleted file mode 100644 index 29980e2..0000000 --- a/driver_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package hotload_test - -import ( - "database/sql" - "database/sql/driver" - "os" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/infobloxopen/hotload" - "github.com/infobloxopen/hotload/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func getDriverFromSqlMock() driver.Driver { - littleBuddy, mock, _ := sqlmock.NewWithDSN("user=pqgotest dbname=pqgotest sslmode=verify-full") - mockDriver = mock - return littleBuddy.Driver() -} - -func getRandomDriver() driver.Driver { - db, _, _ := sqlmock.New() - return db.Driver() -} - -var mockDriver sqlmock.Sqlmock -var configFile string -var configFileDir string - -var _ = BeforeSuite(func() { - driver := getDriverFromSqlMock() - - if driver == nil { - Fail("driver is nil, boo!") - } - - hotload.RegisterSQLDriver("sqlmock", driver) - Expect(hotload.SQLDrivers()).To(ContainElement("sqlmock")) - var err error - configFile, err = os.Getwd() - Expect(err).ToNot(HaveOccurred()) - configFileDir = configFile + "/testdata/" - configFile += "/testdata/myconfig.txt" -}) - -var _ = Describe("Driver", func() { - Context("RegisterSQLDriver", func() { - It("Should panic when registering the same driver twice", func() { - driver := getRandomDriver() - Expect(func() { hotload.RegisterSQLDriver("sqlmock", driver) }). - To(PanicWith(MatchRegexp("Register called twice for driver"))) - }) - - It("Should panic on nil driver", func() { - Expect(func() { hotload.RegisterSQLDriver("", nil) }). - To(PanicWith(MatchRegexp("Register driver is nil"))) - }) - }) - - Context("RegisterStrategy", func() { - It("Should panic when registering the same strategy twice", func() { - strat := fsnotify.NewStrategy() - Expect(func() { hotload.RegisterStrategy("fsnotify", strat) }). - To(PanicWith(MatchRegexp("RegisterStrategy called twice for strategy"))) - }) - - It("Should panic on nil driver", func() { - Expect(func() { hotload.RegisterStrategy("", nil) }). - To(PanicWith(MatchRegexp("strategy is nil"))) - }) - }) - - Context("Open", func() { - It("Should throw an error with unknown driver", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmaybe?"+configFile) - Expect(err).ToNot(HaveOccurred()) - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(hotload.ErrUnknownDriver)) - - }) - - It("Should not throw an error with a registered driver and strategy", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmock"+configFile) - Expect(err).ToNot(HaveOccurred()) - - Expect(db.Ping()).ToNot(HaveOccurred()) - }) - - It("Should throw an unsupported strategy error", func() { - db, err := sql.Open("hotload", "fstransmogrify://sqlmock/"+configFile) - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(hotload.ErrUnsupportedStrategy)) - }) - - It("Should throw an error if it can't find the config file", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmock/temple/run/2021-edition") - err = db.Ping() - Expect(err).To(HaveOccurred()) - }) - - It("Should throw an error the url is unparseable", func() { - db, err := sql.Open("hotload", "://") - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("missing protocol scheme")) - }) - - //It("Should close my connection when the connection information changes", func() { - // db, err := sql.Open("hotload", "fsnotify://sqlmock"+configFileDir+"urconfig.txt") - // Expect(err).ToNot(HaveOccurred()) - // - // Expect(db.Ping()).ToNot(HaveOccurred()) - // // Open dat - // // Do a thing - // // change connection file - // mockDriver.ExpectBegin() - // mockDriver.ExpectExec("SELECT 1") - // mockDriver.ExpectCommit() - // tx, err := db.Begin() - // Expect(err).ToNot(HaveOccurred()) - // tx.Exec("SELECT 1") - // err = ioutil.WriteFile(configFileDir+"urconfig.txt", []byte("user=pqgotest dbname=pqgotestorooni sslmode=verify-full"), 0644) - // Expect(err).ToNot(HaveOccurred()) - // go func () { - // tx.Commit() - // Expect(mockDriver.ExpectationsWereMet()).ToNot(HaveOccurred()) - // }() - //}) - }) -}) diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..45966f9 --- /dev/null +++ b/errors.go @@ -0,0 +1,22 @@ +package hotload + +import "errors" + +var ( + // ErrUnsupportedStrategy is returned when the DSN names a strategy that + // has not been registered with RegisterStrategy. + ErrUnsupportedStrategy = errors.New("unsupported hotload strategy") + + // ErrMalformedConnectionString is returned when the hotload DSN cannot + // be parsed. + ErrMalformedConnectionString = errors.New("malformed hotload connection string") + + // ErrUnknownDriver is returned when the DSN names an underlying driver + // that has not been registered with RegisterSQLDriver. + ErrUnknownDriver = errors.New("target driver is not registered with hotload") + + // ErrHotSwap is the cancellation cause used when hotload retires a + // connection because the connection string changed. Callers can detect + // it with context.Cause and errors.Is. + ErrHotSwap = errors.New("hotload: connection retired by config change") +) diff --git a/export_test.go b/export_test.go new file mode 100644 index 0000000..7f740ac --- /dev/null +++ b/export_test.go @@ -0,0 +1,6 @@ +package hotload + +// ResetHooks removes all registered hooks. Exported to the external test +// package only; tests register per-test hooks and must not see hooks from +// earlier tests. +var ResetHooks = resetHooks diff --git a/fallback_test.go b/fallback_test.go new file mode 100644 index 0000000..956c464 --- /dev/null +++ b/fallback_test.go @@ -0,0 +1,229 @@ +package hotload_test + +import ( + "context" + "database/sql" + "strings" + "testing" + + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// methodsCalled returns the set of methods recorded by the fake driver. +func methodsCalled(drv *dbfake.Driver) map[string]int { + out := map[string]int{} + for _, c := range drv.Log.Calls() { + out[c.Method]++ + } + return out +} + +func assertCalled(t *testing.T, drv *dbfake.Driver, method string) { + t.Helper() + if drv.Log.Count(method) == 0 { + t.Errorf("expected %s to be called; calls: %v", method, methodsCalled(drv)) + } +} + +func assertNotCalled(t *testing.T, drv *dbfake.Driver, method string) { + t.Helper() + if n := drv.Log.Count(method); n > 0 { + t.Errorf("expected %s not to be called, got %d calls; calls: %v", method, n, methodsCalled(drv)) + } +} + +// TestExecDispatch verifies that db.Exec reaches the underlying conn through +// the path matching its capabilities — and, crucially, that a conn with no +// exec support makes database/sql fall back to a prepared statement instead +// of bouncing off driver.ErrSkip as hotload v1 did. +func TestExecDispatch(t *testing.T) { + t.Run("ExecerContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapExecerContext, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "ExecContext") + assertNotCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("legacy Execer", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapExecer, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "ExecContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("no exec support falls back to prepared stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Prepare") + assertCalled(t, fx.drv, "StmtExec") + assertNotCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "ExecContext") + }) +} + +// TestQueryDispatch is the query-side analog of TestExecDispatch. +func TestQueryDispatch(t *testing.T) { + t.Run("QueryerContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapQueryerContext, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "QueryContext") + assertNotCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("legacy Queryer", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapQueryer, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "QueryContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("no query support falls back to prepared stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "Prepare") + assertCalled(t, fx.drv, "StmtQuery") + assertNotCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "QueryContext") + }) +} + +// TestPrepareDispatch verifies PrepareContext synthesis: with underlying +// support it is delegated; without it hotload falls back to Prepare exactly +// like database/sql would. +func TestPrepareDispatch(t *testing.T) { + t.Run("with ConnPrepareContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + stmt, err := fx.db.Prepare("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + assertCalled(t, fx.drv, "PrepareContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("without ConnPrepareContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + stmt, err := fx.db.Prepare("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + assertCalled(t, fx.drv, "Prepare") + assertNotCalled(t, fx.drv, "PrepareContext") + }) +} + +// TestStmtDispatch verifies prepared statements run through the stmt +// wrappers' context methods when supported and the legacy methods when not. +func TestStmtDispatch(t *testing.T) { + t.Run("modern stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + stmt, err := fx.db.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "StmtExecContext") + assertNotCalled(t, fx.drv, "StmtExec") + }) + + t.Run("legacy stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + stmt, err := fx.db.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "StmtExec") + assertNotCalled(t, fx.drv, "StmtExecContext") + }) +} + +// TestPingDispatch: with no underlying Pinger, db.Ping must succeed without +// reaching the driver (database/sql treats absence as "no health check"). +func TestPingDispatch(t *testing.T) { + t.Run("with Pinger", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapPinger, rawCaps: true}) + if err := fx.db.Ping(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Ping") + }) + + t.Run("without Pinger", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if err := fx.db.Ping(); err != nil { + t.Fatal(err) + } + assertNotCalled(t, fx.drv, "Ping") + }) +} + +// TestBeginTxFallback verifies the ConnBeginTx synthesis: delegation when +// supported, the exact database/sql fallback errors when not. +func TestBeginTxFallback(t *testing.T) { + ctx := context.Background() + + t.Run("with ConnBeginTx", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + tx, err := fx.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) + if err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "BeginTx") + }) + + t.Run("without ConnBeginTx default options", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Begin") + }) + + t.Run("without ConnBeginTx non-default isolation", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + _, err := fx.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) + if err == nil || !strings.Contains(err.Error(), "non-default isolation level") { + t.Fatalf("BeginTx error = %v, want non-default isolation level error", err) + } + }) + + t.Run("without ConnBeginTx read-only", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + _, err := fx.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err == nil || !strings.Contains(err.Error(), "read-only") { + t.Fatalf("BeginTx error = %v, want read-only error", err) + } + }) +} diff --git a/fixture_test.go b/fixture_test.go new file mode 100644 index 0000000..bdd909e --- /dev/null +++ b/fixture_test.go @@ -0,0 +1,158 @@ +package hotload_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// fixtureSeq makes registered driver/strategy names unique per fixture; +// hotload registries are global and panic on duplicates. +var fixtureSeq atomic.Int64 + +const fixturePath = "/cfg" + +type fxCfg struct { + caps dbfake.Caps // dbfake.CapsModern if zero and rawCaps unset + rawCaps bool // use caps even if zero (a bare conn) + params string // hotload DSN query params, e.g. "forceKill=true" + initial string // initial underlying DSN; default "dsn-1" + execFn func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) + queryFn func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) + noDB bool // register only; do not sql.Open +} + +type fixture struct { + t *testing.T + db *sql.DB + drv *dbfake.Driver + strat *dbfake.Strategy + dsn string // the hotload DSN + changes chan hotload.ConfigChangeEvent +} + +// newFixture registers a fake driver and strategy under unique names, opens +// a hotload sql.DB on them, and subscribes to config-change events. Hooks +// are reset so events from earlier tests cannot interfere; tests using +// fixtures must not run in parallel. +func newFixture(t *testing.T, cfg fxCfg) *fixture { + t.Helper() + + caps := cfg.caps + if caps == 0 && !cfg.rawCaps { + caps = dbfake.CapsModern + } + initial := cfg.initial + if initial == "" { + initial = "dsn-1" + } + + n := fixtureSeq.Add(1) + stratName := fmt.Sprintf("fakestrat%d", n) + drvName := fmt.Sprintf("fakedrv%d", n) + + fx := &fixture{ + t: t, + drv: &dbfake.Driver{Caps: caps, ExecFn: cfg.execFn, QueryFn: cfg.queryFn}, + strat: dbfake.NewStrategy(map[string]string{fixturePath: initial}), + changes: make(chan hotload.ConfigChangeEvent, 100), + } + + hotload.ResetHooks() + t.Cleanup(hotload.ResetHooks) + hotload.RegisterHooks(hotload.Hooks{ + OnConfigChange: func(ev hotload.ConfigChangeEvent) { + select { + case fx.changes <- ev: + default: + } + }, + }) + + hotload.RegisterSQLDriver(drvName, fx.drv) + hotload.RegisterStrategy(stratName, fx.strat) + t.Cleanup(func() { hotload.UnregisterStrategy(stratName) }) + + fx.dsn = stratName + "://" + drvName + fixturePath + if cfg.params != "" { + fx.dsn += "?" + cfg.params + } + + if !cfg.noDB { + db, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatalf("sql.Open(%q): %v", fx.dsn, err) + } + t.Cleanup(func() { db.Close() }) + fx.db = db + } + return fx +} + +// queryDSN reports which underlying DSN served the query — the fake's +// default result set is one row holding the conn's DSN. +func (fx *fixture) queryDSN() string { + fx.t.Helper() + var dsn string + if err := fx.db.QueryRow("SELECT dsn").Scan(&dsn); err != nil { + fx.t.Fatalf("queryDSN: %v", err) + } + return dsn +} + +// push delivers a new value to the strategy watcher; it returns once the +// group run loop has received (but not necessarily processed) it. +func (fx *fixture) push(value string) { + fx.t.Helper() + fx.strat.Push(fixturePath, value) +} + +// pushAndWait pushes a changed value and waits for the resulting +// config-change event. Connection retirement happens after the event fires, +// so tests must still poll for conn-state assertions. +func (fx *fixture) pushAndWait(value string) hotload.ConfigChangeEvent { + fx.t.Helper() + fx.push(value) + select { + case ev := <-fx.changes: + return ev + case <-time.After(5 * time.Second): + fx.t.Fatalf("timed out waiting for config change event after pushing %q", value) + return hotload.ConfigChangeEvent{} + } +} + +// noPendingChange asserts no config-change event is buffered. +func (fx *fixture) noPendingChange() { + fx.t.Helper() + select { + case ev := <-fx.changes: + fx.t.Fatalf("unexpected config change event: %+v", ev) + default: + } +} + +// rawConn hands the wrapped driver.Conn of a pooled connection to fn. +func (fx *fixture) rawConn(fn func(dc driver.Conn)) { + fx.t.Helper() + ctx := context.Background() + conn, err := fx.db.Conn(ctx) + if err != nil { + fx.t.Fatalf("db.Conn: %v", err) + } + defer conn.Close() + err = conn.Raw(func(dc any) error { + fn(dc.(driver.Conn)) + return nil + }) + if err != nil { + fx.t.Fatalf("conn.Raw: %v", err) + } +} diff --git a/fsnotify/filewatcher.go b/fsnotify/filewatcher.go index 46cf159..f3ca28e 100644 --- a/fsnotify/filewatcher.go +++ b/fsnotify/filewatcher.go @@ -2,6 +2,7 @@ package fsnotify import ( "context" + "errors" "fmt" "os" "path" @@ -10,11 +11,9 @@ import ( "time" rfsnotify "github.com/fsnotify/fsnotify" - "github.com/infobloxopen/hotload" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" - "github.com/pkg/errors" + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/logger" ) func init() { @@ -64,7 +63,7 @@ type pathWatch struct { func (s *Strategy) readConfigFile(path string) (v []byte, err error) { v, err = os.ReadFile(path) if err != nil { - return nil, errors.Wrapf(err, "could not read %v", path) + return nil, fmt.Errorf("could not read %v: %w", path, err) } v = []byte(strings.TrimSpace(string(v))) return @@ -175,12 +174,6 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value if err := s.watcher.Add(pth); err != nil { return "", nil, err } - if err := metrics.AddToDefaultPathChksum(pth); err != nil { - if err != metrics.ErrDuplicatePath { - s.errlogf("fsnotify.Watch", "AddToDefaultPathChksum(%s) failed, err=%v", pth, err) - return "", nil, err - } - } bs, err := s.readConfigFile(pth) if err != nil { s.watcher.Remove(pth) diff --git a/fsnotify/filewatcher_test.go b/fsnotify/filewatcher_test.go deleted file mode 100644 index 2f9e2ca..0000000 --- a/fsnotify/filewatcher_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package fsnotify - -import ( - "context" - "fmt" - "net/url" - "os" - "time" - - rfsnotify "github.com/fsnotify/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func assertStringFromChannel(name string, want string, from <-chan string) { -again: - select { - case got := <-from: - fmt.Printf("assertStringFromChannel: expecting '%s', got update: '%s'\n", want, got) - if got == want { - return - } else { - // dedup fsnotify WRITE events - goto again - } - case <-time.After(resyncPeriod * 2): - Fail(fmt.Sprintf("%s: timeout: expecting '%s'", name, want)) - } -} - -type args struct { - pth string - options url.Values -} - -type test struct { - name string - setup func(*args) - args args - wantErr bool - post func(args *args, value string, values <-chan string) error - tearDown func(*args) -} - -type testWatcher struct { - eventChannel chan rfsnotify.Event - paths map[string]bool - closed bool - errors chan error -} - -func newTestWatcher() *testWatcher { - return &testWatcher{ - eventChannel: make(chan rfsnotify.Event), - paths: make(map[string]bool), - errors: make(chan error), - closed: false, - } -} - -func (tw *testWatcher) Add(s string) error { - tw.paths[s] = true - return nil -} - -func (tw *testWatcher) Remove(s string) error { - tw.paths[s] = false - return nil -} - -func (tw *testWatcher) Close() error { - tw.closed = true - return nil -} - -func (tw *testWatcher) GetEvents() <-chan rfsnotify.Event { - return tw.eventChannel -} - -func (tw *testWatcher) GetErrors() <-chan error { - return tw.errors -} - -var _ = Describe("FileWatcher", func() { - const ( - paramsURL = "postgres://login:password@host:1234/database?sslmode=disable" - paramsParsed = "host=a login=b password=c" - ) - - s := NewStrategy() - DescribeTable("Watch", - func(tt test) { - if tt.setup != nil { - tt.setup(&tt.args) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - gotValue, gotValues, err := s.Watch(ctx, tt.args.pth, tt.args.options.Encode()) - if (err != nil) != tt.wantErr { - Expect(err).To(HaveOccurred()) - return - } - if tt.post != nil { - if err := tt.post(&tt.args, gotValue, gotValues); err != nil { - Expect(err).ToNot(HaveOccurred()) - } - } - if tt.tearDown != nil { - tt.tearDown(&tt.args) - } - - _ = s.CloseWatch(tt.args.pth, tt.args.options.Encode()) - time.Sleep(10 * time.Millisecond) - }, - Entry("file not found", test{ - args: args{ - pth: "somefile does not exist", - }, - wantErr: true, - }), - Entry("URL surrounded with whitespaces --> URL trimmed", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("\r\n \t " + paramsURL + " \t \r\n")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != paramsURL { - return fmt.Errorf("expected '"+paramsURL+"' got %v", value) - } - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("params surrounded with whitespaces --> params trimmed", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("\r\n \t " + paramsParsed + " \t \r\n")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != paramsParsed { - return fmt.Errorf("expected '"+paramsParsed+"' got %v", value) - } - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("a, update b", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - os.WriteFile(args.pth, []byte("b"), 0660) - assertStringFromChannel("waiting for update b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("extra slash in path", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = "/" + f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - os.WriteFile(args.pth, []byte("b"), 0660) - assertStringFromChannel("waiting for update b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("a, rm a, create b", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - err := os.Remove(args.pth) - Expect(err).ToNot(HaveOccurred(), "removing config file") - - select { - case v := <-values: - return fmt.Errorf("expected no change, got %v", v) - case <-time.After(time.Second): - } - err = os.WriteFile(args.pth, []byte("b"), 0660) - - Expect(err).ToNot(HaveOccurred(), "creating new file") - - assertStringFromChannel("waiting for create b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - ) - - Context("run", func() { - var strat *Strategy - var watcher *testWatcher - BeforeEach(func() { - strat = NewStrategy() - watcher = newTestWatcher() - strat.watcher = watcher - }) - It("Should not respond to chmod events", func() { - // add only a bad path to the testWatcher - // This path should not end up removed from the map, ie, marked 'false' - // we'll pass a CHMOD event and verify the 'bad path' is still in the paths map - bp := "badpath" - strat.watcher.Add(bp) - go s.runLoop() - go func() { - watcher.eventChannel <- rfsnotify.Event{ - Name: "chaff", - Op: rfsnotify.Chmod, - } - }() - time.Sleep(1 * time.Millisecond) - _, v := watcher.paths[bp] - // run didn't pass through resync - Expect(v).To(BeTrue()) - }) - }) - - Context("Two watches with same path but diff query-params", func() { - It("Should update channels for both watches", func() { - f, _ := os.CreateTemp("", "hotload_fsnotify_filewatcher_two_watches_unittest_") - f.Write([]byte("_")) - watchPath := f.Name() - f.Close() - defer os.Remove(watchPath) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - gotValue1, updateChan1, err := s.Watch(ctx, watchPath, "") - Expect(err).ToNot(HaveOccurred()) - Expect(updateChan1).ToNot(BeNil()) - Expect(gotValue1).To(Equal("_")) - - urlValues := url.Values{"forceKill": []string{"true"}} - gotValue2, updateChan2, err := s.Watch(ctx, watchPath, urlValues.Encode()) - Expect(err).ToNot(HaveOccurred()) - Expect(updateChan2).ToNot(BeNil()) - Expect(gotValue2).To(Equal("_")) - - os.WriteFile(watchPath, []byte("a"), 0666) - assertStringFromChannel("updateChan1 waiting for update a", "a", updateChan1) - assertStringFromChannel("updateChan2 waiting for update a", "a", updateChan2) - - os.WriteFile(watchPath, []byte("b"), 0666) - assertStringFromChannel("updateChan1 waiting for update b", "b", updateChan1) - assertStringFromChannel("updateChan2 waiting for update b", "b", updateChan2) - - _ = s.CloseWatch(watchPath, "") - _ = s.CloseWatch(watchPath, urlValues.Encode()) - time.Sleep(10 * time.Millisecond) - }) - }) -}) diff --git a/fsnotify/fsnotify_suite_test.go b/fsnotify/fsnotify_suite_test.go deleted file mode 100644 index 010fed9..0000000 --- a/fsnotify/fsnotify_suite_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package fsnotify - -import ( - "log" - "testing" - - "github.com/infobloxopen/hotload/logger" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestFsnotify(t *testing.T) { - //log.SetFlags(log.Flags() | log.Lmicroseconds) - log.SetFlags(log.Ltime | log.Lmicroseconds) - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - RegisterFailHandler(Fail) - RunSpecs(t, "Fsnotify Suite") -} diff --git a/go.mod b/go.mod index 524ba35..e8799b3 100644 --- a/go.mod +++ b/go.mod @@ -1,41 +1,7 @@ -module github.com/infobloxopen/hotload +module github.com/infobloxopen/hotload/v3 go 1.23.0 -toolchain go1.24.6 +require github.com/fsnotify/fsnotify v1.6.0 -require ( - github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/colega/gaugefuncvec v0.1.0 - github.com/fsnotify/fsnotify v1.6.0 - github.com/google/uuid v1.6.0 - github.com/lib/pq v1.10.8 - github.com/onsi/ginkgo/v2 v2.25.3 - github.com/onsi/gomega v1.38.2 - github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.0 - github.com/prometheus/common v0.55.0 - github.com/teivah/onecontext v1.3.0 -) - -require ( - github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.36.0 // indirect - google.golang.org/protobuf v1.36.7 // indirect -) +require golang.org/x/sys v0.35.0 // indirect diff --git a/go.sum b/go.sum index 9500385..d67f982 100644 --- a/go.sum +++ b/go.sum @@ -1,167 +1,5 @@ -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/colega/gaugefuncvec v0.1.0 h1:ocD7PAmGxioCVM6mlYvbSpwB+rYCcvgdFbEuLfNm35Y= -github.com/colega/gaugefuncvec v0.1.0/go.mod h1:uXDw0W/fhrHeWsaRNEJgtJL7hYa+vFZ5oQH5MTN6+nE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= -github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= -github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= -github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= -github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= -github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/teivah/onecontext v1.3.0 h1:tbikMhAlo6VhAuEGCvhc8HlTnpX4xTNPTOseWuhO1J0= -github.com/teivah/onecontext v1.3.0/go.mod h1:hoW1nmdPVK/0jrvGtcx8sCKYs2PiS4z0zzfdeuEVyb0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/group.go b/group.go new file mode 100644 index 0000000..e61dbdc --- /dev/null +++ b/group.go @@ -0,0 +1,308 @@ +package hotload + +import ( + "context" + "database/sql/driver" + "fmt" + "net/url" + "sync" + "sync/atomic" + "time" + + "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/logger" +) + +// DefaultKillWindow is how long a killed generation waits for in-flight +// operations to observe cancellation before their connections are +// force-closed. Override per DSN with the killWindow query parameter. +const DefaultKillWindow = 100 * time.Millisecond + +// generation owns everything whose lifetime matches one value of the +// connection string: the cancellation context, the set of live connections, +// and the drained flag consulted by ResetSession/IsValid. A config change +// swaps generation pointers on the group; nothing else has to be mutated. +type generation struct { + dsn string + redactDsn string + + // ctx is derived from the group's parent context and canceled with + // cause ErrHotSwap when the generation is killed. + ctx context.Context + cancel context.CancelCauseFunc + + // drained marks the generation retired without canceling in-flight + // work; conns answer driver.ErrBadConn on the next pool reuse. + drained atomic.Bool + + // ops counts operations running under opCtx, so kill can wait + // (bounded) for them to observe cancellation before force-closing. + // A plain atomic (rather than a WaitGroup) because operations keep + // starting while kill waits, which WaitGroup forbids. + ops atomic.Int64 + + // mu is a leaf lock guarding conns only. No driver call, hook, or + // other lock acquisition ever happens while holding it. + mu sync.Mutex + conns map[*baseConn]struct{} + + // connector lazily caches the underlying driver's connector when it + // implements driver.DriverContext. + connOnce sync.Once + connector driver.Connector + connErr error +} + +func newGeneration(parent context.Context, dsn string) *generation { + ctx, cancel := context.WithCancelCause(parent) + return &generation{ + dsn: dsn, + redactDsn: internal.RedactUrl(dsn), + ctx: ctx, + cancel: cancel, + conns: make(map[*baseConn]struct{}), + } +} + +// retired reports whether conns of this generation should be discarded by +// the pool. +func (gen *generation) retired() bool { + return gen.drained.Load() || gen.ctx.Err() != nil +} + +func (gen *generation) add(c *baseConn) { + gen.mu.Lock() + defer gen.mu.Unlock() + gen.conns[c] = struct{}{} +} + +func (gen *generation) remove(c *baseConn) { + gen.mu.Lock() + defer gen.mu.Unlock() + delete(gen.conns, c) +} + +// drain retires the generation gracefully: in-flight work continues, and +// the pool discards each conn the next time it tries to reuse it. +func (gen *generation) drain() { + gen.drained.Store(true) +} + +// kill retires the generation immediately: the generation context is +// canceled (with cause ErrHotSwap), in-flight operations get up to window to +// observe the cancellation, and then every remaining conn is force-closed. +// The bounded wait guarantees the caller (the group run loop) cannot be +// wedged by a driver that ignores context cancellation. +func (gen *generation) kill(window time.Duration) { + gen.drained.Store(true) + gen.cancel(ErrHotSwap) + gen.awaitIdle(window) + + gen.mu.Lock() + snap := make([]*baseConn, 0, len(gen.conns)) + for c := range gen.conns { + snap = append(snap, c) + } + gen.mu.Unlock() + + for _, c := range snap { + c.closeConn(true) + } +} + +// awaitIdle polls until no operations are in flight or the window elapses. +// Well-behaved drivers observe the canceled context within microseconds; +// drivers that ignore cancellation are unblocked by the conn close that +// follows in kill. +func (gen *generation) awaitIdle(window time.Duration) { + deadline := time.Now().Add(window) + for gen.ops.Load() > 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } +} + +// group represents one watched hotload DSN. It receives new connection +// string values from the strategy and swaps generations in response. Its +// mutex guards only the generation pointers and the closed flag; all driver +// calls, hook emissions and strategy calls happen outside it. +type group struct { + name string // the full hotload DSN + strategyName string + strategy Strategy + path string + pathQry string + sqlDriver *driverInstance + forceKill bool + killWindow time.Duration + parentCtx context.Context + parentCancel context.CancelFunc + newValChan <-chan string + + // refs and pinned are guarded by hdriver.mu. refs counts live + // connectors; pinned marks groups created through the legacy + // driver.Open path, which have no teardown signal and live forever. + refs int + pinned bool + + mu sync.Mutex + cur *generation + prev *generation + closed bool +} + +// runLoop receives new connection string values from the strategy until the +// group is shut down or the strategy closes the channel. It is the only +// goroutine that swaps generations. +func (g *group) runLoop() { + for { + select { + case <-g.parentCtx.Done(): + g.logf("group.runLoop", "parent context done, terminating") + return + case newValue, ok := <-g.newValChan: + if !ok { + g.logf("group.runLoop", "strategy channel closed, terminating") + EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) + return + } + g.onNewValue(newValue) + } + } +} + +func (g *group) onNewValue(v string) { + g.mu.Lock() + if g.closed || v == g.cur.dsn { + g.mu.Unlock() + g.logf("group.onNewValue", "conn dsn not changed") + return + } + old, older := g.cur, g.prev + next := newGeneration(g.parentCtx, v) + g.cur = next + if g.forceKill { + g.prev = nil + } else { + g.prev = old + } + g.mu.Unlock() + + g.logf("group.onNewValue", "conn dsn changed: '%s' -> '%s'", old.redactDsn, next.redactDsn) + emitConfigChange(ConfigChangeEvent{ + GroupName: g.name, + OldRedactedDSN: old.redactDsn, + NewRedactedDSN: next.redactDsn, + ForceKill: g.forceKill, + At: time.Now(), + }) + + if g.forceKill { + // Cancel in-flight work on the old DSN and close its conns. + old.kill(g.killWindow) + return + } + // Graceful: the old generation keeps serving in-flight work and drains + // through the pool; the generation before it has had its grace period + // and is killed now. + old.drain() + if older != nil { + older.kill(g.killWindow) + } +} + +// conn dials a new connection on the current generation. The underlying +// dial happens outside all locks; if a config change lands mid-dial the new +// conn belongs to an already-retired generation and self-evicts on first +// pool reuse, which the pool handles by dialing again. +func (g *group) conn(ctx context.Context) (driver.Conn, error) { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return nil, fmt.Errorf("hotload: group %q is closed", g.name) + } + gen := g.cur + g.mu.Unlock() + + dsn, err := mergeConnStringOptions(gen.dsn, g.sqlDriver.options) + if err != nil { + return nil, err + } + inner, err := g.dial(ctx, gen, dsn) + if err != nil { + return nil, err + } + + c := &baseConn{ + inner: inner, + gen: gen, + group: g, + dsn: dsn, + redactDsn: internal.RedactUrl(dsn), + } + gen.add(c) + emitConnOpen(ConnEvent{GroupName: g.name, RedactedDSN: c.redactDsn}) + g.logf("group.conn", "opened managed conn: '%s'", c.redactDsn) + return wrapConn(c), nil +} + +// dial opens an underlying connection, using the underlying driver's +// connector when it implements driver.DriverContext (so the dial honors +// ctx), and falling back to Open otherwise — the same split database/sql +// applies. +func (g *group) dial(ctx context.Context, gen *generation, dsn string) (driver.Conn, error) { + dc, ok := g.sqlDriver.driver.(driver.DriverContext) + if !ok { + return g.sqlDriver.driver.Open(dsn) + } + gen.connOnce.Do(func() { + gen.connector, gen.connErr = dc.OpenConnector(dsn) + }) + if gen.connErr != nil { + return nil, gen.connErr + } + return gen.connector.Connect(ctx) +} + +// shutdown tears the group down: the strategy watch is closed and the +// parent context canceled, which terminates runLoop and cancels every +// generation context. Called once the last connector referencing the group +// closes. +func (g *group) shutdown() { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return + } + g.closed = true + g.mu.Unlock() + + if err := g.strategy.CloseWatch(g.path, g.pathQry); err != nil { + g.logf("group.shutdown", "CloseWatch error: %v", err) + } + g.parentCancel() + EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) + g.logf("group.shutdown", "group closed") +} + +func mergeConnStringOptions(dsn string, options map[string]string) (string, error) { + if len(options) == 0 { + return dsn, nil + } + u, err := url.ParseRequestURI(dsn) + if err != nil { + return "", fmt.Errorf("unable to parse connection string when specifying extra driver options: %v", err) + } + values, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", fmt.Errorf("unable to parse query options in connection string when specifying extra driver options: %v", err) + } + for k, v := range options { + values.Set(k, v) + } + u.RawQuery = values.Encode() + return u.String(), nil +} + +func (g *group) logf(prefix, format string, args ...any) { + logger.Logf(fmt.Sprintf("%s[%s]:", prefix, g.name), format, args...) +} diff --git a/hotload_suite_test.go b/hotload_suite_test.go deleted file mode 100644 index 06cd01f..0000000 --- a/hotload_suite_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package hotload_test - -import ( - "log" - "testing" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestHotload(t *testing.T) { - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - nrr := internal.NewNonRandomReader(1) - uuid.SetRand(nrr) - - RegisterFailHandler(Fail) - RunSpecs(t, "Hotload Suite") -} diff --git a/integrationtests/context_test.go b/integrationtests/context_test.go deleted file mode 100644 index 05e58e2..0000000 --- a/integrationtests/context_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("postgres-direct (non-hotload) context test (verify underlying postgres driver is correct)", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - userDb = openDbPostgres(hldatabaseSuperDsn) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - userDb.Close() - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=false)", func(ctx context.Context) { - callerCancelContextTestFn(false, superDb, userDb) - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=true)", func(ctx context.Context) { - callerCancelContextTestFn(true, superDb, userDb) - }) -}) - -var _ = Describe("hotload context test", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - userDb = openDbHotload(false) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - userDb.Close() - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=false)", func(ctx context.Context) { - callerCancelContextTestFn(false, superDb, userDb) - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=true)", func(ctx context.Context) { - callerCancelContextTestFn(true, superDb, userDb) - }) -}) - -func callerCancelContextTestFn(forceKill bool, superDb, userDb *sql.DB) { - callerCtx, cancelFn := context.WithCancel(context.Background()) - csource := "caller-cancel-exec-context" - cnum := 141421 - insertStmt := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(1.0))", cnum, csource) - - By("Insert 1st row") - _, err := userDb.ExecContext(callerCtx, insertStmt) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("error inserting into table test")) - - // Long-running fn to insert 2nd row - errChan := make(chan error) - longExecContextFn := func(errChan chan error) { - log.Printf("start db.ExecContext INSERT PG_SLEEP") - result, err := userDb.ExecContext(callerCtx, insertStmt) - if err == nil { - log.Printf("db.ExecContext INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.ExecContext INSERT PG_SLEEP error=%+v", err) - } - errChan <- err - } - - By("Spawn long-running db background thread") - go longExecContextFn(errChan) - - By("Momentarily sleep/yield to long-running db background thread") - time.Sleep(100 * time.Millisecond) - - By("Cancel context of long-running db background thread") - cancelFn() - time.Sleep(1 * time.Millisecond) - - if forceKill { - By("Close connection of long-running db background thread") - userDb.Close() - } - - By("Wait for return from long-running db background thread") - err = <-errChan - log.Printf("returned from long-running db background thread, err=%v", err) - By("Got return from long-running db background thread") - Expect(err).To(HaveOccurred(), fmt.Sprintf("expect error inserting into table test")) - - expectRowCountInDb(superDb, csource, false, 1, int64(cnum)) -} diff --git a/integrationtests/docker/docker-compose.yaml b/integrationtests/docker/docker-compose.yaml deleted file mode 100644 index 3c4d594..0000000 --- a/integrationtests/docker/docker-compose.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' - -# postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml -services: - db: - image: postgres:10.3 - environment: - - POSTGRES_USER=admin - - POSTGRES_PASSWORD=test - - POSTGRES_DB=hldatabase - - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true - ports: - - '5432:5432' - volumes: - # https://github.com/felipewom/docker-compose-postgres - # https://geshan.com.np/blog/2021/12/docker-postgres/ - # https://github.com/docker-library/docs/tree/master/postgres#initialization-scripts - - ./intgtest_init.sql:/docker-entrypoint-initdb.d/intgtest_init.sql diff --git a/integrationtests/docker/intgtest_init.sql b/integrationtests/docker/intgtest_init.sql deleted file mode 100644 index ff3b4b0..0000000 --- a/integrationtests/docker/intgtest_init.sql +++ /dev/null @@ -1,5 +0,0 @@ --- postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml -CREATE DATABASE hotload_test; -CREATE DATABASE hotload_test1; -GRANT ALL PRIVILEGES ON DATABASE hotload_test TO admin; -GRANT ALL PRIVILEGES ON DATABASE hotload_test1 TO admin; diff --git a/integrationtests/helm/hotload-integration-tests/.helmignore b/integrationtests/helm/hotload-integration-tests/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/integrationtests/helm/hotload-integration-tests/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/integrationtests/helm/hotload-integration-tests/Chart.yaml b/integrationtests/helm/hotload-integration-tests/Chart.yaml deleted file mode 100644 index 78d5b1f..0000000 --- a/integrationtests/helm/hotload-integration-tests/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: hotload-integration-tests -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -# No external dependencies - using official PostgreSQL container directly \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl b/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl deleted file mode 100644 index 95b37ce..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "hotload-integration-tests.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "hotload-integration-tests.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "hotload-integration-tests.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "hotload-integration-tests.labels" -}} -helm.sh/chart: {{ include "hotload-integration-tests.chart" . }} -{{ include "hotload-integration-tests.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hotload-integration-tests.selectorLabels" -}} -app.kubernetes.io/name: {{ include "hotload-integration-tests.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "hotload-integration-tests.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "hotload-integration-tests.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml deleted file mode 100644 index 7b5368c..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.name }}-postgres-initdb - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -data: - init.sql: {{ .Values.postgres.initScript | quote }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml deleted file mode 100644 index 4150306..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}-postgresql - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 6 }} - template: - metadata: - labels: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 8 }} - spec: - containers: - - name: postgres - image: {{ .Values.postgres.image }} - ports: - - containerPort: 5432 - name: postgres - envFrom: - - secretRef: - name: {{ .Values.name }}-postgres-secret - volumeMounts: - - name: initdb - mountPath: /docker-entrypoint-initdb.d - readOnly: true - {{- if not .Values.postgres.persistence.enabled }} - - name: postgres-data - mountPath: /var/lib/postgresql/data - {{- end }} - livenessProbe: - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U {{ .Values.postgres.user }} -d {{ .Values.postgres.database }} -h 127.0.0.1 -p 5432 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U {{ .Values.postgres.user }} -d {{ .Values.postgres.database }} -h 127.0.0.1 -p 5432 - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - volumes: - - name: initdb - configMap: - name: {{ .Values.name }}-postgres-initdb - {{- if not .Values.postgres.persistence.enabled }} - - name: postgres-data - emptyDir: {} - {{- end }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml deleted file mode 100644 index 5b1c5db..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.name }}-postgres-secret - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -type: Opaque -data: - POSTGRES_USER: {{ .Values.postgres.user | b64enc | quote }} - POSTGRES_PASSWORD: {{ .Values.postgres.password | b64enc | quote }} - POSTGRES_DB: {{ .Values.postgres.database | b64enc | quote }} - postgres-password: {{ .Values.postgres.postgresPassword | b64enc | quote }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml deleted file mode 100644 index 14474b4..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.name }}-postgresql - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: postgres - protocol: TCP - name: postgres - selector: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 4 }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml b/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml deleted file mode 100644 index afe85d6..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ .Values.name }}-job" - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} - annotations: - "helm.sh/hook": test-success -spec: - containers: - - name: "hotload-integration-tests" - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: Never - command: ["./integrationtests.test"] - env: - - name: HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST - value: hotload-integration-tests-postgresql.default.svc.cluster.local - - name: HOTLOAD_PATH_CHKSUM_METRICS_ENABLE - value: "true" - restartPolicy: Never diff --git a/integrationtests/helm/hotload-integration-tests/values.yaml b/integrationtests/helm/hotload-integration-tests/values.yaml deleted file mode 100644 index 5ecd906..0000000 --- a/integrationtests/helm/hotload-integration-tests/values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Default values for hotload-integration-tests. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -name: hotload-integration-tests - -image: - repository: hotload-integration-tests - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -# postgres configuration using official postgres:10 image -postgres: - image: postgres:10 - user: admin - password: test - database: hldatabase - postgresPassword: postgres # password for 'postgres' superuser - persistence: - enabled: false - initScript: | - CREATE DATABASE hotload_test; - CREATE DATABASE hotload_test1; - GRANT ALL PRIVILEGES ON DATABASE hotload_test TO admin; - GRANT ALL PRIVILEGES ON DATABASE hotload_test1 TO admin; diff --git a/integrationtests/hotload_test.go b/integrationtests/hotload_test.go deleted file mode 100644 index 475223f..0000000 --- a/integrationtests/hotload_test.go +++ /dev/null @@ -1,314 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "os" - "strings" - "time" - - "github.com/infobloxopen/hotload" - _ "github.com/infobloxopen/hotload/fsnotify" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" - "github.com/infobloxopen/hotload/modtime" - "github.com/lib/pq" - _ "github.com/lib/pq" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -const ( - fsnotifyStrategy = "fsnotify" - configPath = "/tmp/hotload_integration_test_dsn_config.txt" - userSqlSetup = "CREATE USER uuser WITH PASSWORD 'ppass1'" - userSqlTeardown = "DROP USER IF EXISTS uuser" - testSqlSetup = "CREATE TABLE IF NOT EXISTS test (cnum INT, csource TEXT, csleep TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)" - testSqlTeardown = "DROP TABLE IF EXISTS test" - truncSqlSetup = "TRUNCATE TABLE test" - initSqlSetup = "INSERT INTO test (cnum, csource) VALUES (161803, 'initial')" - grantSqlSetup = "GRANT ALL ON test TO PUBLIC" - - setDSNSleepDur = 100 * time.Millisecond -) - -var ( - mtmCtx context.Context - mtmCancelCtxFn context.CancelFunc - mtm *modtime.ModTimeMonitor -) - -func init() { - // this function call registers the lib/pq postgres driver with hotload - hotload.RegisterSQLDriver("postgres", &pq.Driver{}) -} - -func formHotloadDsn(forceKill bool) string { - dsnUrl := "fsnotify://postgres" + configPath - if forceKill { - dsnUrl = dsnUrl + "?forceKill=true" - } - return dsnUrl -} - -func setDSN(dsn string, path string) { - err := os.WriteFile(path, []byte(dsn), 0777) - if err != nil { - Fail(fmt.Sprintf("setDSN: error writing dsn file: %v", err)) - } - log.Printf("setDSN: success writing '%s' to '%s'", dsn, path) - - // Yield thread to let switch over take place - time.Sleep(setDSNSleepDur) - log.Printf("setDSN: slept/yielded %s", setDSNSleepDur) -} - -// Open a db using postgres driver, or die -func openDbPostgres(dsn string) *sql.DB { - db, err := sql.Open("postgres", dsn) - if err != nil { - Fail(fmt.Sprintf("openDbPostgres: error opening db dsn '%s': %v", dsn, err)) - } - log.Printf("openDbPostgres: opened db dsn '%s'", dsn) - - return db -} - -// Open a db using hotload driver, or die -func openDbHotload(forceKill bool) *sql.DB { - dsnUrl := formHotloadDsn(forceKill) - - db, err := sql.Open("hotload", dsnUrl) - if err != nil { - Fail(fmt.Sprintf("openDbHotload: err opening db dsn '%s': %v", dsnUrl, err)) - } - log.Printf("openDbHotload: opened hotload dsn '%s'", dsnUrl) - - err = db.Ping() - if err != nil { - Fail(fmt.Sprintf("openDbHotload: err pinging db dsn '%s': %v", dsnUrl, err)) - } - log.Printf("openDbHotload: pinged hotload dsn '%s'", dsnUrl) - - return db -} - -func expectValueInDb(db *sql.DB, source string, expErr bool, expRowCount int, expVal int64) { - GinkgoHelper() - log.Printf("expectValueInDb: expErr=%v, expRowCount=%d, expVal=%d", expErr, expRowCount, expVal) - r, err := db.Query(fmt.Sprintf("SELECT cnum FROM test WHERE csource = '%s'", source)) - var cnum int64 - if expErr { - Expect(err).To(HaveOccurred(), fmt.Sprintf("expectValueInDb: expect error reading from table test")) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: error reading from table test: %v", err)) - } - gotRowCount := 0 - if !r.Next() { - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: cursor iteration err: %v", r.Err())) - } else { - gotRowCount = 1 - err = r.Scan(&cnum) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: error calling r.Scan(): %v", err)) - Expect(cnum).To(Equal(expVal)) - } - Expect(gotRowCount).To(Equal(expRowCount)) -} - -func expectRowCountInDb(db *sql.DB, source string, expErr bool, expRowCount int, expVal int64) { - GinkgoHelper() - log.Printf("expectRowCountInDb: expErr=%v, expRowCount=%d, expVal=%d", expErr, expRowCount, expVal) - r, err := db.Query(fmt.Sprintf("SELECT COUNT(*) FROM test WHERE csource = '%s' AND cnum = %d", source, expVal)) - if expErr { - Expect(err).To(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: expect error reading from table test")) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: error reading from table test: %v", err)) - } - if !r.Next() { - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: cursor iteration err: %v", r.Err())) - } else { - var gotRowCount int - err = r.Scan(&gotRowCount) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: error calling r.Scan(): %v", err)) - Expect(gotRowCount).To(Equal(expRowCount)) - } -} - -// TODO: expectConnCountInDb is not reliable, pg_stat_activity connections fluctuates -func expectConnCountInDb(db *sql.DB, expConnCount int) { - GinkgoHelper() - log.Printf("expectConnCountInDb: expConnCount=%d", expConnCount) - r, err := db.Query(fmt.Sprintf("SELECT datname, usename, application_name, client_addr, state, backend_type, query FROM pg_stat_activity WHERE client_addr IS NOT NULL")) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: error reading from table pg_stat_activity: %v", err)) - gotConnCount := 0 - for r.Next() { - gotConnCount++ - var datname string - var usename string - var application_name string - var client_addr string - var state string - var backend_type string - var query string - err = r.Scan(&datname, &usename, &application_name, &client_addr, &state, &backend_type, &query) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: error calling r.Scan(): %v", err)) - log.Printf("pg_stat_activity: datname='%s', usename='%s', app_name='%s', client_addr='%s', state='%s', backend_type='%s', query='%s'", - datname, usename, application_name, client_addr, state, backend_type, query) - } - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: cursor iteration err: %v", r.Err())) - Expect(gotConnCount).To(Equal(expConnCount)) -} - -func expectModTime(modPath string, prevModTime time.Time) time.Time { - GinkgoHelper() - nextModTime := prevModTime - - // Get modPath modtime - sts, err := mtm.GetPathStatus(fsnotifyStrategy, modPath) - if err != nil { - Fail(fmt.Sprintf("expectModTime: GetPathStatus(%s) err: %v", modPath, err)) - } - log.Printf("expectModTime: GetPathStatus(%s): %v", modPath, sts) - - // Verify modPath modtime was updated after modPath was updated with new DSN - nextModTime = sts.ModTime - if !nextModTime.After(prevModTime) { - Fail(fmt.Sprintf("expectModTime: %s: new sts.ModTime(%s) <= prevModTime(%s)", modPath, nextModTime, prevModTime)) - } - - return nextModTime -} - -var _ = BeforeSuite(func(ctx context.Context) { - // create tables and chairs - hlt, err := sql.Open("postgres", hotloadTestDsn) - hlt1, err := sql.Open("postgres", hotloadTest1Dsn) - defer hlt.Close() - defer hlt1.Close() - - for { - time.Sleep(5 * time.Second) - _, err = hlt.Exec(testSqlSetup) - if err != nil { - log.Printf("BeforeSuite: error creating test table in hlt: %v", err) - continue - //Fail(fmt.Sprintf()) - } - - _, err = hlt1.Exec(testSqlSetup) - if err != nil { - log.Printf("BeforeSuite: error creating test table in hlt1: %v", err) - continue - } - - break - } - - // enable ModTimeMonitor to monitor configPath - // (do NOT use the ginkgo supplied ctx parm, - // as it will be canceled when BeforeSuite finishes) - mtmCtx, mtmCancelCtxFn = context.WithCancel(context.Background()) - mtm = modtime.NewModTimeMonitor(mtmCtx, - // note the check-interval must be shorter than the sleep interval in setDSN() - modtime.WithCheckInterval(setDSNSleepDur/3), - modtime.WithLogger(testLogger), - modtime.WithErrLogger(testLogger), - ) - mtm.AddMonitoredPath(fsnotifyStrategy, configPath) - time.Sleep(200 * time.Millisecond) -}, NodeTimeout(240*time.Second)) - -var _ = AfterSuite(func(ctx context.Context) { - log.Printf("AfterSuite canceling ModTimeMonitor context") - mtmCancelCtxFn() - time.Sleep(200 * time.Millisecond) - - hlt, err := sql.Open("postgres", hotloadTestDsn) - hlt1, err := sql.Open("postgres", hotloadTest1Dsn) - defer hlt.Close() - defer hlt1.Close() - - _, err = hlt.Exec(testSqlTeardown) - if err != nil { - log.Printf("AfterSuite: error dropping test table in hlt: %v", err) - } - - _, err = hlt1.Exec(testSqlTeardown) - if err != nil { - log.Printf("AfterSuite: error dropping test table in hlt1: %v", err) - } - - //expectConnCountInDb(hlt, 3) - - err = internal.CollectAndRegexpCompare(metrics.HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(metrics.ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(metrics.ExpectHotloadPathChksumTimestampSecondsRegexp, - "/tmp/hotload_integration_test_dsn_config.txt")), - metrics.HotloadPathChksumTimestampSecondsName) - Expect(err).ShouldNot(HaveOccurred()) -}, NodeTimeout(240*time.Second)) - -var _ = Describe("hotload integration tests - sanity", Serial, func() { - var ( - db *sql.DB - hltDb *sql.DB - hlt1Db *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - setDSN(hotloadTestDsn, configPath) - hltDb = openDbPostgres(hotloadTestDsn) - hlt1Db = openDbPostgres(hotloadTest1Dsn) - - _, err := hltDb.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error truncating test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error truncating test table in hlt1Db: %v", err)) - } - - _, err = hltDb.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error initing test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error initing test table in hlt1Db: %v", err)) - } - - db = openDbHotload(false) - }) - - AfterEach(func(ctx context.Context) { - hltDb.Close() - hlt1Db.Close() - db.Close() - }) - - It("should connect to new db when file changes", func(ctx context.Context) { - var prevModTime time.Time - for i := 0; i < 2; i++ { - // Verify configPath modtime was updated after configPath was updated with new DSN - prevModTime = expectModTime(configPath, prevModTime) - - r, err := db.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, 'sanity')", i)) - if err != nil { - Fail(fmt.Sprintf("error inserting cnum=%d row: %v", i, err)) - } else { - log.Printf("inserted cnum=%d row", i) - } - log.Print(r) - - // Set new DSN, note that this sleeps for 250 millisecs - setDSN(hotloadTest1Dsn, configPath) - } - expectValueInDb(hltDb, "sanity", false, 1, 0) - expectValueInDb(hlt1Db, "sanity", false, 1, 1) - }) -}) diff --git a/integrationtests/integrationtests_suite_test.go b/integrationtests/integrationtests_suite_test.go deleted file mode 100644 index dfbab22..0000000 --- a/integrationtests/integrationtests_suite_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package integrationtests - -import ( - "fmt" - "log" - "os" - "strings" - "testing" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var ( - postgresHost = "localhost" - postgresPort = "5432" - - hotloadTestDsn string - hotloadTest1Dsn string - - hldatabaseSuperDsn string - hldatabaseAdminDsn string - - superUser = "admin" - superPass = "test" - adminUser = "admin" - adminPass = "test" - testDbUser = "uuser" -) - -func testDbPass(which int) string { - return fmt.Sprintf("ppass%d", which) -} - -func hldatabasePassDsn(which int) string { - return fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - testDbUser, testDbPass(which), postgresHost, postgresPort) -} - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestIntegrationtests(t *testing.T) { - //log.SetFlags(log.Flags() | log.Lmicroseconds) - log.SetFlags(log.Ltime | log.Lmicroseconds) - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - nrr := internal.NewNonRandomReader(1) - uuid.SetRand(nrr) - - pgHost, ok := os.LookupEnv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST") - pgHost = strings.TrimSpace(pgHost) - if ok && len(pgHost) > 0 { - postgresHost = pgHost - } - - pgPort, ok := os.LookupEnv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT") - pgPort = strings.TrimSpace(pgPort) - if ok && len(pgPort) > 0 { - postgresPort = pgPort - } - - hotloadTestDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hotload_test?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - hotloadTest1Dsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hotload_test1?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - - hldatabaseSuperDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - superUser, superPass, postgresHost, postgresPort) - hldatabaseAdminDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - - RegisterFailHandler(Fail) - RunSpecs(t, "Integration Tests") -} diff --git a/integrationtests/longdbtxn_test.go b/integrationtests/longdbtxn_test.go deleted file mode 100644 index cceefe6..0000000 --- a/integrationtests/longdbtxn_test.go +++ /dev/null @@ -1,503 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - _ "github.com/infobloxopen/hotload" - _ "github.com/infobloxopen/hotload/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -type longTestType int - -const ( - LongDatabaseChange longTestType = iota - LongPasswordChange -) - -var longTestNames = map[longTestType]string{ - LongDatabaseChange: "LongDatabaseChange", - LongPasswordChange: "LongPasswordChange", -} - -func (l longTestType) String() string { - return longTestNames[l] -} - -type longDbMode int - -const ( - LongExec longDbMode = iota - LongExecContext - LongQuery - LongQueryContext - LongBegin - LongBeginTx -) - -var longModeNames = map[longDbMode]string{ - LongExec: "LongExec", - LongExecContext: "LongExecContext", - LongQueryContext: "LongQueryContext", - LongQuery: "LongQuery", - LongBegin: "LongBegin", - LongBeginTx: "LongBeginTx", -} - -func (l longDbMode) String() string { - return longModeNames[l] -} - -type longDbTestCase struct { - forceKill bool - testType longTestType - testMode longDbMode - source string - expErr bool - expRowCount int - expCnum int64 - gotErr error - gotRowCount int - longChan chan longDbTestCase - userDb *sql.DB - superDb *sql.DB -} - -// longDbExecFn DB.Exec's a long-running INSERT stmt -func longDbExecFn(tc longDbTestCase) { - tc.gotRowCount = 0 - log.Printf("start db.Exec INSERT PG_SLEEP") - insertStmt := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(1.0))", tc.expCnum, tc.source) - var result sql.Result - var err error - if tc.testMode == LongExec { - result, err = tc.userDb.Exec(insertStmt) - } else { - result, err = tc.userDb.ExecContext(context.Background(), insertStmt) - } - if err == nil { - log.Printf("db.Exec INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.Exec INSERT PG_SLEEP error=%+v", err) - } - tc.gotErr = err - tc.longChan <- tc -} - -// longDbQueryFn DB.Query's a long-running SELECT stmt -func longDbQueryFn(tc longDbTestCase) { - tc.gotRowCount = 0 - log.Printf("start db.Query SELECT PG_SLEEP") - rows, err := tc.userDb.Query(fmt.Sprintf("SELECT COUNT(*), PG_SLEEP(1.0) FROM test WHERE csource = '%s' AND cnum = %d", tc.source, tc.expCnum)) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP rows=%+v", rows) - if !rows.Next() { - err = rows.Err() - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor iteration error=%+v", err) - } - log.Printf("db.Query SELECT PG_SLEEP cursor returned no rows") - } else { - var csleep string - err = rows.Scan(&tc.gotRowCount, &csleep) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor scan error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP cursor scanned rowcount=%d", tc.gotRowCount) - } - } - } - tc.gotErr = err - tc.longChan <- tc -} - -// longDbBeginFn DB.Begin's (or DB.BeginTx's) and DB.Commit/DB.Rollback's -// around a long-running db txn that does a INSERT, sleep(1s), SELECT. -func longDbBeginFn(tc longDbTestCase) { - tc.gotRowCount = 0 - txnFn := func() error { - var txnDb *sql.Tx - var err error - if tc.testMode == LongBeginTx { - log.Printf("start db.BeginTx") - txnDb, err = tc.userDb.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelReadCommitted, - }) - if err == nil { - log.Printf("db.BeginTx success") - } else { - log.Printf("db.BeginTx failure err=%+v", err) - return err - } - } else { - log.Printf("start db.Begin") - txnDb, err = tc.userDb.Begin() - if err == nil { - log.Printf("db.Begin success") - } else { - log.Printf("db.Begin failure err=%+v", err) - return err - } - } - defer func() { - if err == nil { - commitErr := txnDb.Commit() - log.Printf("txnDb.Commit err=%v", commitErr) - } else { - rollbackErr := txnDb.Rollback() - log.Printf("txnDb.Rollback err=%v", rollbackErr) - } - }() - - log.Printf("start db.Exec INSERT PG_SLEEP") - result, err := txnDb.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(0.000001))", tc.expCnum, tc.source)) - if err == nil { - log.Printf("db.Exec INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.Exec INSERT PG_SLEEP error=%+v", err) - } - - time.Sleep(1 * time.Second) - - log.Printf("start db.Query SELECT PG_SLEEP") - rows, err := txnDb.Query(fmt.Sprintf("SELECT COUNT(*), PG_SLEEP(0.000001) FROM test WHERE csource = '%s' AND cnum = %d", tc.source, tc.expCnum)) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP rows=%+v", rows) - if !rows.Next() { - err = rows.Err() - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor iteration error=%+v", err) - } - log.Printf("db.Query SELECT PG_SLEEP cursor returned no rows") - } else { - var csleep string - err = rows.Scan(&tc.gotRowCount, &csleep) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor scan error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP cursor scanned rowcount=%d", tc.gotRowCount) - } - } - } - return err - } - err := txnFn() - tc.gotErr = err - tc.longChan <- tc -} - -func longDbTestFn(ginkgoCtx context.Context, tc longDbTestCase) { - //GinkgoHelper() - - dsn1 := hotloadTestDsn - dsn2 := hotloadTest1Dsn - if tc.testType == LongPasswordChange { - dsn1 = hldatabasePassDsn(1) - dsn2 = hldatabasePassDsn(2) - } - - db := openDbHotload(tc.forceKill) - defer func() { - log.Println("longDbTestFn: closing hotload db conn") - db.Close() - }() - - var prevModTime time.Time - prevModTime = expectModTime(configPath, prevModTime) - - longChan := make(chan longDbTestCase) - longDbFn := longDbExecFn - switch tc.testMode { - case LongExec, LongExecContext: - longDbFn = longDbExecFn - case LongQuery, LongQueryContext: - longDbFn = longDbQueryFn - case LongBegin, LongBeginTx: - longDbFn = longDbBeginFn - default: - longDbFn = nil - Fail(fmt.Sprintf("invalid testMode=%d", int(tc.testMode))) - } - tc.longChan = longChan - tc.userDb = db - - // 1st background long-running db thread, set new DSN in middle of long-running db txn - - By("1st: Spawn long-running db background thread") - go longDbFn(tc) - - By("1st: Momentarily sleep/yield to long-running db background thread") - time.Sleep(10 * time.Millisecond) - - By("1st: Set new DSN, this sleeps/yields for 250 millisecs") - if tc.testType == LongPasswordChange { - dbExecAlterUserPass(tc.superDb, testDbUser, testDbPass(2)) - } - setDSN(dsn2, configPath) - - By("1st: Verify configPath modtime was updated after configPath was updated with new DSN") - prevModTime = expectModTime(configPath, prevModTime) - - By("1st: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("1st: tc returned from long-running db background thread = %#v", tc) - By("1st: Got return from long-running db background thread") - if tc.expErr { - Expect(tc.gotErr).To(HaveOccurred(), fmt.Sprintf("expect error inserting into table test")) - } else { - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("error inserting into table test: %v", tc.gotErr)) - } - - Expect(tc.gotRowCount).To(Equal(tc.expRowCount)) - - // 2nd background long-running db thread, without setting new DSN - - By("2nd: Spawn long-running db background thread") - go longDbFn(tc) - - By("2nd: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("2nd: tc returned from long-running db background thread = %#v", tc) - By("2nd: Got return from long-running db background thread") - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("2nd: error executing long db test: %v", tc.gotErr)) - - // 3rd background long-running db thread, after setting new DSN - - By("3rd: Revert to old DSN, this sleeps/yields for 250 millisecs") - if tc.testType == LongPasswordChange { - dbExecAlterUserPass(tc.superDb, testDbUser, testDbPass(1)) - } - setDSN(dsn1, configPath) - - By("3rd: Verify configPath modtime was updated after configPath was updated with new DSN") - prevModTime = expectModTime(configPath, prevModTime) - - By("3rd: Spawn long-running db background thread") - go longDbFn(tc) - - By("3rd: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("3rd: tc returned from long-running db background thread = %#v", tc) - By("3rd: Got return from long-running db background thread") - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("3rd: error executing long db test: %v", tc.gotErr)) -} - -var _ = Describe("hotload integration tests - long running db transaction", Serial, func() { - var ( - hltDb *sql.DB - hlt1Db *sql.DB - ) - - BeforeEach(func(ginkgoCtx context.Context) { - //hotload.UnregisterStrategy("fsnotify") - //hotload.RegisterStrategy("fsnotify", fsnotify.NewStrategy()) - //time.Sleep(100 * time.Millisecond) // Sleep/yield to fsnotify background threads - - By("Resetting and truncating test db") - setDSN(hotloadTestDsn, configPath) - hltDb = openDbPostgres(hotloadTestDsn) - hlt1Db = openDbPostgres(hotloadTest1Dsn) - - _, err := hltDb.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error truncating test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error truncating test table in hlt1Db: %v", err)) - } - - // insert 1 initial row into hltDb - _, err = hltDb.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 1 into test table in hltDb: %v", err)) - } - - // insert 2 initial rows into hlt1Db - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 1 into test table in hlt1Db: %v", err)) - } - time.Sleep(100 * time.Millisecond) - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 2 into test table in hlt1Db: %v", err)) - } - }, NodeTimeout(60*time.Second)) - - AfterEach(func(ginkgoCtx context.Context) { - //expectConnCountInDb(hltDb, 3) - hltDb.Close() - hlt1Db.Close() - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongExec, - source: LongExec.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongExec, - source: LongExec.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecContextFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: false, - expRowCount: 1, - expCnum: 161803, // see initSqlSetup variable - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: true, - expRowCount: 0, - expCnum: 161803, // see initSqlSetup variable - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongBegin, - source: LongBegin.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongBegin, - source: LongBegin.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) -}) diff --git a/integrationtests/multipleconn_test.go b/integrationtests/multipleconn_test.go deleted file mode 100644 index dfd5be4..0000000 --- a/integrationtests/multipleconn_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("hotload context test", Serial, func() { - var ( - superDb *sql.DB - userDb [10]*sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(0)) - setDSN(hldatabasePassDsn(0), configPath) - - for i := 0; i < len(userDb); i++ { - userDb[i] = openDbHotload(false) - } - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - - for i := 0; i < len(userDb); i++ { - userDb[i].Close() - } - }) - - It("forceKill=false multiple long db.Exec in succession", func(ctx context.Context) { - multipleLongTestFn(LongExec, superDb, userDb) - }) -}) - -func multipleLongTestFn(lMode longDbMode, superDb *sql.DB, userDb [10]*sql.DB) { - callerCtx := context.Background() - csource := "multiple-long-in-succession" - cnum := 141421 - sleepSecs := 2 - insertStmtSleep := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(%d))", cnum, csource, sleepSecs) - insertStmtNoSleep := fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, '%s')", cnum, csource) - - // userDb[0] inserts 1st row - _, err := userDb[0].ExecContext(callerCtx, insertStmtNoSleep) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("0: error inserting into table test")) - - // userDb[1] inserts 2nd row - _, err = userDb[1].ExecContext(callerCtx, insertStmtNoSleep) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("1: error inserting into table test")) - - // Long-running fn to insert more rows - errChan := make(chan error) - longDbFn := func(which int, lMode longDbMode, userDb *sql.DB, errChan chan error) { - log.Printf("%d: start db.ExecContext INSERT PG_SLEEP", which) - result, err := userDb.ExecContext(callerCtx, insertStmtSleep) - if err == nil { - log.Printf("%d: db.ExecContext INSERT PG_SLEEP result=%+v", which, result) - } else { - log.Printf("%d: db.ExecContext INSERT PG_SLEEP error=%+v", which, err) - } - errChan <- err - } - - go longDbFn(2, lMode, userDb[2], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[2]), - // but since forceKill=false, will gracefully let userDb[2] txn continue. - // Any previous conn/txn will be canceled/closed (but there - // isn't any previous txn at this point in the test). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - - go longDbFn(3, lMode, userDb[3], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[3]), - // but since forceKill=false, will gracefully let userDb[3] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[2]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(2)) - setDSN(hldatabasePassDsn(2), configPath) - - go longDbFn(4, lMode, userDb[4], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[4]), - // but since forceKill=false, will gracefully let userDb[4] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[3]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(3)) - setDSN(hldatabasePassDsn(3), configPath) - - go longDbFn(5, lMode, userDb[5], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[5]), - // but since forceKill=false, will gracefully let userDb[5] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[4]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(4)) - setDSN(hldatabasePassDsn(4), configPath) - - // At this point in the test, only userDb[5] txn is allowed to gracefully continue - // and insert 3rd row; the other txn (userDb[2]/3/4 have been canceled). - - for i := 2; i <= 5; i++ { - err = <-errChan - log.Printf("%d: returned from long-running db background thread, err=%v", i, err) - if i < 5 { - Expect(err).To(HaveOccurred(), fmt.Sprintf("%d: expect error inserting into table test", i)) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("%d: unexpected error inserting into table test: %v", i, err)) - } - } - expectRowCountInDb(superDb, csource, false, 3, int64(cnum)) - - dbExecSqlStmt(userDb[0], insertStmtNoSleep) - dbExecSqlStmt(userDb[1], insertStmtNoSleep) - dbExecSqlStmt(userDb[2], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(5)) - setDSN(hldatabasePassDsn(5), configPath) - - dbExecSqlStmt(userDb[3], insertStmtNoSleep) - dbExecSqlStmt(userDb[4], insertStmtNoSleep) - dbExecSqlStmt(userDb[5], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(6)) - setDSN(hldatabasePassDsn(6), configPath) - - dbExecSqlStmt(userDb[6], insertStmtNoSleep) - dbExecSqlStmt(userDb[7], insertStmtNoSleep) - dbExecSqlStmt(userDb[8], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(7)) - setDSN(hldatabasePassDsn(7), configPath) - - dbExecSqlStmt(userDb[9], insertStmtNoSleep) - expectRowCountInDb(superDb, csource, false, 13, int64(cnum)) -} diff --git a/integrationtests/passwdchange_test.go b/integrationtests/passwdchange_test.go deleted file mode 100644 index 6929a97..0000000 --- a/integrationtests/passwdchange_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" -) - -func dbExecSqlStmt(db *sql.DB, sqlStmt string) { - GinkgoHelper() - _, err := db.Exec(sqlStmt) - if err != nil { - Fail(fmt.Sprintf("db.Exec(%s) error: %v", sqlStmt, err)) - } - time.Sleep(1 * time.Millisecond) -} - -func dbExecAlterUserPass(db *sql.DB, user, pass string) { - GinkgoHelper() - sqlStmt := fmt.Sprintf("ALTER USER %s WITH PASSWORD '%s'", user, pass) - dbExecSqlStmt(db, sqlStmt) -} - -var _ = Describe("hotload integration tests - db passwd change", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - }) - - It("should reconnect to db when password changes", func(ctx context.Context) { - userDb = openDbHotload(false) - defer userDb.Close() - - csource := "db-passwd-change" - cnum := 184775 - var prevModTime time.Time - for i := 0; i < 2; i++ { - // Verify configPath modtime was updated after configPath was updated with new DSN - prevModTime = expectModTime(configPath, prevModTime) - - r, err := userDb.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, '%s')", - cnum, csource)) - if err != nil { - Fail(fmt.Sprintf("%d: error inserting cnum=%d row: %v", i, cnum, err)) - } else { - log.Printf("%d: inserted cnum=%d row", i, cnum) - } - log.Print(r) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(2)) - - // Set new DSN, note that this sleeps for 250 millisecs - setDSN(hldatabasePassDsn(2), configPath) - } - expectRowCountInDb(superDb, csource, false, 2, int64(cnum)) - }) - - It("long-running db.Exec, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongExec, - source: LongExec.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongExec, - source: LongExec.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecContextFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - //expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: false, - expRowCount: 1, - expCnum: 161803, // see initSqlSetup variable - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: true, - expRowCount: 0, - expCnum: 161803, // see initSqlSetup variable - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongBegin, - source: LongBegin.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongBegin, - source: LongBegin.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) -}) diff --git a/interfaces_test.go b/interfaces_test.go new file mode 100644 index 0000000..590cfc9 --- /dev/null +++ b/interfaces_test.go @@ -0,0 +1,132 @@ +package hotload_test + +import ( + "database/sql/driver" + "fmt" + "testing" + + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// TestConnInterfaceTruthfulness verifies the heart of the wrapper design: +// the conn hotload hands to database/sql implements an optional interface if +// and only if the underlying conn supports the capability. It walks every +// combination of the mirrored capabilities plus the legacy-interface +// variants and asserts presence and absence of each interface. +func TestConnInterfaceTruthfulness(t *testing.T) { + // The four mirrored axes. + axes := []dbfake.Caps{ + dbfake.CapExecerContext, + dbfake.CapQueryerContext, + dbfake.CapPinger, + dbfake.CapNamedValueChecker, + } + + var combos []dbfake.Caps + for mask := 0; mask < 1< 0 { + time.Sleep(d.OpenDelay) + } + if d.OpenErr != nil { + if err := d.OpenErr(dsn); err != nil { + return nil, err + } + } + d.mu.Lock() + d.nextID++ + c := &Conn{drv: d, ID: d.nextID, DSN: dsn, caps: d.Caps} + d.conns = append(d.conns, c) + d.mu.Unlock() + d.Log.add(c.ID, "Open", dsn) + return viewConn(c), nil +} + +// Conns returns a snapshot of every conn the driver has opened, in order. +func (d *Driver) Conns() []*Conn { + d.mu.Lock() + defer d.mu.Unlock() + out := make([]*Conn, len(d.conns)) + copy(out, d.conns) + return out +} + +// OpenConns returns the conns that have not been closed yet. +func (d *Driver) OpenConns() []*Conn { + d.mu.Lock() + defer d.mu.Unlock() + var out []*Conn + for _, c := range d.conns { + if !c.Closed() { + out = append(out, c) + } + } + return out +} + +// Conn is the fake connection. It is always handed to hotload behind a +// capability view (see views_gen.go), never directly. +type Conn struct { + drv *Driver + ID int + DSN string + caps Caps + + closed atomic.Bool + closeCount atomic.Int64 + valid atomic.Bool // inverted: true means IsValid returns false +} + +// Closed reports whether the conn was closed. +func (c *Conn) Closed() bool { return c.closed.Load() } + +// CloseCount reports how many times Close was called on the conn; hotload +// must never close an underlying conn twice. +func (c *Conn) CloseCount() int64 { return c.closeCount.Load() } + +// SetInvalid makes a CapValidator conn report IsValid() == false. +func (c *Conn) SetInvalid() { c.valid.Store(true) } + +func (c *Conn) log(method, detail string) { c.drv.Log.add(c.ID, method, detail) } + +func (c *Conn) doPrepare(query string) (driver.Stmt, error) { + c.log("Prepare", query) + return viewStmt(&Stmt{conn: c, query: query}), nil +} + +func (c *Conn) doPrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + c.log("PrepareContext", query) + if err := ctx.Err(); err != nil { + return nil, err + } + return viewStmt(&Stmt{conn: c, query: query}), nil +} + +func (c *Conn) doClose() error { + c.closeCount.Add(1) + c.closed.Store(true) + c.log("Close", "") + return nil +} + +func (c *Conn) doBegin() (driver.Tx, error) { + c.log("Begin", "") + return &Tx{conn: c}, nil +} + +func (c *Conn) doBeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + c.log("BeginTx", "") + if err := ctx.Err(); err != nil { + return nil, err + } + return &Tx{conn: c}, nil +} + +func (c *Conn) doExec(ctx context.Context, method, query string, args []driver.NamedValue) (driver.Result, error) { + c.log(method, query) + if c.drv.ExecFn != nil { + return c.drv.ExecFn(c, ctx, query, args) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return driver.RowsAffected(1), nil +} + +func (c *Conn) doQuery(ctx context.Context, method, query string, args []driver.NamedValue) (driver.Rows, error) { + c.log(method, query) + if c.drv.QueryFn != nil { + return c.drv.QueryFn(c, ctx, query, args) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return &Rows{cols: []string{"dsn"}, rows: [][]driver.Value{{c.DSN}}}, nil +} + +func (c *Conn) doPing(ctx context.Context) error { + c.log("Ping", "") + return ctx.Err() +} + +func (c *Conn) doCheckNamedValue(nv *driver.NamedValue) error { + c.log("CheckNamedValue", fmt.Sprint(nv.Value)) + return nil +} + +func (c *Conn) doResetSession(ctx context.Context) error { + c.log("ResetSession", "") + return nil +} + +func (c *Conn) doIsValid() bool { + c.log("IsValid", "") + return !c.valid.Load() +} + +func namedToValues(args []driver.Value) []driver.NamedValue { + out := make([]driver.NamedValue, len(args)) + for i, v := range args { + out[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return out +} + +// fvBase implements the mandatory driver.Conn methods for every view. +type fvBase struct{ c *Conn } + +func (v fvBase) Prepare(query string) (driver.Stmt, error) { return v.c.doPrepare(query) } +func (v fvBase) Close() error { return v.c.doClose() } +func (v fvBase) Begin() (driver.Tx, error) { return v.c.doBegin() } + +// Unwrap exposes the fake conn behind a view for test assertions. +func (v fvBase) Unwrap() *Conn { return v.c } + +// Unwrapper is implemented by every conn view; tests use it to reach the +// underlying fake conn. +type Unwrapper interface{ Unwrap() *Conn } + +type fvExecer struct{ c *Conn } + +func (v fvExecer) Exec(query string, args []driver.Value) (driver.Result, error) { + return v.c.doExec(context.Background(), "Exec", query, namedToValues(args)) +} + +type fvExecerCtx struct{ c *Conn } + +func (v fvExecerCtx) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return v.c.doExec(ctx, "ExecContext", query, args) +} + +type fvQueryer struct{ c *Conn } + +func (v fvQueryer) Query(query string, args []driver.Value) (driver.Rows, error) { + return v.c.doQuery(context.Background(), "Query", query, namedToValues(args)) +} + +type fvQueryerCtx struct{ c *Conn } + +func (v fvQueryerCtx) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return v.c.doQuery(ctx, "QueryContext", query, args) +} + +type fvPinger struct{ c *Conn } + +func (v fvPinger) Ping(ctx context.Context) error { return v.c.doPing(ctx) } + +type fvNVChecker struct{ c *Conn } + +func (v fvNVChecker) CheckNamedValue(nv *driver.NamedValue) error { return v.c.doCheckNamedValue(nv) } + +// fvClassA groups the optional interfaces hotload always synthesizes when +// the underlying conn lacks them: ConnPrepareContext, ConnBeginTx, +// SessionResetter and Validator. +type fvClassA struct{ c *Conn } + +func (v fvClassA) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + return v.c.doPrepareContext(ctx, query) +} + +func (v fvClassA) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + return v.c.doBeginTx(ctx, opts) +} + +func (v fvClassA) ResetSession(ctx context.Context) error { return v.c.doResetSession(ctx) } +func (v fvClassA) IsValid() bool { return v.c.doIsValid() } + +// Stmt is the fake prepared statement, always handed out behind a +// capability view. +type Stmt struct { + conn *Conn + query string +} + +func (s *Stmt) doClose() error { + s.conn.log("StmtClose", s.query) + return nil +} + +func (s *Stmt) doNumInput() int { return -1 } + +func (s *Stmt) doExec(ctx context.Context, method string, args []driver.NamedValue) (driver.Result, error) { + return s.conn.doExec(ctx, method, s.query, args) +} + +func (s *Stmt) doQuery(ctx context.Context, method string, args []driver.NamedValue) (driver.Rows, error) { + return s.conn.doQuery(ctx, method, s.query, args) +} + +type fsBase struct{ s *Stmt } + +func (v fsBase) Close() error { return v.s.doClose() } +func (v fsBase) NumInput() int { return v.s.doNumInput() } +func (v fsBase) Exec(args []driver.Value) (driver.Result, error) { + return v.s.doExec(context.Background(), "StmtExec", namedToValues(args)) +} +func (v fsBase) Query(args []driver.Value) (driver.Rows, error) { + return v.s.doQuery(context.Background(), "StmtQuery", namedToValues(args)) +} + +type fsExecerCtx struct{ s *Stmt } + +func (v fsExecerCtx) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return v.s.doExec(ctx, "StmtExecContext", args) +} + +type fsQueryerCtx struct{ s *Stmt } + +func (v fsQueryerCtx) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return v.s.doQuery(ctx, "StmtQueryContext", args) +} + +type fsColConv struct{ s *Stmt } + +func (v fsColConv) ColumnConverter(idx int) driver.ValueConverter { + return driver.DefaultParameterConverter +} + +type fsNVChecker struct{ s *Stmt } + +func (v fsNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return v.s.conn.doCheckNamedValue(nv) +} + +// Tx is the fake transaction. +type Tx struct{ conn *Conn } + +func (t *Tx) Commit() error { + t.conn.log("Commit", "") + return nil +} + +func (t *Tx) Rollback() error { + t.conn.log("Rollback", "") + return nil +} + +// Rows is the fake result set. +type Rows struct { + cols []string + rows [][]driver.Value + pos int +} + +func (r *Rows) Columns() []string { return r.cols } +func (r *Rows) Close() error { return nil } + +func (r *Rows) Next(dest []driver.Value) error { + if r.pos >= len(r.rows) { + return io.EOF + } + copy(dest, r.rows[r.pos]) + r.pos++ + return nil +} + +// Call is one recorded driver operation. +type Call struct { + ConnID int + Method string + Detail string +} + +// CallLog records driver operations across all conns of a Driver. +type CallLog struct { + mu sync.Mutex + calls []Call +} + +func (l *CallLog) add(connID int, method, detail string) { + l.mu.Lock() + defer l.mu.Unlock() + l.calls = append(l.calls, Call{ConnID: connID, Method: method, Detail: detail}) +} + +// Calls returns a snapshot of all recorded calls in order. +func (l *CallLog) Calls() []Call { + l.mu.Lock() + defer l.mu.Unlock() + out := make([]Call, len(l.calls)) + copy(out, l.calls) + return out +} + +// Count returns how many times the named method was called. +func (l *CallLog) Count(method string) int { + l.mu.Lock() + defer l.mu.Unlock() + n := 0 + for _, c := range l.calls { + if c.Method == method { + n++ + } + } + return n +} + +// Reset clears the log. +func (l *CallLog) Reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.calls = nil +} + +// Strategy is a fake hotload strategy driven by tests. It matches +// hotload.Strategy structurally. +type Strategy struct { + mu sync.Mutex + initial map[string]string + chans map[string]chan string + watches int +} + +// NewStrategy returns a Strategy that answers Watch with the given initial +// values keyed by path. +func NewStrategy(initial map[string]string) *Strategy { + cp := make(map[string]string, len(initial)) + for k, v := range initial { + cp[k] = v + } + return &Strategy{initial: cp, chans: make(map[string]chan string)} +} + +// Push delivers a new value to the watcher of path. It blocks until the +// hotload run loop receives it, which makes change injection deterministic. +func (s *Strategy) Push(path, value string) { + s.mu.Lock() + ch := s.chans[path] + s.mu.Unlock() + if ch == nil { + panic("dbfake: Push on unwatched path " + path) + } + ch <- value +} + +// CloseChan closes the update channel for path, simulating a strategy that +// stops watching. +func (s *Strategy) CloseChan(path string) { + s.mu.Lock() + defer s.mu.Unlock() + if ch := s.chans[path]; ch != nil { + close(ch) + delete(s.chans, path) + } +} + +// Watches reports how many watches are currently open. +func (s *Strategy) Watches() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.watches +} + +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, <-chan string, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.initial[pth] + if !ok { + return "", nil, fmt.Errorf("dbfake: no initial value for path %q", pth) + } + ch, ok := s.chans[pth] + if !ok { + ch = make(chan string) + s.chans[pth] = ch + } + s.watches++ + return value, ch, nil +} + +func (s *Strategy) CloseWatch(pth string, pathQry string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.watches-- + if ch := s.chans[pth]; ch != nil { + close(ch) + delete(s.chans, pth) + } + return nil +} + +func (s *Strategy) Close() { + s.mu.Lock() + defer s.mu.Unlock() + for p, ch := range s.chans { + close(ch) + delete(s.chans, p) + } + s.watches = 0 +} diff --git a/internal/dbfake/views_gen.go b/internal/dbfake/views_gen.go new file mode 100644 index 0000000..c3fedbc --- /dev/null +++ b/internal/dbfake/views_gen.go @@ -0,0 +1,1403 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package dbfake + +import "database/sql/driver" + +type fconn_bare struct { + fvBase +} + +type fconn_e struct { + fvBase + fvExecer +} + +type fconn_E struct { + fvBase + fvExecerCtx +} + +type fconn_eE struct { + fvBase + fvExecer + fvExecerCtx +} + +type fconn_q struct { + fvBase + fvQueryer +} + +type fconn_eq struct { + fvBase + fvExecer + fvQueryer +} + +type fconn_Eq struct { + fvBase + fvExecerCtx + fvQueryer +} + +type fconn_eEq struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer +} + +type fconn_Q struct { + fvBase + fvQueryerCtx +} + +type fconn_eQ struct { + fvBase + fvExecer + fvQueryerCtx +} + +type fconn_EQ struct { + fvBase + fvExecerCtx + fvQueryerCtx +} + +type fconn_eEQ struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx +} + +type fconn_qQ struct { + fvBase + fvQueryer + fvQueryerCtx +} + +type fconn_eqQ struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx +} + +type fconn_EqQ struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx +} + +type fconn_eEqQ struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx +} + +type fconn_P struct { + fvBase + fvPinger +} + +type fconn_eP struct { + fvBase + fvExecer + fvPinger +} + +type fconn_EP struct { + fvBase + fvExecerCtx + fvPinger +} + +type fconn_eEP struct { + fvBase + fvExecer + fvExecerCtx + fvPinger +} + +type fconn_qP struct { + fvBase + fvQueryer + fvPinger +} + +type fconn_eqP struct { + fvBase + fvExecer + fvQueryer + fvPinger +} + +type fconn_EqP struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger +} + +type fconn_eEqP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger +} + +type fconn_QP struct { + fvBase + fvQueryerCtx + fvPinger +} + +type fconn_eQP struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger +} + +type fconn_EQP struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger +} + +type fconn_eEQP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger +} + +type fconn_qQP struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_eqQP struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_EqQP struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_eEqQP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_N struct { + fvBase + fvNVChecker +} + +type fconn_eN struct { + fvBase + fvExecer + fvNVChecker +} + +type fconn_EN struct { + fvBase + fvExecerCtx + fvNVChecker +} + +type fconn_eEN struct { + fvBase + fvExecer + fvExecerCtx + fvNVChecker +} + +type fconn_qN struct { + fvBase + fvQueryer + fvNVChecker +} + +type fconn_eqN struct { + fvBase + fvExecer + fvQueryer + fvNVChecker +} + +type fconn_EqN struct { + fvBase + fvExecerCtx + fvQueryer + fvNVChecker +} + +type fconn_eEqN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvNVChecker +} + +type fconn_QN struct { + fvBase + fvQueryerCtx + fvNVChecker +} + +type fconn_eQN struct { + fvBase + fvExecer + fvQueryerCtx + fvNVChecker +} + +type fconn_EQN struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvNVChecker +} + +type fconn_eEQN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvNVChecker +} + +type fconn_qQN struct { + fvBase + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_eqQN struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_EqQN struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_eEqQN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_PN struct { + fvBase + fvPinger + fvNVChecker +} + +type fconn_ePN struct { + fvBase + fvExecer + fvPinger + fvNVChecker +} + +type fconn_EPN struct { + fvBase + fvExecerCtx + fvPinger + fvNVChecker +} + +type fconn_eEPN struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvNVChecker +} + +type fconn_qPN struct { + fvBase + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_eqPN struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_EqPN struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_eEqPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_QPN struct { + fvBase + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eQPN struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_EQPN struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eEQPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_qQPN struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eqQPN struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_EqQPN struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eEqQPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_A struct { + fvBase + fvClassA +} + +type fconn_eA struct { + fvBase + fvExecer + fvClassA +} + +type fconn_EA struct { + fvBase + fvExecerCtx + fvClassA +} + +type fconn_eEA struct { + fvBase + fvExecer + fvExecerCtx + fvClassA +} + +type fconn_qA struct { + fvBase + fvQueryer + fvClassA +} + +type fconn_eqA struct { + fvBase + fvExecer + fvQueryer + fvClassA +} + +type fconn_EqA struct { + fvBase + fvExecerCtx + fvQueryer + fvClassA +} + +type fconn_eEqA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvClassA +} + +type fconn_QA struct { + fvBase + fvQueryerCtx + fvClassA +} + +type fconn_eQA struct { + fvBase + fvExecer + fvQueryerCtx + fvClassA +} + +type fconn_EQA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvClassA +} + +type fconn_eEQA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvClassA +} + +type fconn_qQA struct { + fvBase + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_eqQA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_EqQA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_eEqQA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_PA struct { + fvBase + fvPinger + fvClassA +} + +type fconn_ePA struct { + fvBase + fvExecer + fvPinger + fvClassA +} + +type fconn_EPA struct { + fvBase + fvExecerCtx + fvPinger + fvClassA +} + +type fconn_eEPA struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvClassA +} + +type fconn_qPA struct { + fvBase + fvQueryer + fvPinger + fvClassA +} + +type fconn_eqPA struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvClassA +} + +type fconn_EqPA struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvClassA +} + +type fconn_eEqPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvClassA +} + +type fconn_QPA struct { + fvBase + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eQPA struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_EQPA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eEQPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_qQPA struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eqQPA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_EqQPA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eEqQPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_NA struct { + fvBase + fvNVChecker + fvClassA +} + +type fconn_eNA struct { + fvBase + fvExecer + fvNVChecker + fvClassA +} + +type fconn_ENA struct { + fvBase + fvExecerCtx + fvNVChecker + fvClassA +} + +type fconn_eENA struct { + fvBase + fvExecer + fvExecerCtx + fvNVChecker + fvClassA +} + +type fconn_qNA struct { + fvBase + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_eqNA struct { + fvBase + fvExecer + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_EqNA struct { + fvBase + fvExecerCtx + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_eEqNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_QNA struct { + fvBase + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eQNA struct { + fvBase + fvExecer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_EQNA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eEQNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_qQNA struct { + fvBase + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eqQNA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_EqQNA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eEqQNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_PNA struct { + fvBase + fvPinger + fvNVChecker + fvClassA +} + +type fconn_ePNA struct { + fvBase + fvExecer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EPNA struct { + fvBase + fvExecerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEPNA struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_qPNA struct { + fvBase + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eqPNA struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EqPNA struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEqPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_QPNA struct { + fvBase + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eQPNA struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EQPNA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEQPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_qQPNA struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eqQPNA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EqQPNA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEqQPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fstmt_bare struct { + fsBase +} + +type fstmt_E struct { + fsBase + fsExecerCtx +} + +type fstmt_Q struct { + fsBase + fsQueryerCtx +} + +type fstmt_EQ struct { + fsBase + fsExecerCtx + fsQueryerCtx +} + +type fstmt_C struct { + fsBase + fsColConv +} + +type fstmt_EC struct { + fsBase + fsExecerCtx + fsColConv +} + +type fstmt_QC struct { + fsBase + fsQueryerCtx + fsColConv +} + +type fstmt_EQC struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsColConv +} + +type fstmt_N struct { + fsBase + fsNVChecker +} + +type fstmt_EN struct { + fsBase + fsExecerCtx + fsNVChecker +} + +type fstmt_QN struct { + fsBase + fsQueryerCtx + fsNVChecker +} + +type fstmt_EQN struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsNVChecker +} + +type fstmt_CN struct { + fsBase + fsColConv + fsNVChecker +} + +type fstmt_ECN struct { + fsBase + fsExecerCtx + fsColConv + fsNVChecker +} + +type fstmt_QCN struct { + fsBase + fsQueryerCtx + fsColConv + fsNVChecker +} + +type fstmt_EQCN struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsColConv + fsNVChecker +} + +// viewConn returns c behind a view exposing exactly the optional +// interfaces selected by c.caps. The four interfaces hotload always +// synthesizes (ConnPrepareContext, ConnBeginTx, SessionResetter, +// Validator) are grouped: if any of their caps is set the view exposes +// all four. +func viewConn(c *Conn) driver.Conn { + var f int + if c.caps&CapExecer != 0 { + f |= 1 << 0 + } + if c.caps&CapExecerContext != 0 { + f |= 1 << 1 + } + if c.caps&CapQueryer != 0 { + f |= 1 << 2 + } + if c.caps&CapQueryerContext != 0 { + f |= 1 << 3 + } + if c.caps&CapPinger != 0 { + f |= 1 << 4 + } + if c.caps&CapNamedValueChecker != 0 { + f |= 1 << 5 + } + if c.caps&(CapConnPrepareContext|CapConnBeginTx|CapSessionResetter|CapValidator) != 0 { + f |= 1 << 6 + } + switch f { + case 0: + return fconn_bare{fvBase{c}} + case 1: + return fconn_e{fvBase{c}, fvExecer{c}} + case 2: + return fconn_E{fvBase{c}, fvExecerCtx{c}} + case 3: + return fconn_eE{fvBase{c}, fvExecer{c}, fvExecerCtx{c}} + case 4: + return fconn_q{fvBase{c}, fvQueryer{c}} + case 5: + return fconn_eq{fvBase{c}, fvExecer{c}, fvQueryer{c}} + case 6: + return fconn_Eq{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}} + case 7: + return fconn_eEq{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}} + case 8: + return fconn_Q{fvBase{c}, fvQueryerCtx{c}} + case 9: + return fconn_eQ{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}} + case 10: + return fconn_EQ{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}} + case 11: + return fconn_eEQ{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}} + case 12: + return fconn_qQ{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 13: + return fconn_eqQ{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 14: + return fconn_EqQ{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 15: + return fconn_eEqQ{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 16: + return fconn_P{fvBase{c}, fvPinger{c}} + case 17: + return fconn_eP{fvBase{c}, fvExecer{c}, fvPinger{c}} + case 18: + return fconn_EP{fvBase{c}, fvExecerCtx{c}, fvPinger{c}} + case 19: + return fconn_eEP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}} + case 20: + return fconn_qP{fvBase{c}, fvQueryer{c}, fvPinger{c}} + case 21: + return fconn_eqP{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}} + case 22: + return fconn_EqP{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}} + case 23: + return fconn_eEqP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}} + case 24: + return fconn_QP{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}} + case 25: + return fconn_eQP{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 26: + return fconn_EQP{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}} + case 27: + return fconn_eEQP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}} + case 28: + return fconn_qQP{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 29: + return fconn_eqQP{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 30: + return fconn_EqQP{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 31: + return fconn_eEqQP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 32: + return fconn_N{fvBase{c}, fvNVChecker{c}} + case 33: + return fconn_eN{fvBase{c}, fvExecer{c}, fvNVChecker{c}} + case 34: + return fconn_EN{fvBase{c}, fvExecerCtx{c}, fvNVChecker{c}} + case 35: + return fconn_eEN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvNVChecker{c}} + case 36: + return fconn_qN{fvBase{c}, fvQueryer{c}, fvNVChecker{c}} + case 37: + return fconn_eqN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvNVChecker{c}} + case 38: + return fconn_EqN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}} + case 39: + return fconn_eEqN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}} + case 40: + return fconn_QN{fvBase{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 41: + return fconn_eQN{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 42: + return fconn_EQN{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 43: + return fconn_eEQN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 44: + return fconn_qQN{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 45: + return fconn_eqQN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 46: + return fconn_EqQN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 47: + return fconn_eEqQN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 48: + return fconn_PN{fvBase{c}, fvPinger{c}, fvNVChecker{c}} + case 49: + return fconn_ePN{fvBase{c}, fvExecer{c}, fvPinger{c}, fvNVChecker{c}} + case 50: + return fconn_EPN{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 51: + return fconn_eEPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 52: + return fconn_qPN{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 53: + return fconn_eqPN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 54: + return fconn_EqPN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 55: + return fconn_eEqPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 56: + return fconn_QPN{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 57: + return fconn_eQPN{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 58: + return fconn_EQPN{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 59: + return fconn_eEQPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 60: + return fconn_qQPN{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 61: + return fconn_eqQPN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 62: + return fconn_EqQPN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 63: + return fconn_eEqQPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 64: + return fconn_A{fvBase{c}, fvClassA{c}} + case 65: + return fconn_eA{fvBase{c}, fvExecer{c}, fvClassA{c}} + case 66: + return fconn_EA{fvBase{c}, fvExecerCtx{c}, fvClassA{c}} + case 67: + return fconn_eEA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvClassA{c}} + case 68: + return fconn_qA{fvBase{c}, fvQueryer{c}, fvClassA{c}} + case 69: + return fconn_eqA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvClassA{c}} + case 70: + return fconn_EqA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvClassA{c}} + case 71: + return fconn_eEqA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvClassA{c}} + case 72: + return fconn_QA{fvBase{c}, fvQueryerCtx{c}, fvClassA{c}} + case 73: + return fconn_eQA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 74: + return fconn_EQA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvClassA{c}} + case 75: + return fconn_eEQA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvClassA{c}} + case 76: + return fconn_qQA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 77: + return fconn_eqQA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 78: + return fconn_EqQA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 79: + return fconn_eEqQA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 80: + return fconn_PA{fvBase{c}, fvPinger{c}, fvClassA{c}} + case 81: + return fconn_ePA{fvBase{c}, fvExecer{c}, fvPinger{c}, fvClassA{c}} + case 82: + return fconn_EPA{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvClassA{c}} + case 83: + return fconn_eEPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvClassA{c}} + case 84: + return fconn_qPA{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 85: + return fconn_eqPA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 86: + return fconn_EqPA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 87: + return fconn_eEqPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 88: + return fconn_QPA{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 89: + return fconn_eQPA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 90: + return fconn_EQPA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 91: + return fconn_eEQPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 92: + return fconn_qQPA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 93: + return fconn_eqQPA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 94: + return fconn_EqQPA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 95: + return fconn_eEqQPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 96: + return fconn_NA{fvBase{c}, fvNVChecker{c}, fvClassA{c}} + case 97: + return fconn_eNA{fvBase{c}, fvExecer{c}, fvNVChecker{c}, fvClassA{c}} + case 98: + return fconn_ENA{fvBase{c}, fvExecerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 99: + return fconn_eENA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 100: + return fconn_qNA{fvBase{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 101: + return fconn_eqNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 102: + return fconn_EqNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 103: + return fconn_eEqNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 104: + return fconn_QNA{fvBase{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 105: + return fconn_eQNA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 106: + return fconn_EQNA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 107: + return fconn_eEQNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 108: + return fconn_qQNA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 109: + return fconn_eqQNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 110: + return fconn_EqQNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 111: + return fconn_eEqQNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 112: + return fconn_PNA{fvBase{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 113: + return fconn_ePNA{fvBase{c}, fvExecer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 114: + return fconn_EPNA{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 115: + return fconn_eEPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 116: + return fconn_qPNA{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 117: + return fconn_eqPNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 118: + return fconn_EqPNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 119: + return fconn_eEqPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 120: + return fconn_QPNA{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 121: + return fconn_eQPNA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 122: + return fconn_EQPNA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 123: + return fconn_eEQPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 124: + return fconn_qQPNA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 125: + return fconn_eqQPNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 126: + return fconn_EqQPNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 127: + return fconn_eEqQPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + } + panic("unreachable") +} + +// viewStmt returns s behind a view exposing exactly the optional stmt +// interfaces selected by the conn's caps. +func viewStmt(s *Stmt) driver.Stmt { + var f int + if s.conn.caps&CapStmtExecContext != 0 { + f |= 1 << 0 + } + if s.conn.caps&CapStmtQueryContext != 0 { + f |= 1 << 1 + } + if s.conn.caps&CapColumnConverter != 0 { + f |= 1 << 2 + } + if s.conn.caps&CapStmtNamedValueChecker != 0 { + f |= 1 << 3 + } + switch f { + case 0: + return fstmt_bare{fsBase{s}} + case 1: + return fstmt_E{fsBase{s}, fsExecerCtx{s}} + case 2: + return fstmt_Q{fsBase{s}, fsQueryerCtx{s}} + case 3: + return fstmt_EQ{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}} + case 4: + return fstmt_C{fsBase{s}, fsColConv{s}} + case 5: + return fstmt_EC{fsBase{s}, fsExecerCtx{s}, fsColConv{s}} + case 6: + return fstmt_QC{fsBase{s}, fsQueryerCtx{s}, fsColConv{s}} + case 7: + return fstmt_EQC{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsColConv{s}} + case 8: + return fstmt_N{fsBase{s}, fsNVChecker{s}} + case 9: + return fstmt_EN{fsBase{s}, fsExecerCtx{s}, fsNVChecker{s}} + case 10: + return fstmt_QN{fsBase{s}, fsQueryerCtx{s}, fsNVChecker{s}} + case 11: + return fstmt_EQN{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsNVChecker{s}} + case 12: + return fstmt_CN{fsBase{s}, fsColConv{s}, fsNVChecker{s}} + case 13: + return fstmt_ECN{fsBase{s}, fsExecerCtx{s}, fsColConv{s}, fsNVChecker{s}} + case 14: + return fstmt_QCN{fsBase{s}, fsQueryerCtx{s}, fsColConv{s}, fsNVChecker{s}} + case 15: + return fstmt_EQCN{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsColConv{s}, fsNVChecker{s}} + } + panic("unreachable") +} diff --git a/internal/gen/main.go b/internal/gen/main.go new file mode 100644 index 0000000..3e99325 --- /dev/null +++ b/internal/gen/main.go @@ -0,0 +1,279 @@ +// Command gen generates the optional-interface combination wrappers for +// hotload (conn_combos_gen.go, stmt_combos_gen.go) and the capability views +// for the dbfake test driver (internal/dbfake/views_gen.go). +// +// Hotload must expose an optional driver interface on a wrapped conn or stmt +// if and only if the underlying object implements it: database/sql discovers +// capabilities with type assertions, so the method must actually exist or +// not exist. One struct per interface subset is required for that, and this +// program emits them so adding an interface later is a regen instead of +// hand-editing dozens of types. +// +// It is invoked from the repository root via go:generate (see conn.go). +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "os" +) + +// pick names the piece type embedded for one optional interface, the letter +// used in generated type-name suffixes, and the interfaces the combination +// gains when the piece is present. +type pick struct { + letter string + piece string + ifaces []string +} + +func main() { + writeFile("conn_combos_gen.go", genHotloadCombos( + "conn", "*baseConn", + "// wrapConn wraps b so that the returned driver.Conn exposes an optional\n"+ + "// interface if and only if the underlying conn supports the capability.\n"+ + "// The legacy Execer/Queryer interfaces are collapsed into their context\n"+ + "// flavors: the wrapper only ever exposes ExecerContext/QueryerContext and\n"+ + "// replicates database/sql's legacy fallback internally (see\n"+ + "// baseConn.execContext and baseConn.queryContext).\n"+ + "func connFlags(c driver.Conn) uint8 {\n"+ + " var f uint8\n"+ + " if _, ok := c.(driver.ExecerContext); ok {\n"+ + " f |= 1 << 0\n"+ + " } else if _, ok := c.(driver.Execer); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 0\n"+ + " }\n"+ + " if _, ok := c.(driver.QueryerContext); ok {\n"+ + " f |= 1 << 1\n"+ + " } else if _, ok := c.(driver.Queryer); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 1\n"+ + " }\n"+ + " if _, ok := c.(driver.Pinger); ok {\n"+ + " f |= 1 << 2\n"+ + " }\n"+ + " if _, ok := c.(driver.NamedValueChecker); ok {\n"+ + " f |= 1 << 3\n"+ + " }\n"+ + " return f\n"+ + "}\n", + "wrapConn", "b", "driver.Conn", "connFlags(b.inner)", + []pick{ + {"E", "cExecer", []string{"driver.ExecerContext"}}, + {"Q", "cQueryer", []string{"driver.QueryerContext"}}, + {"P", "cPinger", []string{"driver.Pinger"}}, + {"N", "cNVChecker", []string{"driver.NamedValueChecker"}}, + }, + )) + + writeFile("stmt_combos_gen.go", genHotloadCombos( + "stmt", "*baseStmt", + "// stmtFlags reports which optional interfaces the wrapped stmt must expose.\n"+ + "func stmtFlags(s driver.Stmt) uint8 {\n"+ + " var f uint8\n"+ + " if _, ok := s.(driver.StmtExecContext); ok {\n"+ + " f |= 1 << 0\n"+ + " }\n"+ + " if _, ok := s.(driver.StmtQueryContext); ok {\n"+ + " f |= 1 << 1\n"+ + " }\n"+ + " if _, ok := s.(driver.ColumnConverter); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 2\n"+ + " }\n"+ + " if _, ok := s.(driver.NamedValueChecker); ok {\n"+ + " f |= 1 << 3\n"+ + " }\n"+ + " return f\n"+ + "}\n", + "wrapStmt", "b", "driver.Stmt", "stmtFlags(b.inner)", + []pick{ + {"E", "sExecer", []string{"driver.StmtExecContext"}}, + {"Q", "sQueryer", []string{"driver.StmtQueryContext"}}, + {"C", "sColConv", []string{"driver.ColumnConverter"}}, + {"N", "sNVChecker", []string{"driver.NamedValueChecker"}}, + }, + )) + + writeFile("internal/dbfake/views_gen.go", genDbfakeViews()) +} + +func writeFile(path string, src []byte) { + formatted, err := format.Source(src) + if err != nil { + log.Fatalf("formatting %s: %v\n%s", path, err, src) + } + if err := os.WriteFile(path, formatted, 0o644); err != nil { + log.Fatalf("writing %s: %v", path, err) + } + fmt.Printf("wrote %s\n", path) +} + +func header(pkg string) string { + return "// Code generated by internal/gen. DO NOT EDIT.\n\npackage " + pkg + "\n\nimport \"database/sql/driver\"\n\n" +} + +// suffix builds the type-name suffix for a flag mask, e.g. mask 0b0011 over +// conn picks -> "EQ". +func suffix(mask int, picks []pick) string { + s := "" + for i, p := range picks { + if mask&(1< 0 { - return errors.New(diffStr) - } - return nil -} diff --git a/internal/secret_sink.go b/internal/secret_sink.go index 8afa669..48e2180 100644 --- a/internal/secret_sink.go +++ b/internal/secret_sink.go @@ -1,10 +1,10 @@ package internal import ( + "crypto/rand" + "encoding/hex" "sync" "time" - - "github.com/google/uuid" ) const ( @@ -69,11 +69,11 @@ func (rss *RandomSecretSink) Add(actualSecret string) (randomSecret string, err delete(rss.secretStore, oldestData.actualSecret) } - guid, err := uuid.NewRandom() - if err != nil { + buf := make([]byte, randomLen/2) + if _, err := rand.Read(buf); err != nil { return "", err } - randomSecret = guid.String()[:randomLen] + randomSecret = hex.EncodeToString(buf) secretData = &RandomSecretData{ actualSecret: actualSecret, diff --git a/internal/testutil/leak.go b/internal/testutil/leak.go new file mode 100644 index 0000000..412f3e8 --- /dev/null +++ b/internal/testutil/leak.go @@ -0,0 +1,87 @@ +// Package testutil holds small stdlib-only helpers for hotload's tests. +package testutil + +import ( + "runtime" + "strings" + "testing" + "time" +) + +// NoLeaks registers a cleanup that fails the test if goroutines are still +// running once the test — including cleanups registered after this call — +// has finished. Call it first in the test, before opening any resources, so +// its cleanup runs last (cleanups run in LIFO order). +// +// Transient goroutines get a grace period: the check retries until the +// stragglers exit or a deadline passes. +func NoLeaks(t *testing.T) { + t.Helper() + t.Cleanup(func() { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + var extra []string + for { + extra = leakedGoroutines() + if len(extra) == 0 { + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(5 * time.Millisecond) + } + t.Errorf("leaked %d goroutine(s):\n%s", len(extra), strings.Join(extra, "\n\n")) + }) +} + +// allowedStackFragments mark goroutines that belong to the test harness or +// the runtime rather than the code under test. +var allowedStackFragments = []string{ + "testing.", // test runner goroutines (tRunner, parallel subtests, …) + "runtime.goexit", // never matches a frame list alone; kept for safety + "os/signal.", + "runtime/trace.", + "runtime.ReadTrace", +} + +func leakedGoroutines() []string { + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + stacks := strings.Split(strings.TrimSpace(string(buf[:n])), "\n\n") + + var out []string + for i, s := range stacks { + if i == 0 { + // The goroutine running this check. + continue + } + if isAllowed(s) { + continue + } + out = append(out, s) + } + return out +} + +func isAllowed(stack string) bool { + for _, frag := range allowedStackFragments { + if strings.Contains(stack, frag) { + return true + } + } + return false +} + +// WaitFor polls cond every millisecond until it returns true or the timeout +// elapses, failing the test in the latter case. +func WaitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out after %v waiting for %s", timeout, what) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/url_test.go b/internal/url_test.go index 48ab8a4..ee5089f 100644 --- a/internal/url_test.go +++ b/internal/url_test.go @@ -3,43 +3,61 @@ package internal import ( "fmt" "net/url" + "regexp" + "strings" "testing" - - "github.com/google/uuid" ) func TestRedactUrl(t *testing.T) { - nrr := NewNonRandomReader(1) - uuid.SetRand(nrr) - testcases := []struct { - inputDsn string - expectDsn string + inputDsn string + // expectPattern matches the redacted DSN; the password is replaced + // with a random token, asserted separately. + expectPattern string }{ { - inputDsn: "qwerty", - expectDsn: "//u---r:01020304@qwerty", + inputDsn: "qwerty", + expectPattern: `^//u---r:[0-9a-f]{8}@qwerty$`, }, { - inputDsn: "mysql://u:p@amazon.rds.com:5432/contacts", - expectDsn: "mysql://u---u:11121314@amazon.rds.com:5432/contacts", + inputDsn: "mysql://u:p@amazon.rds.com:5432/contacts", + expectPattern: `^mysql://u---u:[0-9a-f]{8}@amazon\.rds\.com:5432/contacts$`, }, { - inputDsn: "postgresql://admin:test@localhost:5432/hotload_test?sslmode=disable", - expectDsn: "postgresql://a---n:21222324@localhost:5432/hotload_test?sslmode=disable", + inputDsn: "postgresql://admin:test@localhost:5432/hotload_test?sslmode=disable", + expectPattern: `^postgresql://a---n:[0-9a-f]{8}@localhost:5432/hotload_test\?sslmode=disable$`, }, } for _, tt := range testcases { t.Run(tt.inputDsn, func(t *testing.T) { gotDsn := RedactUrl(tt.inputDsn) - if gotDsn != tt.expectDsn { - t.Errorf("expectDsn='%s' gotDsn='%s'", tt.expectDsn, gotDsn) + re := regexp.MustCompile(tt.expectPattern) + if !re.MatchString(gotDsn) { + t.Errorf("RedactUrl(%q) = %q, want match for %q", tt.inputDsn, gotDsn, tt.expectPattern) + } + if strings.Contains(gotDsn, ":test@") { + t.Errorf("RedactUrl(%q) = %q leaked the password", tt.inputDsn, gotDsn) } }) } } +// TestRedactUrlStablePassword verifies that the same password maps to the +// same random token across calls, so log lines remain correlatable. +func TestRedactUrlStablePassword(t *testing.T) { + first := RedactUrl("postgresql://admin:hunter2@localhost/db") + second := RedactUrl("postgresql://admin:hunter2@localhost/db") + if first != second { + t.Errorf("redaction not stable for identical input: %q vs %q", first, second) + } + + other := RedactUrl("postgresql://admin:different@localhost/db") + if other == first { + t.Errorf("different passwords redacted to the same DSN: %q", other) + } +} + func TestUrlEncodedQueryParams(t *testing.T) { testcases := []struct { inputParams url.Values diff --git a/lifecycle_test.go b/lifecycle_test.go new file mode 100644 index 0000000..93dec17 --- /dev/null +++ b/lifecycle_test.go @@ -0,0 +1,346 @@ +package hotload_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/dbfake" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// TestGracefulSwap: after a config change without forceKill, the pool +// discards old-generation conns on reuse and dials the new DSN; the old +// underlying conn is closed exactly once. +func TestGracefulSwap(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + + fx.pushAndWait("dsn-2") + + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN after change = %q, want dsn-2", got) + } + + conns := fx.drv.Conns() + if len(conns) < 2 { + t.Fatalf("expected a second conn to be dialed, got %d conns", len(conns)) + } + old := conns[0] + testutil.WaitFor(t, 2*time.Second, "old conn to close", old.Closed) + if n := old.CloseCount(); n != 1 { + t.Errorf("old conn closed %d times, want exactly 1", n) + } +} + +// TestGracefulGracePeriod: in graceful mode the previous generation's idle +// conns survive one change (the grace period) and are killed on the next +// change, without the pool ever touching them. +func TestGracefulGracePeriod(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() // dial conn 1 on dsn-1 + c1 := fx.drv.Conns()[0] + + fx.pushAndWait("dsn-2") + // Grace period: conn 1 must not be force-closed by this change. + time.Sleep(20 * time.Millisecond) + if c1.Closed() { + t.Fatal("conn from previous generation was closed during its grace period") + } + + fx.pushAndWait("dsn-3") + // Now conn 1 belonged to the generation before the previous one — killed. + testutil.WaitFor(t, 2*time.Second, "prev-prev conn to close", c1.Closed) + if n := c1.CloseCount(); n != 1 { + t.Errorf("conn closed %d times, want exactly 1", n) + } + + if got := fx.queryDSN(); got != "dsn-3" { + t.Fatalf("queryDSN = %q, want dsn-3", got) + } +} + +// TestSwapBackToOriginal: a -> b -> a is two real changes and ends with +// fresh conns on the original DSN. +func TestSwapBackToOriginal(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.pushAndWait("dsn-2") + ev := fx.pushAndWait("dsn-1") + if ev.GroupName != fx.dsn { + t.Errorf("event group = %q, want %q", ev.GroupName, fx.dsn) + } + + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + if len(fx.drv.Conns()) < 2 { + t.Errorf("expected a fresh conn after swapping back, got %d conns", len(fx.drv.Conns())) + } +} + +// TestUnchangedValueIgnored: pushing the same value is not a change. +func TestUnchangedValueIgnored(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.push("dsn-1") // identical; processed but ignored + fx.pushAndWait("dsn-2") + fx.noPendingChange() + + c1 := fx.drv.Conns()[0] + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } + _ = c1 +} + +// TestForceKillCancelsInflightExec: with forceKill, an exec blocked in the +// driver is canceled by the config change, the error surfaces as ErrHotSwap, +// and the conn is closed exactly once. +func TestForceKillCancelsInflightExec(t *testing.T) { + testutil.NoLeaks(t) + + started := make(chan struct{}, 10) + fx := newFixture(t, fxCfg{ + params: "forceKill=true", + execFn: func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + started <- struct{}{} + <-ctx.Done() + return nil, context.Cause(ctx) + }, + }) + + errCh := make(chan error, 1) + go func() { + _, err := fx.db.Exec("UPDATE x") + errCh <- err + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("exec never reached the driver") + } + + fx.pushAndWait("dsn-2") + + select { + case err := <-errCh: + if !errors.Is(err, hotload.ErrHotSwap) { + t.Fatalf("exec error = %v, want ErrHotSwap", err) + } + case <-time.After(2 * time.Second): + t.Fatal("exec did not return after forceKill change") + } + + old := fx.drv.Conns()[0] + testutil.WaitFor(t, 2*time.Second, "old conn to close", old.Closed) + if n := old.CloseCount(); n != 1 { + t.Errorf("old conn closed %d times, want exactly 1", n) + } + + fx.drv.ExecFn = nil // restore default behavior for the new generation + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } +} + +// TestForceKillBoundedByKillWindow: a driver that ignores context +// cancellation cannot wedge the run loop — after killWindow elapses the conn +// is force-closed and new work proceeds on the new DSN while the old +// operation is still parked. +func TestForceKillBoundedByKillWindow(t *testing.T) { + testutil.NoLeaks(t) + + started := make(chan struct{}, 10) + release := make(chan struct{}) + fx := newFixture(t, fxCfg{ + params: "forceKill=true&killWindow=30ms", + execFn: func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + started <- struct{}{} + <-release // deliberately ignores ctx + return driver.RowsAffected(1), nil + }, + }) + defer close(release) + + errCh := make(chan error, 1) + go func() { + _, err := fx.db.Exec("UPDATE x") + errCh <- err + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("exec never reached the driver") + } + + fx.pushAndWait("dsn-2") + + old := fx.drv.Conns()[0] + testutil.WaitFor(t, 2*time.Second, "old conn to be force-closed", old.Closed) + + // The group must remain responsive with the old exec still parked. + fx.drv.ExecFn = nil + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } + select { + case err := <-errCh: + t.Logf("parked exec returned early: %v", err) + default: + // Still parked, as expected; released by the deferred close. + } +} + +// TestRunLoopNotBlockedBySlowDial: dials happen outside group locks, so a +// slow dial must not delay config-change processing (hotload v1 serialized +// these on one mutex). +func TestRunLoopNotBlockedBySlowDial(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + fx.drv.OpenDelay = 300 * time.Millisecond + + dialDone := make(chan struct{}) + go func() { + defer close(dialDone) + fx.queryDSN() // triggers the slow dial + }() + + time.Sleep(20 * time.Millisecond) // let the dial start + t0 := time.Now() + fx.pushAndWait("dsn-2") + if elapsed := time.Since(t0); elapsed > 200*time.Millisecond { + t.Errorf("config change took %v while a dial was in flight; the run loop appears blocked by dialing", elapsed) + } + <-dialDone +} + +// TestOpenErrors: bad DSNs and missing registrations fail at sql.Open (the +// connector is created there), with the sentinel errors preserved from v1. +func TestOpenErrors(t *testing.T) { + t.Run("unsupported strategy", func(t *testing.T) { + _, err := sql.Open("hotload", "nosuchstrategy://nosuchdriver/cfg") + if !errors.Is(err, hotload.ErrUnsupportedStrategy) { + t.Fatalf("err = %v, want ErrUnsupportedStrategy", err) + } + }) + + t.Run("unknown driver", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + // Reuse the registered strategy but point at an unregistered driver. + dsnURL := fmt.Sprintf("%s://%s%s", schemeOf(t, fx.dsn), "nosuchdriver", fixturePath) + _, err := sql.Open("hotload", dsnURL) + if !errors.Is(err, hotload.ErrUnknownDriver) { + t.Fatalf("err = %v, want ErrUnknownDriver", err) + } + }) + + t.Run("malformed killWindow", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + _, err := sql.Open("hotload", fx.dsn+"?killWindow=bogus") + if !errors.Is(err, hotload.ErrMalformedConnectionString) { + t.Fatalf("err = %v, want ErrMalformedConnectionString", err) + } + }) + + t.Run("watch error", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + _, err := sql.Open("hotload", fx.dsn+"nosuchpath") + if err == nil { + t.Fatal("expected watch error for unknown path") + } + }) +} + +// TestGroupTeardownAndSharing: two sql.DB handles on the same DSN share one +// watch; the watch survives the first Close and stops after the last one, +// leaving no goroutines behind. +func TestGroupTeardownAndSharing(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + db2, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatal(err) + } + + if n := fx.strat.Watches(); n != 1 { + t.Fatalf("watches = %d, want 1 (groups must be shared per DSN)", n) + } + + if err := db2.Close(); err != nil { + t.Fatal(err) + } + if n := fx.strat.Watches(); n != 1 { + t.Fatalf("watches after first close = %d, want 1", n) + } + + if err := fx.db.Close(); err != nil { + t.Fatal(err) + } + testutil.WaitFor(t, 2*time.Second, "watch teardown", func() bool { + return fx.strat.Watches() == 0 + }) +} + +// TestStrategyChannelClose: when the strategy closes its update channel the +// run loop exits, existing connections keep serving the last value, and a +// later db.Close still tears down cleanly. +func TestStrategyChannelClose(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.strat.CloseChan(fixturePath) + + // Existing conns keep working on the last known value. + testutil.WaitFor(t, 2*time.Second, "queries to keep working", func() bool { + return fx.queryDSN() == "dsn-1" + }) +} + +// TestValidatorEviction: an underlying conn reporting IsValid()==false is +// discarded by the pool and replaced. +func TestValidatorEviction(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + c1 := fx.drv.Conns()[0] + c1.SetInvalid() + + // The pool validates on put-back/reuse; the next queries must succeed + // on a fresh conn. + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + testutil.WaitFor(t, 2*time.Second, "invalid conn to be discarded", c1.Closed) +} + +func schemeOf(t *testing.T, dsn string) string { + t.Helper() + for i := range dsn { + if dsn[i] == ':' { + return dsn[:i] + } + } + t.Fatalf("no scheme in %q", dsn) + return "" +} diff --git a/metrics/pathchksum_metrics.go b/metrics/pathchksum_metrics.go deleted file mode 100644 index a77616a..0000000 --- a/metrics/pathchksum_metrics.go +++ /dev/null @@ -1,153 +0,0 @@ -package metrics - -import ( - "errors" - "hash/crc64" - "os" - "path" - "strings" - "sync" - "time" - - "github.com/colega/gaugefuncvec" - "github.com/infobloxopen/hotload/logger" - "github.com/prometheus/client_golang/prometheus" -) - -const ( - PathChksumMetricsEnableEnvVar = "HOTLOAD_PATH_CHKSUM_METRICS_ENABLE" -) - -var ( - ErrDuplicatePath = errors.New("duplicate path") - ErrPathNotFound = errors.New("path not found") - - HotloadPathChksumTimestampSecondsName = "hotload_path_chksum_timestamp_seconds" - HotloadPathChksumTimestampSecondsHelp = "Hotload path checksum last changed (unix timestamp), by path" - HotloadPathChksumTimestampSecondsGaugeFuncVec = gaugefuncvec.New(prometheus.GaugeOpts{ - Name: HotloadPathChksumTimestampSecondsName, - Help: HotloadPathChksumTimestampSecondsHelp, - }, []string{PathKey}) - - crc64Table = crc64.MakeTable(crc64.ECMA) - - defaultPathChksum *pathChksum -) - -func init() { - defaultPathChksum = newPathChksum(DefaultFileHasher) - prometheus.MustRegister(HotloadPathChksumTimestampSecondsGaugeFuncVec) -} - -// AddToDefaultPathChksum adds a path to the global defaultPathChksum for checksum metrics -func AddToDefaultPathChksum(pathStr string) error { - return defaultPathChksum.addPath(pathStr) -} - -type pathChksum struct { - sync.RWMutex // used to synchronize changes to the set of paths being monitored - enabled bool - fileHasher FileHasher - paths map[string]*pathRecord -} - -type pathRecord struct { - path string - crc64 uint64 - lastChanged int64 -} - -// Define FileHasher type so we can mock it for unit-testing -type FileHasher func(filePath string) (uint64, error) - -// DefaultFileHasher hashes file contents using CRC64 -func DefaultFileHasher(filePath string) (uint64, error) { - pathBytes, err := os.ReadFile(filePath) - if err != nil { - logger.ErrLogf("DefaultFileHasher", "ReadFile(%s) err=%s", filePath, err) - return 0, err - } - - newCrc64 := crc64.Checksum(pathBytes, crc64Table) - return newCrc64, nil -} - -// newPathChksum returns a new PathChksum -func newPathChksum(fileHasher FileHasher) *pathChksum { - if fileHasher == nil { - panic("nil FileHasher") - } - - enabledStr := strings.ToLower(strings.TrimSpace(os.Getenv(PathChksumMetricsEnableEnvVar))) - enabledFlg := false - switch enabledStr { - case "1", "true", "yes": - enabledFlg = true - } - - pthm := &pathChksum{ - enabled: enabledFlg, - fileHasher: fileHasher, - paths: make(map[string]*pathRecord), - } - return pthm -} - -// addPath adds a path to be checksum'd for change in contents, -// and registers path for metrics collection -func (pthm *pathChksum) addPath(pathStr string) error { - if !pthm.enabled { - return nil - } - - pathStr = CleanPath(pathStr) - - pthm.Lock() - defer pthm.Unlock() - - pathRec, found := pthm.paths[pathStr] - if found { - return ErrDuplicatePath - } - - pathRec = &pathRecord{ - path: pathStr, - } - pthm.paths[pathStr] = pathRec - - scraperFn := func() float64 { - if !pthm.enabled { - return float64(0) - } - - newCrc64, err := pthm.fileHasher(pathRec.path) - if err != nil { - // log error, but continue - logger.ErrLogf("PathChksum.scraper", "fileHasher(%s) err=%s", pathRec.path, err) - } else if pathRec.crc64 != newCrc64 { - pathRec.crc64 = newCrc64 - pathRec.lastChanged = time.Now().Unix() - } - - return float64(pathRec.lastChanged) - } - - HotloadPathChksumTimestampSecondsGaugeFuncVec.MustRegister( - prometheus.Labels{PathKey: pathStr}, - scraperFn, - ) - - return nil -} - -// CleanPath cleans and trimspaces path strings -func CleanPath(pathStr string) string { - return path.Clean(strings.TrimSpace(pathStr)) -} - -var ExpectHotloadPathChksumTimestampSecondsPreamble = ` -# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path -# TYPE hotload_path_chksum_timestamp_seconds gauge` - -var ExpectHotloadPathChksumTimestampSecondsRegexp = ` -hotload_path_chksum_timestamp_seconds{path="%s"} \d\.\d+e\+\d+` diff --git a/metrics/pathchksum_test.go b/metrics/pathchksum_test.go deleted file mode 100644 index 3810f34..0000000 --- a/metrics/pathchksum_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package metrics - -import ( - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/infobloxopen/hotload/internal" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -var delayDur = 616789 * time.Microsecond - -func MyTestFileHasher(filePath string) (uint64, error) { - return uint64(time.Now().UnixMicro()), nil -} - -func TestPathChksumMetricsDisabled(t *testing.T) { - os.Unsetenv(PathChksumMetricsEnableEnvVar) - pthmDisabled := newPathChksum(MyTestFileHasher) - - time.Sleep(delayDur) - err := testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (zero paths) err=%v", err) - } - - time.Sleep(delayDur) - rwDsn := "/env/unset/db-dsn/dsn.txt" - err = pthmDisabled.addPath(rwDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", rwDsn, err) - } - err = testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (one path) err=%v", err) - } - - time.Sleep(delayDur) - roDsn := "/env/unset/db-dsn/ro-dsn.txt" - err = pthmDisabled.addPath("/env/unset/db-dsn/ro-dsn.txt") - if err != nil { - t.Errorf("addPath(%s) err=%v", roDsn, err) - } - err = testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (two paths) err=%v", err) - } -} - -func TestPathChksumMetricsEnabled(t *testing.T) { - os.Setenv(PathChksumMetricsEnableEnvVar, "true") - pthmEnabled := newPathChksum(MyTestFileHasher) - - time.Sleep(delayDur) - err := testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (zero paths) err=%v", err) - } - - time.Sleep(delayDur) - rwDsn := "/env/true/db-dsn/dsn.txt" - err = pthmEnabled.addPath(rwDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", rwDsn, err) - } - err = internal.CollectAndRegexpCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, rwDsn)), - HotloadPathChksumTimestampSecondsName) - if err != nil { - t.Errorf("CollectAndRegexpCompare (one path) err=%v", err) - } - - time.Sleep(delayDur) - roDsn := "/env/true/db-dsn/ro-dsn.txt" - err = pthmEnabled.addPath(roDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", roDsn, err) - } - err = internal.CollectAndRegexpCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, rwDsn)+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, roDsn)), - HotloadPathChksumTimestampSecondsName) - if err != nil { - t.Errorf("CollectAndRegexpCompare (two paths) err=%v", err) - } - - err = pthmEnabled.addPath(rwDsn) - if err != ErrDuplicatePath { - t.Errorf("addPath(%s): expecting ErrDuplicatePath, but got err=%v", rwDsn, err) - } -} diff --git a/metrics/prometheus.go b/metrics/prometheus.go deleted file mode 100644 index 6f841e2..0000000 --- a/metrics/prometheus.go +++ /dev/null @@ -1,85 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -const ( - GRPCMethodKey = "grpc_method" - GRPCServiceKey = "grpc_service" - StatementKey = "stmt" // either exec or query - ExecStatement = "exec" - QueryStatement = "query" - - StrategyKey = "strategy" - PathKey = "path" - UrlKey = "url" -) - -// SqlStmtsSummary is a prometheus metric to keep track of the number of times -// a sql statement is called in a transaction by statement type per grpc service -var SqlStmtsSummaryName = "transaction_sql_stmts" -var SqlStmtsSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Name: SqlStmtsSummaryName, - Help: "The number of sql stmts called in a transaction by statement type per grpc service and method", -}, []string{GRPCServiceKey, GRPCMethodKey, StatementKey}) - -// HotloadModtimeLatencyHistogram is modtime latency histogram (in seconds) -// ie: each sample datapoint is time.Now().Sub(Modtime) -var HotloadModtimeLatencyHistogramName = "hotload_modtime_latency_histogram" -var HotloadModtimeLatencyHistogramHelp = "Hotload modtime latency histogram (seconds) by strategy and path" -var HotloadModtimeLatencyHistogramDefBuckets = []float64{900, 1800, 2700, 3600, 4500, 5400, 7200, 10800, 14400, 28800, 86400} -var HotloadModtimeLatencyHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: HotloadModtimeLatencyHistogramName, - Help: HotloadModtimeLatencyHistogramHelp, - Buckets: HotloadModtimeLatencyHistogramDefBuckets, -}, []string{StrategyKey, PathKey}) - -func ObserveHotloadModtimeLatencyHistogram(strategy, path string, val float64) { - HotloadModtimeLatencyHistogram.WithLabelValues(strategy, path).Observe(val) -} - -// HotloadChangeTotal is count of changes detected by hotload -var HotloadChangeTotalName = "hotload_change_total" -var HotloadChangeTotalHelp = "Hotload change total by url" -var HotloadChangeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: HotloadChangeTotalName, - Help: HotloadChangeTotalHelp, -}, []string{UrlKey}) - -func IncHotloadChangeTotal(url string) { - HotloadChangeTotal.WithLabelValues(url).Inc() -} - -// HotloadLastChangedTimestampSeconds is timestamp when hotload last detected change (unix timestamp) -var HotloadLastChangedTimestampSecondsName = "hotload_last_changed_timestamp_seconds" -var HotloadLastChangedTimestampSecondsHelp = "Hotload last changed (unix timestamp), by url" -var HotloadLastChangedTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: HotloadLastChangedTimestampSecondsName, - Help: HotloadLastChangedTimestampSecondsHelp, -}, []string{UrlKey}) - -func SetHotloadLastChangedTimestampSeconds(url string, val float64) { - HotloadLastChangedTimestampSeconds.WithLabelValues(url).Set(val) -} - -func GetCollectors() []prometheus.Collector { - return []prometheus.Collector{ - SqlStmtsSummary, - HotloadModtimeLatencyHistogram, - HotloadChangeTotal, - HotloadLastChangedTimestampSeconds, - } -} - -// ResetCollectors is useful for testing -func ResetCollectors() { - SqlStmtsSummary.Reset() - HotloadModtimeLatencyHistogram.Reset() - HotloadChangeTotal.Reset() - HotloadLastChangedTimestampSeconds.Reset() -} - -func init() { - prometheus.MustRegister(GetCollectors()...) -} diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go deleted file mode 100644 index ff49903..0000000 --- a/metrics/prometheus_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package metrics - -import ( - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" -) - -var _ = Describe("PrometheusMetric", func() { - It("Should register a prometheus metric", func() { - // This test is a placeholder for a real test - err := prometheus.Register(SqlStmtsSummary) - Expect(err).Should(HaveOccurred()) - Expect(errors.As(err, &prometheus.AlreadyRegisteredError{})).Should(BeTrue()) - }) -}) diff --git a/modtime/modtime_monitor.go b/modtime/modtime_monitor.go index 274d34b..414426c 100644 --- a/modtime/modtime_monitor.go +++ b/modtime/modtime_monitor.go @@ -11,8 +11,8 @@ import ( "sync/atomic" "time" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/logger" ) var ( @@ -155,9 +155,12 @@ func (mtm *ModTimeMonitor) checkPathModTimes(ctx context.Context, nowTime time.T pathRec.modTime.Store(newTime) } - latencyNano := nowTime.Sub(pathRec.modTime.Load().(time.Time)) - latencySecs := latencyNano.Seconds() - metrics.ObserveHotloadModtimeLatencyHistogram(pkey.strategy, pkey.path, latencySecs) + latency := nowTime.Sub(pathRec.modTime.Load().(time.Time)) + hotload.EmitModTimeEvent(hotload.ModTimeEvent{ + Strategy: pkey.strategy, + Path: pkey.path, + Latency: latency, + }) } } diff --git a/modtime/modtime_options.go b/modtime/modtime_options.go index de491a1..8ef3865 100644 --- a/modtime/modtime_options.go +++ b/modtime/modtime_options.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/infobloxopen/hotload/logger" + "github.com/infobloxopen/hotload/v3/logger" ) var ( diff --git a/modtime/modtime_test.go b/modtime/modtime_test.go deleted file mode 100644 index e53acd1..0000000 --- a/modtime/modtime_test.go +++ /dev/null @@ -1,379 +0,0 @@ -package modtime - -import ( - "context" - "fmt" - "log" - "math" - "strings" - "testing" - "testing/fstest" - "time" - - internal "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" -) - -var fsnotifyStrategy = "fsnotify" - -// TestAgainstUnixFS verifies that the use of io/fs.FS in the implementation -// works against the real Unix FS -func TestAgainstUnixFS(t *testing.T) { - var zeroTime time.Time - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() - - metrics.ResetCollectors() - - logfn := func(args ...any) { - log.Println(args...) - } - - // Create ModTimeMonitor that monitors the (default) real host Unix FS - mtm := NewModTimeMonitor(ctx, - WithCheckInterval(time.Millisecond*200), - WithLogger(logfn), - WithErrLogger(logfn), - ) - - // Add well-known Unix path to monitor the mod-time of - pth := "/dev/null" - mtm.AddMonitoredPath(fsnotifyStrategy, pth) - - // Give time for ModTimeMonitor background thread to check mod-times - time.Sleep(time.Millisecond * 500) - - // Verify valid mod-time has been retrieved - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if zeroTime.After(sts.ModTime) { - t.Errorf("GetPathStatus(%s): unexpected ModTime=%+v", pth, sts.ModTime) - } - } - - // Unfortunately for some reason, os.Chtimes() returns - // "chtimes /dev/null: operation not permitted" - // so we can't update /dev/null modtime for additional testing - - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 200) -} - -// TestAgainstMapFS verifies using MapFS mock FS -func TestAgainstMapFS(t *testing.T) { - var zeroTime time.Time - prevTime := zeroTime - - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() - - metrics.ResetCollectors() - - // Create MapFS mock FS - pth := "/foo/bar" - mfs := internal.NewSafeMapFS() - mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: MustParseRFC3339("0002-02-02T02:02:02Z")}) - - checkIntv := time.Millisecond * 1000 - - // Create ModTimeMonitor that monitors mock FS - mtm := NewModTimeMonitor(ctx, - WithStatFS(mfs), - WithCheckInterval(checkIntv), - WithLogger(func(args ...any) { - log.Println(args...) - }), - ) - - // Add mock path to monitor the mod-time of. - mtm.AddMonitoredPath(fsnotifyStrategy, pth) - - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 2 cycles of checks. - time.Sleep(2*checkIntv + 10*time.Millisecond) - - // Verify valid mod-time has been retrieved - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.Before(prevTime) || sts.ModTime.Equal(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly not updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime - } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpInitial), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) - } - - // Update mock path mod-time - mapf, err := mfs.GetMapFile(pth) - if err != nil { - t.Errorf("GetMapFile(%s): unexpected err=%s", pth, err) - } - mapf.ModTime = time.Now() - mfs.UpsertMapFile(pth, mapf) - - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 1 cycle of checks. - time.Sleep(1*checkIntv + 10*time.Millisecond) - - // Verify modtime updated - sts, err = mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.Before(prevTime) || sts.ModTime.Equal(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly not updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime - } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpAfterModtimeUpdated), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) - } - - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 1 cycle of checks. - time.Sleep(1*checkIntv + 10*time.Millisecond) - - // Verify modtime NOT updated because mock path modtime was NOT updated - sts, err = mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.After(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime - } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpAfterModtimeNotUpdated), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) - } - - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 200) -} - -// TestConcurrency verifies thread-safety by spawning multiple -// threads all running with the same interval. -// Should be tested with go test -race flag. -func TestConcurrency(t *testing.T) { - var zeroTime time.Time - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() - - metrics.ResetCollectors() - - commonIntv := time.Millisecond * 100 - - // Create MapFS mock FS - mfs := internal.NewSafeMapFS() - - modTimePaths := []struct { - pathStr string - startMtime time.Time - }{ - { - pathStr: "/concurrency/sub10", - startMtime: time.Now(), - }, - { - pathStr: "/concurrency/sub20", - startMtime: zeroTime, - }, - { - pathStr: "/concurrency/sub30", - startMtime: MustParseRFC3339("2010-04-01T23:07:59Z"), - }, - } - - // Add paths to mock FS - for _, pathRec := range modTimePaths { - mfs.UpsertMapFile(pathRec.pathStr, &fstest.MapFile{ModTime: pathRec.startMtime}) - } - - // Create ModTimeMonitor that monitors mock FS - mtm := NewModTimeMonitor(context.Background(), - WithStatFS(mfs), - WithCheckInterval(commonIntv), - WithLogger(func(args ...any) { - log.Println(args...) - }), - ) - - modTimeUpdaterLoop := func(ctx context.Context, t *testing.T, pathStr string, updateIntv time.Duration) { - mtm.log(fmt.Sprintf("modTimeUpdaterLoop(%s) started", pathStr)) - updateTicker := time.NewTicker(updateIntv) - defer updateTicker.Stop() - loop: - for { - select { - case <-ctx.Done(): - break loop - case curTime := <-updateTicker.C: - mapf, err := mfs.GetMapFile(pathStr) - if err != nil { - t.Errorf("modTimeUpdaterLoop(%s): GetMapFile() err=%s", pathStr, err) - } else { - mapf.ModTime = curTime - mfs.UpsertMapFile(pathStr, mapf) - } - } - } - mtm.log(fmt.Sprintf("modTimeUpdaterLoop(%s) terminated", pathStr)) - } - - modTimeReaderLoop := func(ctx context.Context, t *testing.T, pathStr string, readIntv time.Duration) { - mtm.log(fmt.Sprintf("modTimeReaderLoop(%s) started", pathStr)) - readTicker := time.NewTicker(readIntv) - defer readTicker.Stop() - loop: - for { - select { - case <-ctx.Done(): - break loop - case <-readTicker.C: - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pathStr) - if err != nil { - t.Errorf("modTimeReaderLoop(%s): GetPathStatus() err=%s", pathStr, err) - } else { - t.Logf("sts=%+v", sts) - } - } - } - mtm.log(fmt.Sprintf("modTimeReaderLoop(%s) terminated", pathStr)) - } - - // Add paths to monitor - for _, pathRec := range modTimePaths { - mtm.AddMonitoredPath(fsnotifyStrategy, pathRec.pathStr) - go modTimeUpdaterLoop(ctx, t, pathRec.pathStr, commonIntv) - go modTimeReaderLoop(ctx, t, pathRec.pathStr, commonIntv) - } - - // Give time for background threads to do their thing - time.Sleep(time.Millisecond * 3000) - - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 1000) -} - -// TestMaxTimeSubtractionDuration verifies that time.subtraction -// difference that is too large to be represented by time.Duration -// results in the max int64 value (approx 290 years), and does not fail. -func TestMaxTimeSubtractionDuration(t *testing.T) { - var zeroTime time.Time - modTime := MustParseRFC3339("2025-04-01T23:07:59Z") - hugeDuration := modTime.Sub(zeroTime) - t.Logf("zeroTime=%s", zeroTime) - t.Logf("modTime=%s", modTime) - t.Logf("hugeDuration=modTime.Sub(zeroTime)=0x%x=%d=%s", - int64(hugeDuration), hugeDuration, hugeDuration) - if int64(hugeDuration) != math.MaxInt64 { - t.Errorf("unexpected hugeDuration=%x, should be %x", int64(hugeDuration), math.MaxInt64) - } -} - -func TestTimeComparisons(t *testing.T) { - modTime := MustParseRFC3339("2025-04-01T23:07:59Z") - isEqual := modTime.Equal(modTime) - isAfter := modTime.After(modTime) - isBefore := modTime.Before(modTime) - t.Logf("modTime=%s", modTime) - t.Logf("modTime.Equal(modTime)=%v, modTime.After(modTime)=%v, modTime.Before(modTime)=%v", - isEqual, isAfter, isBefore) - if !isEqual { - t.Errorf("modTime.Equal(modTime)=false, should be true") - } - if isAfter { - t.Errorf("modTime.After(modTime)=true, should be false") - } - if isBefore { - t.Errorf("modTime.Before(modTime)=true, should be false") - } -} - -// MustParseRFC3339 calls time.Parse(time.RFC3339,...), -// and panics on error, otherwise returns parsed time.Time result -func MustParseRFC3339(str string) time.Time { - t, err := time.Parse(time.RFC3339, str) - if err != nil { - panic(fmt.Sprintf("time.Parse(%s) err=%s", str, err)) - } - return t -} - -var expectMetricsRegexpInitial = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 2 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 2 -` - -var expectMetricsRegexpAfterModtimeUpdated = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 3 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 3 -` - -var expectMetricsRegexpAfterModtimeNotUpdated = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 4 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 4 -` diff --git a/observer.go b/observer.go new file mode 100644 index 0000000..e9482e3 --- /dev/null +++ b/observer.go @@ -0,0 +1,150 @@ +package hotload + +import ( + "context" + "sync" + "time" +) + +// ConfigChangeEvent is emitted when a strategy reports a new connection +// string for a watched hotload DSN. +type ConfigChangeEvent struct { + // GroupName is the full hotload DSN (e.g. "fsnotify://postgres/etc/dsn"). + GroupName string + // OldRedactedDSN and NewRedactedDSN are the previous and new underlying + // connection strings with credentials redacted. + OldRedactedDSN string + NewRedactedDSN string + // ForceKill reports whether the group closes old connections immediately. + ForceKill bool + At time.Time +} + +// ConnEvent is emitted when hotload opens or closes an underlying connection. +type ConnEvent struct { + GroupName string + RedactedDSN string + // Killed is true on close events caused by a config change (rather than + // the pool retiring the connection). + Killed bool +} + +// TxEvent is emitted when a transaction completes. ExecStmts and QueryStmts +// are the number of exec and query statements observed on the connection +// since the previous transaction completed. +type TxEvent struct { + // Ctx is the context the transaction was started with; adapters can + // extract labels from it with GetExecLabelsFromContext. + Ctx context.Context + ExecStmts int64 + QueryStmts int64 + Committed bool +} + +// WatchEvent is emitted when a strategy watch is established or closed. +type WatchEvent struct { + GroupName string + Strategy string + Path string + Closed bool +} + +// ModTimeEvent is emitted by the modtime monitor when it samples the +// modification time of a watched path. +type ModTimeEvent struct { + Strategy string + Path string + // Latency is the time elapsed since the file was last modified. + Latency time.Duration +} + +// Hooks receives notifications about hotload activity. All fields are +// optional; nil fields are skipped. Hooks must be fast and must not call +// back into hotload. Adapters (e.g. the observability module) use Hooks to +// export metrics without hotload depending on any metrics library. +type Hooks struct { + OnConfigChange func(ConfigChangeEvent) + OnConnOpen func(ConnEvent) + OnConnClose func(ConnEvent) + OnTxComplete func(TxEvent) + OnWatch func(WatchEvent) + OnModTimeCheck func(ModTimeEvent) +} + +var hooksMu sync.RWMutex +var hooks []Hooks + +// RegisterHooks adds h to the set of registered hooks. Hooks cannot be +// unregistered; register them once during program initialization, before +// opening connections. +func RegisterHooks(h Hooks) { + hooksMu.Lock() + defer hooksMu.Unlock() + hooks = append(hooks, h) +} + +// resetHooks removes all registered hooks. For tests. +func resetHooks() { + hooksMu.Lock() + defer hooksMu.Unlock() + hooks = nil +} + +func snapshotHooks() []Hooks { + hooksMu.RLock() + defer hooksMu.RUnlock() + return hooks +} + +func emitConfigChange(ev ConfigChangeEvent) { + for _, h := range snapshotHooks() { + if h.OnConfigChange != nil { + h.OnConfigChange(ev) + } + } +} + +func emitConnOpen(ev ConnEvent) { + for _, h := range snapshotHooks() { + if h.OnConnOpen != nil { + h.OnConnOpen(ev) + } + } +} + +func emitConnClose(ev ConnEvent) { + for _, h := range snapshotHooks() { + if h.OnConnClose != nil { + h.OnConnClose(ev) + } + } +} + +func emitTxComplete(ev TxEvent) { + for _, h := range snapshotHooks() { + if h.OnTxComplete != nil { + h.OnTxComplete(ev) + } + } +} + +// EmitWatchEvent notifies registered hooks that a strategy watch was +// established or closed. It is exported for strategy implementations +// (e.g. the fsnotify subpackage). +func EmitWatchEvent(ev WatchEvent) { + for _, h := range snapshotHooks() { + if h.OnWatch != nil { + h.OnWatch(ev) + } + } +} + +// EmitModTimeEvent notifies registered hooks of a modtime sample. It is +// exported for the modtime subpackage. +func EmitModTimeEvent(ev ModTimeEvent) { + for _, h := range snapshotHooks() { + if h.OnModTimeCheck != nil { + h.OnModTimeCheck(ev) + } + } +} diff --git a/stmt.go b/stmt.go new file mode 100644 index 0000000..2393e37 --- /dev/null +++ b/stmt.go @@ -0,0 +1,69 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// baseStmt wraps an underlying driver.Stmt prepared on a baseConn. It +// implements the mandatory driver.Stmt methods; the optional interfaces +// (StmtExecContext, StmtQueryContext, ColumnConverter, NamedValueChecker) +// are exposed only when the underlying stmt supports them, via the generated +// combination wrappers returned by wrapStmt. +type baseStmt struct { + inner driver.Stmt + conn *baseConn +} + +func (s *baseStmt) Close() error { + return s.inner.Close() +} + +func (s *baseStmt) NumInput() int { + return s.inner.NumInput() +} + +func (s *baseStmt) Exec(args []driver.Value) (driver.Result, error) { + s.conn.execStmts.Add(1) + return s.inner.Exec(args) +} + +func (s *baseStmt) Query(args []driver.Value) (driver.Rows, error) { + s.conn.queryStmts.Add(1) + return s.inner.Query(args) +} + +// execContext backs the generated StmtExecContext wrappers; only reachable +// when the underlying stmt implements driver.StmtExecContext. +func (s *baseStmt) execContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + octx, release := s.conn.opCtx(ctx) + if octx == nil { + return nil, driver.ErrBadConn + } + defer release() + s.conn.execStmts.Add(1) + return s.inner.(driver.StmtExecContext).ExecContext(octx, args) +} + +// queryContext backs the generated StmtQueryContext wrappers. As with +// baseConn.queryContext, the caller's context passes through unmerged +// because the returned driver.Rows captures it. +func (s *baseStmt) queryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if s.conn.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn + } + s.conn.queryStmts.Add(1) + return s.inner.(driver.StmtQueryContext).QueryContext(ctx, args) +} + +// columnConverter backs the generated ColumnConverter wrappers; only +// reachable when the underlying stmt implements driver.ColumnConverter. +func (s *baseStmt) columnConverter(idx int) driver.ValueConverter { + return s.inner.(driver.ColumnConverter).ColumnConverter(idx) +} + +// checkNamedValue backs the generated NamedValueChecker wrappers; only +// reachable when the underlying stmt implements driver.NamedValueChecker. +func (s *baseStmt) checkNamedValue(nv *driver.NamedValue) error { + return s.inner.(driver.NamedValueChecker).CheckNamedValue(nv) +} diff --git a/stmt_combos_gen.go b/stmt_combos_gen.go new file mode 100644 index 0000000..9edec85 --- /dev/null +++ b/stmt_combos_gen.go @@ -0,0 +1,244 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package hotload + +import "database/sql/driver" + +// stmtFlags reports which optional interfaces the wrapped stmt must expose. +func stmtFlags(s driver.Stmt) uint8 { + var f uint8 + if _, ok := s.(driver.StmtExecContext); ok { + f |= 1 << 0 + } + if _, ok := s.(driver.StmtQueryContext); ok { + f |= 1 << 1 + } + if _, ok := s.(driver.ColumnConverter); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 2 + } + if _, ok := s.(driver.NamedValueChecker); ok { + f |= 1 << 3 + } + return f +} + +type stmt_E struct { + *baseStmt + sExecer +} + +var ( + _ driver.Stmt = stmt_E{} + _ driver.StmtExecContext = stmt_E{} +) + +type stmt_Q struct { + *baseStmt + sQueryer +} + +var ( + _ driver.Stmt = stmt_Q{} + _ driver.StmtQueryContext = stmt_Q{} +) + +type stmt_EQ struct { + *baseStmt + sExecer + sQueryer +} + +var ( + _ driver.Stmt = stmt_EQ{} + _ driver.StmtExecContext = stmt_EQ{} + _ driver.StmtQueryContext = stmt_EQ{} +) + +type stmt_C struct { + *baseStmt + sColConv +} + +var ( + _ driver.Stmt = stmt_C{} + _ driver.ColumnConverter = stmt_C{} +) + +type stmt_EC struct { + *baseStmt + sExecer + sColConv +} + +var ( + _ driver.Stmt = stmt_EC{} + _ driver.StmtExecContext = stmt_EC{} + _ driver.ColumnConverter = stmt_EC{} +) + +type stmt_QC struct { + *baseStmt + sQueryer + sColConv +} + +var ( + _ driver.Stmt = stmt_QC{} + _ driver.StmtQueryContext = stmt_QC{} + _ driver.ColumnConverter = stmt_QC{} +) + +type stmt_EQC struct { + *baseStmt + sExecer + sQueryer + sColConv +} + +var ( + _ driver.Stmt = stmt_EQC{} + _ driver.StmtExecContext = stmt_EQC{} + _ driver.StmtQueryContext = stmt_EQC{} + _ driver.ColumnConverter = stmt_EQC{} +) + +type stmt_N struct { + *baseStmt + sNVChecker +} + +var ( + _ driver.Stmt = stmt_N{} + _ driver.NamedValueChecker = stmt_N{} +) + +type stmt_EN struct { + *baseStmt + sExecer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EN{} + _ driver.StmtExecContext = stmt_EN{} + _ driver.NamedValueChecker = stmt_EN{} +) + +type stmt_QN struct { + *baseStmt + sQueryer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_QN{} + _ driver.StmtQueryContext = stmt_QN{} + _ driver.NamedValueChecker = stmt_QN{} +) + +type stmt_EQN struct { + *baseStmt + sExecer + sQueryer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EQN{} + _ driver.StmtExecContext = stmt_EQN{} + _ driver.StmtQueryContext = stmt_EQN{} + _ driver.NamedValueChecker = stmt_EQN{} +) + +type stmt_CN struct { + *baseStmt + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_CN{} + _ driver.ColumnConverter = stmt_CN{} + _ driver.NamedValueChecker = stmt_CN{} +) + +type stmt_ECN struct { + *baseStmt + sExecer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_ECN{} + _ driver.StmtExecContext = stmt_ECN{} + _ driver.ColumnConverter = stmt_ECN{} + _ driver.NamedValueChecker = stmt_ECN{} +) + +type stmt_QCN struct { + *baseStmt + sQueryer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_QCN{} + _ driver.StmtQueryContext = stmt_QCN{} + _ driver.ColumnConverter = stmt_QCN{} + _ driver.NamedValueChecker = stmt_QCN{} +) + +type stmt_EQCN struct { + *baseStmt + sExecer + sQueryer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EQCN{} + _ driver.StmtExecContext = stmt_EQCN{} + _ driver.StmtQueryContext = stmt_EQCN{} + _ driver.ColumnConverter = stmt_EQCN{} + _ driver.NamedValueChecker = stmt_EQCN{} +) + +func wrapStmt(b *baseStmt) driver.Stmt { + switch stmtFlags(b.inner) { + case 1: + return stmt_E{b, sExecer{b}} + case 2: + return stmt_Q{b, sQueryer{b}} + case 3: + return stmt_EQ{b, sExecer{b}, sQueryer{b}} + case 4: + return stmt_C{b, sColConv{b}} + case 5: + return stmt_EC{b, sExecer{b}, sColConv{b}} + case 6: + return stmt_QC{b, sQueryer{b}, sColConv{b}} + case 7: + return stmt_EQC{b, sExecer{b}, sQueryer{b}, sColConv{b}} + case 8: + return stmt_N{b, sNVChecker{b}} + case 9: + return stmt_EN{b, sExecer{b}, sNVChecker{b}} + case 10: + return stmt_QN{b, sQueryer{b}, sNVChecker{b}} + case 11: + return stmt_EQN{b, sExecer{b}, sQueryer{b}, sNVChecker{b}} + case 12: + return stmt_CN{b, sColConv{b}, sNVChecker{b}} + case 13: + return stmt_ECN{b, sExecer{b}, sColConv{b}, sNVChecker{b}} + case 14: + return stmt_QCN{b, sQueryer{b}, sColConv{b}, sNVChecker{b}} + case 15: + return stmt_EQCN{b, sExecer{b}, sQueryer{b}, sColConv{b}, sNVChecker{b}} + default: + return b + } +} diff --git a/stmt_pieces.go b/stmt_pieces.go new file mode 100644 index 0000000..617b003 --- /dev/null +++ b/stmt_pieces.go @@ -0,0 +1,37 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// The piece types below each carry exactly one optional driver.Stmt method, +// mirroring the conn pieces in conn_pieces.go. See that file for the design. + +// sExecer carries ExecContext (driver.StmtExecContext). +type sExecer struct{ b *baseStmt } + +func (p sExecer) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return p.b.execContext(ctx, args) +} + +// sQueryer carries QueryContext (driver.StmtQueryContext). +type sQueryer struct{ b *baseStmt } + +func (p sQueryer) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return p.b.queryContext(ctx, args) +} + +// sColConv carries ColumnConverter (driver.ColumnConverter). +type sColConv struct{ b *baseStmt } + +func (p sColConv) ColumnConverter(idx int) driver.ValueConverter { + return p.b.columnConverter(idx) +} + +// sNVChecker carries CheckNamedValue (driver.NamedValueChecker). +type sNVChecker struct{ b *baseStmt } + +func (p sNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return p.b.checkNamedValue(nv) +} diff --git a/transaction.go b/transaction.go deleted file mode 100644 index 3761f11..0000000 --- a/transaction.go +++ /dev/null @@ -1,82 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" -) - -// managedTx wraps a sql/driver.Tx so that it can store the context of the -// transaction and clean up the execqueryCallsCounter on Commit or Rollback. -type managedTx struct { - tx driver.Tx - conn *managedConn - ctx context.Context -} - -func (t *managedTx) Commit() error { - var log = logger.GetLogger() - log("managedTx.Commit") - err := t.tx.Commit() - t.cleanup() - return err -} - -func (t *managedTx) Rollback() error { - var log = logger.GetLogger() - log("managedTx.Rollback") - err := t.tx.Rollback() - t.cleanup() - return err -} - -func observeSQLStmtsSummary(ctx context.Context, execStmtsCounter, queryStmtsCounter int64) { - labels := GetExecLabelsFromContext(ctx) - service := labels[metrics.GRPCServiceKey] - method := labels[metrics.GRPCMethodKey] - - metrics.SqlStmtsSummary.WithLabelValues(service, method, metrics.ExecStatement).Observe(float64(execStmtsCounter)) - metrics.SqlStmtsSummary.WithLabelValues(service, method, metrics.QueryStatement).Observe(float64(queryStmtsCounter)) -} - -func (t *managedTx) cleanup() { - observeSQLStmtsSummary(t.ctx, t.conn.execStmtsCounter.Load(), t.conn.queryStmtsCounter.Load()) - t.conn.resetExecStmtsCounter() - t.conn.resetQueryStmtsCounter() -} - -type promLabelKeyType struct{} - -var promLabelKey = promLabelKeyType{} - -func ContextWithExecLabels(ctx context.Context, labels map[string]string) context.Context { - var log = logger.GetLogger() - if labels == nil { - log("ContextWithExecLabels called with nil label set") - return ctx - } - return context.WithValue(ctx, promLabelKey, labels) -} - -func GetExecLabelsFromContext(ctx context.Context) map[string]string { - var log = logger.GetLogger() - if ctx == nil { - log("No context provided, returning") - return nil - } - - value := ctx.Value(promLabelKey) - if value == nil { - log("No value for promLabelKey, returning") - return nil - } - labelMap, ok := value.(map[string]string) - if !ok { - log("Bad value type used for promLabelKey, conversion error") - return nil - } - - return labelMap -} diff --git a/tx.go b/tx.go new file mode 100644 index 0000000..cceaac9 --- /dev/null +++ b/tx.go @@ -0,0 +1,37 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// managedTx wraps a driver.Tx so the statement counters accumulated on the +// conn can be reported to hooks when the transaction completes. +type managedTx struct { + tx driver.Tx + conn *baseConn + ctx context.Context +} + +func (t *managedTx) Commit() error { + err := t.tx.Commit() + t.complete(err == nil) + return err +} + +func (t *managedTx) Rollback() error { + err := t.tx.Rollback() + t.complete(false) + return err +} + +func (t *managedTx) complete(committed bool) { + emitTxComplete(TxEvent{ + Ctx: t.ctx, + ExecStmts: t.conn.execStmts.Load(), + QueryStmts: t.conn.queryStmts.Load(), + Committed: committed, + }) + t.conn.execStmts.Store(0) + t.conn.queryStmts.Store(0) +} diff --git a/tx_test.go b/tx_test.go new file mode 100644 index 0000000..40e3ca4 --- /dev/null +++ b/tx_test.go @@ -0,0 +1,189 @@ +package hotload_test + +import ( + "context" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// collectTxEvents subscribes a hook capturing TxEvents into a channel. +func collectTxEvents(t *testing.T) chan hotload.TxEvent { + t.Helper() + events := make(chan hotload.TxEvent, 100) + hotload.RegisterHooks(hotload.Hooks{ + OnTxComplete: func(ev hotload.TxEvent) { + select { + case events <- ev: + default: + } + }, + }) + return events +} + +func awaitTxEvent(t *testing.T, events chan hotload.TxEvent) hotload.TxEvent { + t.Helper() + select { + case ev := <-events: + return ev + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for TxEvent") + return hotload.TxEvent{} + } +} + +// TestTxStatementCounters: exec and query statements run on a conn are +// reported when its transaction completes, along with the labels carried by +// the transaction's context. +func TestTxStatementCounters(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + labels := map[string]string{"grpc_service": "svc", "grpc_method": "m"} + ctx := hotload.ContextWithExecLabels(context.Background(), labels) + + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE y"); err != nil { + t.Fatal(err) + } + var dsn string + if err := tx.QueryRow("SELECT dsn").Scan(&dsn); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.ExecStmts != 2 || ev.QueryStmts != 1 { + t.Errorf("counters = exec %d query %d, want exec 2 query 1", ev.ExecStmts, ev.QueryStmts) + } + if !ev.Committed { + t.Error("Committed = false, want true") + } + got := hotload.GetExecLabelsFromContext(ev.Ctx) + if got["grpc_service"] != "svc" || got["grpc_method"] != "m" { + t.Errorf("labels from event ctx = %v, want %v", got, labels) + } +} + +// TestTxRollbackEvent: rollbacks report Committed=false and reset counters +// for the next transaction. +func TestTxRollbackEvent(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + ctx := context.Background() + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.Committed { + t.Error("Committed = true, want false") + } + if ev.ExecStmts != 1 { + t.Errorf("ExecStmts = %d, want 1", ev.ExecStmts) + } + + // Counters must reset between transactions on the same conn. + tx2, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := tx2.Commit(); err != nil { + t.Fatal(err) + } + ev2 := awaitTxEvent(t, events) + if ev2.ExecStmts != 0 || ev2.QueryStmts != 0 { + t.Errorf("second tx counters = exec %d query %d, want 0/0 (reset)", ev2.ExecStmts, ev2.QueryStmts) + } +} + +// TestTxCountsPreparedStatements: statements executed through prepared +// statements count toward the transaction counters — a blind spot in v1, +// which did not wrap driver.Stmt. +func TestTxCountsPreparedStatements(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + tx, err := fx.db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + stmt, err := tx.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + if err := stmt.Close(); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.ExecStmts != 2 { + t.Errorf("ExecStmts = %d, want 2 (prepared statement execs must count)", ev.ExecStmts) + } +} + +// TestConnAndWatchEvents: conn open/close and watch hooks fire with the +// group name and redacted DSNs. +func TestConnAndWatchEvents(t *testing.T) { + testutil.NoLeaks(t) + + fx := newFixture(t, fxCfg{}) + opens := make(chan hotload.ConnEvent, 10) + closes := make(chan hotload.ConnEvent, 10) + hotload.RegisterHooks(hotload.Hooks{ + OnConnOpen: func(ev hotload.ConnEvent) { opens <- ev }, + OnConnClose: func(ev hotload.ConnEvent) { closes <- ev }, + }) + + fx.queryDSN() + select { + case ev := <-opens: + if ev.GroupName != fx.dsn { + t.Errorf("open event group = %q, want %q", ev.GroupName, fx.dsn) + } + case <-time.After(2 * time.Second): + t.Fatal("no conn open event") + } + + fx.pushAndWait("dsn-2") + fx.queryDSN() + select { + case ev := <-closes: + if ev.GroupName != fx.dsn { + t.Errorf("close event group = %q, want %q", ev.GroupName, fx.dsn) + } + case <-time.After(2 * time.Second): + t.Fatal("no conn close event after swap") + } +} From 5162dc828b642aae65e7d113394347c82b9f82f9 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 00:05:23 -0500 Subject: [PATCH 02/14] Port fsnotify and modtime to v3; fix strategy leaks and symlink-swap detection fsnotify: - Replace pkg/errors with stdlib error wrapping; drop the metrics import (the core emits WatchEvent hooks instead). - Restructure watch closure: CloseWatch/Close now clean up synchronously under the strategy lock and close the per-watch queue, terminating the delivery goroutine, which previously leaked after every CloseWatch. - Make update delivery non-blocking: only the latest value matters, so a full queue drops the oldest entry instead of blocking the strategy behind a subscriber that stopped receiving (previously this could wedge every watcher sharing the strategy). - Resync all watched paths when an event arrives under an unknown name: kqueue reports events under the resolved symlink target, which broke Kubernetes ConfigMap-style symlink swaps on macOS. - Fix Watch panicking after Close (nil map); make the resync period a per-strategy field. - New stdlib test suite against real files: writes, truncation, atomic rename, remove/recreate recovery, the ConfigMap ..data symlink dance, multiple subscribers, close semantics, abandoned-subscriber wedging. modtime: report latency through hotload hooks instead of the metrics package; tests ported to stdlib testing with a mock fs.StatFS and hook-based assertions. internal: secret sink uses crypto/rand instead of google/uuid. --- fsnotify/filewatcher.go | 249 +++++++++++------------ fsnotify/filewatcher_test.go | 372 +++++++++++++++++++++++++++++++++++ modtime/modtime_test.go | 177 +++++++++++++++++ 3 files changed, 675 insertions(+), 123 deletions(-) create mode 100644 fsnotify/filewatcher_test.go create mode 100644 modtime/modtime_test.go diff --git a/fsnotify/filewatcher.go b/fsnotify/filewatcher.go index f3ca28e..ad62b96 100644 --- a/fsnotify/filewatcher.go +++ b/fsnotify/filewatcher.go @@ -20,37 +20,44 @@ func init() { hotload.RegisterStrategy("fsnotify", NewStrategy()) } -var resyncPeriod = time.Second * 2 +const defaultResyncPeriod = time.Second * 2 // NewStrategy implements a hotload strategy that monitors config changes // in a file using fsnotify. func NewStrategy() *Strategy { return &Strategy{ - paths: make(map[string]*pathWatch), + paths: make(map[string]*pathWatch), + resyncPeriod: defaultResyncPeriod, } } -// Strategy implements the hotload Strategy inferface by using +// Strategy implements the hotload Strategy interface by using // fsnotify under the covers. type Strategy struct { - mu sync.RWMutex - paths map[string]*pathWatch - watcher watcher + mu sync.RWMutex + paths map[string]*pathWatch + watcher watcher + resyncPeriod time.Duration } -type pendingOperation struct { - operation string - watchPath string - pathQuery string +// update is one value change queued for delivery to a watcher. +type update struct { dsn string redactDsn string } +// queryWatch fans one path's updates out to one subscriber (the hotload +// group watching pth+pathQry). Its opLoop goroutine is the only writer of +// updateChan: it forwards queued updates and closes the channel when +// operChan closes, so channel operations never race. done is closed (under +// the strategy lock) when the watch is closed, unblocking an opLoop stuck +// sending to a subscriber that stopped receiving. type queryWatch struct { parentPathW *pathWatch pathQuery string updateChan chan string - operChan chan pendingOperation + operChan chan update + done chan struct{} } type pathWatch struct { @@ -82,11 +89,11 @@ func (s *Strategy) resync(w watcher, pth string) (string, error) { return string(bs), w.Add(pth) } -func (s *Strategy) runLoop() { +func (s *Strategy) runLoop(w watcher) { failedPaths := make(map[string]struct{}) for { select { - case ev, ok := <-s.watcher.GetEvents(): + case ev, ok := <-w.GetEvents(): if !ok { s.logf("fsnotify.runLoop", "Events chan closed, terminating") return @@ -97,27 +104,28 @@ func (s *Strategy) runLoop() { continue } - val, err := s.resync(s.watcher, ev.Name) - if err != nil { - s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", ev.Name, err) - failedPaths[ev.Name] = struct{}{} - break + for _, pth := range s.affectedPaths(ev.Name) { + val, err := s.resync(w, pth) + if err != nil { + s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", pth, err) + failedPaths[pth] = struct{}{} + continue + } + delete(failedPaths, pth) + s.setVal(pth, val) } - s.setVal(ev.Name, val) - - case err, ok := <-s.watcher.GetErrors(): + case err, ok := <-w.GetErrors(): if !ok { s.logf("fsnotify.runLoop", "Errors chan closed, terminating") return } s.logf("fsnotify.runLoop", "got error: %s", err.Error()) - case <-time.After(resyncPeriod): - s.logf("fsnotify.runLoop", "resyncPeriod %s timedout", resyncPeriod.String()) + case <-time.After(s.resyncPeriod): var fixedPaths []string for pth := range failedPaths { - val, err := s.resync(s.watcher, pth) + val, err := s.resync(w, pth) if err != nil { s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", pth, err) } else { @@ -132,22 +140,61 @@ func (s *Strategy) runLoop() { } } +// affectedPaths maps a notification name to the watched paths that need a +// resync. The name usually is a watched path, but when the watched path is +// a symlink (the Kubernetes ConfigMap pattern) some platforms report events +// under the resolved target — kqueue even adds a /private prefix on macOS — +// so an unknown name conservatively resyncs every watched path. Events are +// rare and the watched files are small, so the cost is negligible. +func (s *Strategy) affectedPaths(name string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + if _, ok := s.paths[name]; ok { + return []string{name} + } + out := make([]string, 0, len(s.paths)) + for pth := range s.paths { + out = append(out, pth) + } + return out +} + func (s *Strategy) setVal(pth string, val string) { s.mu.Lock() defer s.mu.Unlock() - if _, ok := s.paths[pth]; !ok { + pathW, ok := s.paths[pth] + if !ok { s.logf("fsnotify.setVal", "ignoring path not in map: '%s'", pth) return } - s.paths[pth].value = val + pathW.value = val redactDsn := internal.RedactUrl(val) - for _, qryW := range s.paths[pth].queries { - pendOp := pendingOperation{ - operation: "send", - dsn: val, - redactDsn: redactDsn, + for _, qryW := range pathW.queries { + qryW.enqueue(update{dsn: val, redactDsn: redactDsn}) + } +} + +// enqueue queues an update for delivery without ever blocking: only the +// latest value matters to a hotload group, so when a subscriber's queue is +// full (it stopped receiving, or is slow), the oldest queued value is +// dropped to make room. A blocking send here would wedge the whole strategy +// behind one slow subscriber, since enqueue runs under the strategy lock. +func (qw *queryWatch) enqueue(up update) { + for { + select { + case qw.operChan <- up: + return + case <-qw.done: + return + default: + } + // Queue full: drop the oldest queued update and retry. opLoop may + // have consumed one concurrently, so the drain is non-blocking too. + select { + case <-qw.operChan: + qw.logf("fsnotify.enqueue", "subscriber slow; dropped oldest queued update") + default: } - qryW.operChan <- pendOp } } @@ -157,6 +204,10 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value pathQry = strings.TrimSpace(pathQry) s.mu.Lock() defer s.mu.Unlock() + // Re-initialize after Close (which nils the maps out). + if s.paths == nil { + s.paths = make(map[string]*pathWatch) + } // if this is the first time this strategy is called, initialize ourselves if s.watcher == nil { watcher, err := notifyConstructor() @@ -164,7 +215,7 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value return "", nil, err } s.watcher = watcher - go s.runLoop() + go s.runLoop(watcher) } pathW, found := s.paths[pth] if found { @@ -197,7 +248,8 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value parentPathW: pathW, pathQuery: pathQry, updateChan: make(chan string), - operChan: make(chan pendingOperation, 30), + operChan: make(chan update, 30), + done: make(chan struct{}), } pathW.queries[pathQry] = qryW go qryW.opLoop() @@ -207,53 +259,36 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value } // CloseWatch implements the hotload.Strategy interface. -// Closes the specified watch by removing the path -// from the watcher and closing the path's update channel. +// Closes the specified watch; its update channel is closed (asynchronously, +// by the watch's delivery goroutine) and, if this was the last watch on the +// path, the path is removed from the file watcher. func (s *Strategy) CloseWatch(pth string, pathQry string) error { pth = path.Clean(pth) pathQry = strings.TrimSpace(pathQry) s.mu.Lock() defer s.mu.Unlock() pathW, found := s.paths[pth] - if found { - qryW, ok := pathW.queries[pathQry] - if ok { - pendOp := pendingOperation{ - operation: "close", - watchPath: pth, - pathQuery: pathQry, - } - qryW.operChan <- pendOp - qryW.logf("fsnotify.CloseWatch", "sent pending close operation") - } + if !found { + return nil + } + qryW, found := pathW.queries[pathQry] + if !found { + return nil } - return nil -} -func (s *Strategy) processWatchClosure(pendOp pendingOperation) error { - var err error - s.mu.Lock() - defer s.mu.Unlock() - pathW, found := s.paths[pendOp.watchPath] - if found { - qryW, ok := pathW.queries[pendOp.pathQuery] - if ok { - delete(pathW.queries, pendOp.pathQuery) - qryW.closeUpdateChan() - qryW.logf("fsnotify.processWatchClosure", "closed update channel") - } - if len(pathW.queries) <= 0 { - err = s.watcher.Remove(pendOp.watchPath) - if err == nil { - s.logf("fsnotify.processWatchClosure", "removed path from being watched '%s'", pendOp.watchPath) - } else { - s.errlogf("fsnotify.processWatchClosure", "failed to remove '%s' from watcher, err=%v", pendOp.watchPath, err) - } - delete(s.paths, pendOp.watchPath) - s.logf("fsnotify.processWatchClosure", "strategy removed path '%s'", pendOp.watchPath) + delete(pathW.queries, pathQry) + qryW.shutdown() + qryW.logf("fsnotify.CloseWatch", "closed watch") + + if len(pathW.queries) == 0 { + delete(s.paths, pth) + if err := s.watcher.Remove(pth); err != nil { + s.errlogf("fsnotify.CloseWatch", "failed to remove '%s' from watcher, err=%v", pth, err) + return err } + s.logf("fsnotify.CloseWatch", "removed path from being watched '%s'", pth) } - return err + return nil } // Close implements the hotload.Strategy interface. @@ -269,60 +304,38 @@ func (s *Strategy) Close() { } for _, pathW := range s.paths { for _, qryW := range pathW.queries { - qryW.closeUpdateChan() - qryW.logf("fsnotify.Close", "closed update channel") + qryW.shutdown() + qryW.logf("fsnotify.Close", "closed watch") } pathW.queries = nil } s.paths = nil } -func (qw *queryWatch) sendUpdate(val, redactDsn string) { - if qw.updateChan == nil { - return - } - - defer func() { - // Recover/ignore from "panic: send on closed channel" - r := recover() - if r != nil { - qw.logf("fsnotify.sendUpdate", "panic recovery '%s'", r) - } - }() - - qw.logf("fsnotify.sendUpdate", "block-sending redactDsn='%s'", redactDsn) - qw.updateChan <- val - qw.logf("fsnotify.sendUpdate", "successfully sent redactDsn='%s'", redactDsn) -} - -func (qw *queryWatch) closeUpdateChan() { - close(qw.updateChan) - qw.updateChan = nil +// shutdown stops the watch's delivery goroutine. Callers must hold the +// strategy lock (it is the lock that serializes shutdown with setVal's +// sends, making the channel close safe). +func (qw *queryWatch) shutdown() { + close(qw.done) + close(qw.operChan) } +// opLoop forwards queued updates to the subscriber. It is the only +// goroutine that sends on or closes updateChan. The done channel unblocks +// the forwarding send if the subscriber stopped receiving (e.g. the hotload +// group was torn down before the watch was closed). func (qw *queryWatch) opLoop() { - for { + for op := range qw.operChan { + qw.logf("fsnotify.opLoop", "sending redactDsn='%s'", op.redactDsn) select { - case pendOp, ok := <-qw.operChan: - if !ok { - qw.logf("fsnotify.opLoop", "operChan closed, terminating") - return - } - switch pendOp.operation { - case "close": - qw.logf("fsnotify.opLoop", "pendingOperation '%s', pendingPath=%s, pendingQuery='%s'", - pendOp.operation, pendOp.watchPath, pendOp.pathQuery) - qw.parentPathW.parentStrat.processWatchClosure(pendOp) - case "send": - qw.logf("fsnotify.opLoop", "pendingOperation '%s', redactDsn='%s'", - pendOp.operation, pendOp.redactDsn) - qw.sendUpdate(pendOp.dsn, pendOp.redactDsn) - default: - qw.logf("fsnotify.opLoop", "ignore invalid pendingOperation '%s'", - pendOp.operation) - } + case qw.updateChan <- op.dsn: + qw.logf("fsnotify.opLoop", "successfully sent redactDsn='%s'", op.redactDsn) + case <-qw.done: + qw.logf("fsnotify.opLoop", "watch closed while sending redactDsn='%s'", op.redactDsn) } } + close(qw.updateChan) + qw.logf("fsnotify.opLoop", "operChan closed, terminating") } func (s *Strategy) logf(prefix, format string, args ...any) { @@ -344,13 +357,3 @@ func (s *Strategy) errlogf(prefix, format string, args ...any) { logPrefix := fmt.Sprintf("%s:", prefix) logger.ErrLogf(logPrefix, format, args...) } - -func (pw *pathWatch) errlogf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, pw.watchPath) - logger.ErrLogf(logPrefix, format, args...) -} - -func (qw *queryWatch) errlogf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s?%s]:", prefix, qw.parentPathW.watchPath, qw.pathQuery) - logger.ErrLogf(logPrefix, format, args...) -} diff --git a/fsnotify/filewatcher_test.go b/fsnotify/filewatcher_test.go new file mode 100644 index 0000000..ada16cb --- /dev/null +++ b/fsnotify/filewatcher_test.go @@ -0,0 +1,372 @@ +package fsnotify + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +// newTestStrategy returns a strategy with a short resync period (the +// recovery path after failed re-watches) that is closed when the test ends. +func newTestStrategy(t *testing.T) *Strategy { + t.Helper() + s := NewStrategy() + s.resyncPeriod = 50 * time.Millisecond + t.Cleanup(s.Close) + return s +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// awaitValue reads updates until want arrives, tolerating duplicate +// notifications (fsnotify can deliver several events per change). +func awaitValue(t *testing.T, ch <-chan string, want string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case got, ok := <-ch: + if !ok { + t.Fatalf("update channel closed while waiting for %q", want) + } + t.Logf("update: %q", got) + if got == want { + return + } + case <-deadline: + t.Fatalf("timed out waiting for value %q", want) + } + } +} + +// awaitClosed waits for the update channel to be closed. +func awaitClosed(t *testing.T, ch <-chan string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("timed out waiting for update channel to close") + } + } +} + +func TestWatchMissingFile(t *testing.T) { + s := newTestStrategy(t) + _, _, err := s.Watch(context.Background(), filepath.Join(t.TempDir(), "missing"), "") + if err == nil { + t.Fatal("expected an error watching a missing file") + } +} + +func TestWatchInitialValueTrimmed(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, " dsn-1 \n") + + value, _, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + if value != "dsn-1" { + t.Errorf("initial value = %q, want %q (whitespace must be trimmed)", value, "dsn-1") + } +} + +func TestWatchSeesWrites(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + _, updates, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + + writeFile(t, p, "dsn-2") + awaitValue(t, updates, "dsn-2") + + writeFile(t, p, "dsn-3") + awaitValue(t, updates, "dsn-3") +} + +// TestWatchSeesAtomicRename covers the write-then-rename pattern used by +// most config writers (and by viper, kustomize, etc.). +func TestWatchSeesAtomicRename(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + p := filepath.Join(dir, "config") + writeFile(t, p, "dsn-1") + + _, updates, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + + tmp := filepath.Join(dir, "config.tmp") + writeFile(t, tmp, "dsn-2") + if err := os.Rename(tmp, p); err != nil { + t.Fatal(err) + } + awaitValue(t, updates, "dsn-2") +} + +// TestWatchRecoversFromRemoveAndRecreate: deleting the file fails the +// resync; the periodic retry must pick the value up once the file returns. +func TestWatchRecoversFromRemoveAndRecreate(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + p := filepath.Join(dir, "config") + writeFile(t, p, "dsn-1") + + _, updates, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + writeFile(t, p, "dsn-2") + awaitValue(t, updates, "dsn-2") +} + +// TestWatchSeesKubernetesConfigMapSwap reproduces the kubelet's ConfigMap +// update dance: the watched path is a symlink chain through a `..data` +// symlink that is swapped atomically to a new timestamped directory. +func TestWatchSeesKubernetesConfigMapSwap(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + + tsDir1 := filepath.Join(dir, "..2026_06_09_01") + if err := os.Mkdir(tsDir1, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(tsDir1, "config"), "dsn-1") + + dataLink := filepath.Join(dir, "..data") + if err := os.Symlink(tsDir1, dataLink); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "config") + if err := os.Symlink(filepath.Join(dataLink, "config"), p); err != nil { + t.Fatal(err) + } + + value, updates, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + if value != "dsn-1" { + t.Fatalf("initial value = %q, want dsn-1", value) + } + + // The swap: new timestamped dir, retarget ..data atomically via rename, + // remove the old dir. + tsDir2 := filepath.Join(dir, "..2026_06_09_02") + if err := os.Mkdir(tsDir2, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(tsDir2, "config"), "dsn-2") + + tmpLink := filepath.Join(dir, "..data_tmp") + if err := os.Symlink(tsDir2, tmpLink); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmpLink, dataLink); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(tsDir1); err != nil { + t.Fatal(err) + } + + awaitValue(t, updates, "dsn-2") +} + +// TestMultipleWatchersOnePath: watchers with different query strings get +// independent channels fed from one underlying file watch. +func TestMultipleWatchersOnePath(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, p, "forceKill=true") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, p, "") + if err != nil { + t.Fatal(err) + } + + writeFile(t, p, "dsn-2") + awaitValue(t, ch1, "dsn-2") + awaitValue(t, ch2, "dsn-2") +} + +// TestWatchSamePathAndQueryShared: the same path+query pair returns the +// same channel rather than a second watch. +func TestWatchSamePathAndQueryShared(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, p, "q=1") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, p, "q=1") + if err != nil { + t.Fatal(err) + } + if ch1 != ch2 { + t.Error("same path+query should share one update channel") + } +} + +func TestCloseWatchClosesChannel(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + _, updates, err := s.Watch(context.Background(), p, "") + if err != nil { + t.Fatal(err) + } + if err := s.CloseWatch(p, ""); err != nil { + t.Fatal(err) + } + awaitClosed(t, updates) +} + +// TestCloseWatchKeepsOtherQueries: closing one query watch leaves the other +// subscriber of the same path working. +func TestCloseWatchKeepsOtherQueries(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, p, "q=1") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, p, "q=2") + if err != nil { + t.Fatal(err) + } + + if err := s.CloseWatch(p, "q=1"); err != nil { + t.Fatal(err) + } + awaitClosed(t, ch1) + + writeFile(t, p, "dsn-2") + awaitValue(t, ch2, "dsn-2") +} + +func TestStrategyCloseClosesAllChannels(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + p1 := filepath.Join(dir, "a") + p2 := filepath.Join(dir, "b") + writeFile(t, p1, "dsn-a") + writeFile(t, p2, "dsn-b") + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, p1, "") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, p2, "") + if err != nil { + t.Fatal(err) + } + + s.Close() + awaitClosed(t, ch1) + awaitClosed(t, ch2) +} + +// TestWatchAfterClose: the strategy re-initializes itself after Close (the +// registered global instance must survive UnregisterStrategy+re-register +// cycles). +func TestWatchAfterClose(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := context.Background() + _, _, err := s.Watch(ctx, p, "") + if err != nil { + t.Fatal(err) + } + s.Close() + + value, updates, err := s.Watch(ctx, p, "") + if err != nil { + t.Fatalf("Watch after Close: %v", err) + } + if value != "dsn-1" { + t.Errorf("value = %q, want dsn-1", value) + } + writeFile(t, p, "dsn-2") + awaitValue(t, updates, "dsn-2") +} + +// TestAbandonedSubscriberDoesNotWedgeStrategy: if a subscriber stops +// receiving (its hotload group is gone), pending updates must not block the +// strategy's delivery or its CloseWatch path. +func TestAbandonedSubscriberDoesNotWedgeStrategy(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := context.Background() + _, abandoned, err := s.Watch(ctx, p, "q=abandoned") + if err != nil { + t.Fatal(err) + } + _ = abandoned // never read from + + _, live, err := s.Watch(ctx, p, "q=live") + if err != nil { + t.Fatal(err) + } + + // Generate more updates than the abandoned watch's queue can hold. + for i := 2; i < 40; i++ { + writeFile(t, p, "dsn-x") + writeFile(t, p, "dsn-2") + } + awaitValue(t, live, "dsn-2") + + // Closing the abandoned watch must not deadlock. + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + if err := s.CloseWatch(p, "q=abandoned"); err != nil { + t.Errorf("CloseWatch: %v", err) + } + }() + select { + case <-doneCh: + case <-time.After(5 * time.Second): + t.Fatal("CloseWatch deadlocked on an abandoned subscriber") + } + awaitClosed(t, abandoned) +} diff --git a/modtime/modtime_test.go b/modtime/modtime_test.go new file mode 100644 index 0000000..77600aa --- /dev/null +++ b/modtime/modtime_test.go @@ -0,0 +1,177 @@ +package modtime + +import ( + "context" + "fmt" + "sync" + "testing" + "testing/fstest" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal" +) + +const testStrategy = "fsnotify" + +func mustParseRFC3339(t *testing.T, s string) time.Time { + t.Helper() + ts, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) + } + return ts +} + +// eventRecorder captures ModTimeEvents emitted through hotload hooks. +type eventRecorder struct { + mu sync.Mutex + events []hotload.ModTimeEvent +} + +func newEventRecorder(t *testing.T) *eventRecorder { + t.Helper() + r := &eventRecorder{} + hotload.RegisterHooks(hotload.Hooks{ + OnModTimeCheck: func(ev hotload.ModTimeEvent) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, ev) + }, + }) + return r +} + +func (r *eventRecorder) forPath(path string) []hotload.ModTimeEvent { + r.mu.Lock() + defer r.mu.Unlock() + var out []hotload.ModTimeEvent + for _, ev := range r.events { + if ev.Path == path { + out = append(out, ev) + } + } + return out +} + +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +// TestAgainstUnixFS verifies the monitor works against the real filesystem. +func TestAgainstUnixFS(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mtm := NewModTimeMonitor(ctx, WithCheckInterval(20*time.Millisecond)) + pth := "/dev/null" + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Fatal(err) + } + + waitFor(t, 2*time.Second, "a modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && !sts.ModTime.IsZero() + }) +} + +// TestAgainstMapFS drives the monitor with a mock filesystem and verifies +// path status updates and latency events emitted through hotload hooks. +func TestAgainstMapFS(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rec := newEventRecorder(t) + + pth := "/foo/bar" + mfs := internal.NewSafeMapFS() + start := mustParseRFC3339(t, "2026-01-02T03:04:05Z") + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: start}); err != nil { + t.Fatal(err) + } + + mtm := NewModTimeMonitor(ctx, WithStatFS(mfs), WithCheckInterval(20*time.Millisecond)) + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Fatal(err) + } + if err := mtm.AddMonitoredPath(testStrategy, pth); err != ErrDuplicatePath { + t.Errorf("second AddMonitoredPath err = %v, want ErrDuplicatePath", err) + } + + waitFor(t, 2*time.Second, "first modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && sts.ModTime.Equal(start) + }) + + // Bump the file's modtime; the monitor must observe it and the latency + // events must shrink accordingly. + now := time.Now() + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: now}); err != nil { + t.Fatal(err) + } + waitFor(t, 2*time.Second, "updated modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && sts.ModTime.Equal(now) + }) + + waitFor(t, 2*time.Second, "latency events", func() bool { + return len(rec.forPath(pth)) >= 2 + }) + evs := rec.forPath(pth) + first, last := evs[0], evs[len(evs)-1] + if first.Strategy != testStrategy { + t.Errorf("event strategy = %q, want %q", first.Strategy, testStrategy) + } + // The first sample's latency is measured against the old modtime (huge); + // after the update the latency must be small. + if first.Latency < 24*time.Hour { + t.Errorf("first latency = %v, want large (old modtime)", first.Latency) + } + if last.Latency > time.Hour { + t.Errorf("latency after modtime update = %v, want small", last.Latency) + } + + if _, err := mtm.GetPathStatus(testStrategy, "/no/such/path"); err != ErrPathNotFound { + t.Errorf("GetPathStatus(unknown) err = %v, want ErrPathNotFound", err) + } +} + +// TestConcurrency exercises concurrent AddMonitoredPath/GetPathStatus calls +// against a running monitor; meaningful under -race. +func TestConcurrency(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mfs := internal.NewSafeMapFS() + mtm := NewModTimeMonitor(ctx, WithStatFS(mfs), WithCheckInterval(5*time.Millisecond)) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + pth := fmt.Sprintf("/concurrency/sub%d", i) + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: time.Now()}); err != nil { + t.Error(err) + return + } + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Error(err) + return + } + for j := 0; j < 50; j++ { + if _, err := mtm.GetPathStatus(testStrategy, pth); err != nil { + t.Error(err) + return + } + } + }(i) + } + wg.Wait() +} From 59685fbcc9221a9f7461fa0d021e2bf9da9dae11 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 00:09:36 -0500 Subject: [PATCH 03/14] Add observability module: prometheus adapter over hotload hooks New nested module github.com/infobloxopen/hotload/observability keeping prometheus (and its transitive dependencies) out of the hotload core. Registration is explicit via EnablePrometheus, which preserves hotload v1's metric names and labels: transaction_sql_stmts, hotload_change_total, hotload_last_changed_timestamp_seconds, hotload_modtime_latency_histogram and hotload_path_chksum_timestamp_seconds (still gated by HOTLOAD_PATH_CHKSUM_METRICS_ENABLE). The path checksum metric is a hand-rolled scrape-time prometheus.Collector, replacing the colega/gaugefuncvec dependency. The CollectAndRegexpCompare test helper moves here from the core's internal package as observability/promtest. The replace directive in observability/go.mod serves in-repo development; consumers resolve the pinned require version (to be bumped to the real tag at release, see release ordering notes). --- go.work | 6 ++ go.work.sum | 9 ++ observability/go.mod | 22 ++++ observability/go.sum | 24 +++++ observability/pathchksum.go | 120 ++++++++++++++++++++++ observability/pathchksum_test.go | 81 +++++++++++++++ observability/prometheus.go | 155 +++++++++++++++++++++++++++++ observability/prometheus_test.go | 95 ++++++++++++++++++ observability/promtest/promtest.go | 79 +++++++++++++++ 9 files changed, 591 insertions(+) create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 observability/go.mod create mode 100644 observability/go.sum create mode 100644 observability/pathchksum.go create mode 100644 observability/pathchksum_test.go create mode 100644 observability/prometheus.go create mode 100644 observability/prometheus_test.go create mode 100644 observability/promtest/promtest.go diff --git a/go.work b/go.work new file mode 100644 index 0000000..677c5db --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.23.0 + +use ( + . + ./observability +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..e1e37ab --- /dev/null +++ b/go.work.sum @@ -0,0 +1,9 @@ +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/observability/go.mod b/observability/go.mod new file mode 100644 index 0000000..669a7c5 --- /dev/null +++ b/observability/go.mod @@ -0,0 +1,22 @@ +module github.com/infobloxopen/hotload/observability + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + github.com/prometheus/client_golang v1.20.0 + github.com/prometheus/common v0.55.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) + +replace github.com/infobloxopen/hotload/v3 => ../ diff --git a/observability/go.sum b/observability/go.sum new file mode 100644 index 0000000..2923745 --- /dev/null +++ b/observability/go.sum @@ -0,0 +1,24 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/observability/pathchksum.go b/observability/pathchksum.go new file mode 100644 index 0000000..4b538a5 --- /dev/null +++ b/observability/pathchksum.go @@ -0,0 +1,120 @@ +package observability + +import ( + "hash/crc64" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/infobloxopen/hotload/v3/logger" +) + +// PathChksumMetricsEnableEnvVar gates checksum collection: hashing every +// watched file on each scrape is not free, so it is opt-in, exactly as in +// hotload v1. +const PathChksumMetricsEnableEnvVar = "HOTLOAD_PATH_CHKSUM_METRICS_ENABLE" + +// HotloadPathChksumTimestampSecondsName is the metric name, identical to +// hotload v1. +const HotloadPathChksumTimestampSecondsName = "hotload_path_chksum_timestamp_seconds" + +// FileHasher hashes a file's contents; replaceable for unit tests. +type FileHasher func(filePath string) (uint64, error) + +var crc64Table = crc64.MakeTable(crc64.ECMA) + +// DefaultFileHasher hashes file contents using CRC64. +func DefaultFileHasher(filePath string) (uint64, error) { + pathBytes, err := os.ReadFile(filePath) + if err != nil { + return 0, err + } + return crc64.Checksum(pathBytes, crc64Table), nil +} + +// PathChksumCollector is a prometheus.Collector reporting, per watched +// path, the unix timestamp at which the file's content checksum last +// changed. The checksum is computed at scrape time, so the metric stays +// accurate without a background poller. It replaces hotload v1's +// gaugefuncvec-based implementation. +type PathChksumCollector struct { + desc *prometheus.Desc + hasher FileHasher + enabled bool + + mu sync.Mutex + paths map[string]*chksumRecord +} + +type chksumRecord struct { + crc64 uint64 + lastChanged int64 +} + +// NewPathChksumCollector creates a collector using hasher (DefaultFileHasher +// when nil). Collection is enabled by PathChksumMetricsEnableEnvVar. +func NewPathChksumCollector(hasher FileHasher) *PathChksumCollector { + if hasher == nil { + hasher = DefaultFileHasher + } + enabled := false + switch strings.ToLower(strings.TrimSpace(os.Getenv(PathChksumMetricsEnableEnvVar))) { + case "1", "true", "yes": + enabled = true + } + return &PathChksumCollector{ + desc: prometheus.NewDesc( + HotloadPathChksumTimestampSecondsName, + "Hotload path checksum last changed (unix timestamp), by path", + []string{PathKey}, nil, + ), + hasher: hasher, + enabled: enabled, + paths: make(map[string]*chksumRecord), + } +} + +// AddPath starts reporting the checksum timestamp of pathStr. Duplicate adds +// are ignored. Fed automatically from hotload's watch events when the +// collector is wired through Collectors.Hooks. +func (p *PathChksumCollector) AddPath(pathStr string) { + if !p.enabled { + return + } + pathStr = cleanPath(pathStr) + p.mu.Lock() + defer p.mu.Unlock() + if _, found := p.paths[pathStr]; !found { + p.paths[pathStr] = &chksumRecord{} + } +} + +// Describe implements prometheus.Collector. +func (p *PathChksumCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- p.desc +} + +// Collect implements prometheus.Collector. Each watched file is hashed; a +// changed checksum bumps the path's last-changed timestamp. +func (p *PathChksumCollector) Collect(ch chan<- prometheus.Metric) { + p.mu.Lock() + defer p.mu.Unlock() + for pathStr, rec := range p.paths { + newCrc, err := p.hasher(pathStr) + if err != nil { + logger.ErrLogf("PathChksumCollector", "hashing %s failed: %v", pathStr, err) + } else if rec.crc64 != newCrc { + rec.crc64 = newCrc + rec.lastChanged = time.Now().Unix() + } + ch <- prometheus.MustNewConstMetric(p.desc, prometheus.GaugeValue, float64(rec.lastChanged), pathStr) + } +} + +func cleanPath(pathStr string) string { + return path.Clean(strings.TrimSpace(pathStr)) +} diff --git a/observability/pathchksum_test.go b/observability/pathchksum_test.go new file mode 100644 index 0000000..411b736 --- /dev/null +++ b/observability/pathchksum_test.go @@ -0,0 +1,81 @@ +package observability + +import ( + "fmt" + "strings" + "sync/atomic" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/infobloxopen/hotload/observability/promtest" +) + +func TestPathChksumCollector(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "true") + + var hash atomic.Uint64 + hash.Store(1) + c := NewPathChksumCollector(func(filePath string) (uint64, error) { + return hash.Load(), nil + }) + + c.AddPath("/etc/dsn") + c.AddPath("/etc/dsn") // duplicate adds are ignored + + expect := ` +# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path +# TYPE hotload_path_chksum_timestamp_seconds gauge +hotload_path_chksum_timestamp_seconds\{path="/etc/dsn"\} 1\.\d+e\+09 +` + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(expect), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("first scrape diff:\n%s", err) + } + + // An unchanged checksum keeps the timestamp; a changed one bumps it. + first := collectValue(t, c) + if v := collectValue(t, c); v != first { + t.Errorf("timestamp changed without content change: %v -> %v", first, v) + } + hash.Store(2) + if v := collectValue(t, c); v < first { + t.Errorf("timestamp went backwards after content change: %v -> %v", first, v) + } +} + +func TestPathChksumCollectorDisabled(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "") + + c := NewPathChksumCollector(func(filePath string) (uint64, error) { return 1, nil }) + c.AddPath("/etc/dsn") + + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(""), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("disabled collector should produce no series, got diff:\n%s", err) + } +} + +func TestPathChksumCollectorHashError(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "yes") + + c := NewPathChksumCollector(func(filePath string) (uint64, error) { + return 0, fmt.Errorf("boom") + }) + c.AddPath("/etc/dsn") + + // Hash errors keep the last value (zero here) rather than dropping the + // series or panicking. + expect := ` +# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path +# TYPE hotload_path_chksum_timestamp_seconds gauge +hotload_path_chksum_timestamp_seconds\{path="/etc/dsn"\} 0 +` + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(expect), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("unexpected diff:\n%s", err) + } +} + +// collectValue scrapes the single-series collector and returns its value. +func collectValue(t *testing.T, c *PathChksumCollector) float64 { + t.Helper() + return testutil.ToFloat64(c) +} diff --git a/observability/prometheus.go b/observability/prometheus.go new file mode 100644 index 0000000..6c8b85b --- /dev/null +++ b/observability/prometheus.go @@ -0,0 +1,155 @@ +// Package observability exports hotload activity as prometheus metrics. +// +// Hotload v3's core has no metrics dependency; it emits events through +// hotload.Hooks. This module adapts those events to prometheus collectors +// that preserve the metric names of hotload v1, so existing dashboards keep +// working. Unlike v1, registration is explicit: +// +// import "github.com/infobloxopen/hotload/observability" +// +// func main() { +// observability.MustEnablePrometheus(nil) // nil = prometheus.DefaultRegisterer +// ... +// } +package observability + +import ( + "github.com/prometheus/client_golang/prometheus" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// Label keys and values, identical to hotload v1's metrics package. +const ( + GRPCMethodKey = "grpc_method" + GRPCServiceKey = "grpc_service" + StatementKey = "stmt" // either exec or query + ExecStatement = "exec" + QueryStatement = "query" + + StrategyKey = "strategy" + PathKey = "path" + UrlKey = "url" +) + +// Metric names, identical to hotload v1. +const ( + SqlStmtsSummaryName = "transaction_sql_stmts" + HotloadChangeTotalName = "hotload_change_total" + HotloadLastChangedTimestampSecondsName = "hotload_last_changed_timestamp_seconds" + HotloadModtimeLatencyHistogramName = "hotload_modtime_latency_histogram" +) + +// HotloadModtimeLatencyHistogramDefBuckets are the default buckets (seconds) +// of the modtime latency histogram, identical to hotload v1. +var HotloadModtimeLatencyHistogramDefBuckets = []float64{900, 1800, 2700, 3600, 4500, 5400, 7200, 10800, 14400, 28800, 86400} + +// Collectors bundles the prometheus collectors fed by hotload hooks. +type Collectors struct { + // SqlStmtsSummary tracks the number of sql statements per transaction + // by statement type and the grpc service/method labels carried by the + // transaction context (see hotload.ContextWithExecLabels). + SqlStmtsSummary *prometheus.SummaryVec + // HotloadChangeTotal counts config changes per hotload DSN. + HotloadChangeTotal *prometheus.CounterVec + // HotloadLastChangedTimestampSeconds is the unix timestamp of the last + // config change per hotload DSN. + HotloadLastChangedTimestampSeconds *prometheus.GaugeVec + // HotloadModtimeLatencyHistogram tracks how stale watched files are, + // fed by the modtime monitor. + HotloadModtimeLatencyHistogram *prometheus.HistogramVec + // HotloadPathChksumTimestampSeconds reports when each watched file's + // content checksum last changed, computed at scrape time. Gated by the + // HOTLOAD_PATH_CHKSUM_METRICS_ENABLE environment variable. + HotloadPathChksumTimestampSeconds *PathChksumCollector +} + +// NewCollectors creates unregistered collectors with hotload v1's metric +// names. +func NewCollectors() *Collectors { + return &Collectors{ + SqlStmtsSummary: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Name: SqlStmtsSummaryName, + Help: "The number of sql stmts called in a transaction by statement type per grpc service and method", + }, []string{GRPCServiceKey, GRPCMethodKey, StatementKey}), + HotloadChangeTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: HotloadChangeTotalName, + Help: "Hotload change total by url", + }, []string{UrlKey}), + HotloadLastChangedTimestampSeconds: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: HotloadLastChangedTimestampSecondsName, + Help: "Hotload last changed (unix timestamp), by url", + }, []string{UrlKey}), + HotloadModtimeLatencyHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: HotloadModtimeLatencyHistogramName, + Help: "Hotload modtime latency histogram (seconds) by strategy and path", + Buckets: HotloadModtimeLatencyHistogramDefBuckets, + }, []string{StrategyKey, PathKey}), + HotloadPathChksumTimestampSeconds: NewPathChksumCollector(DefaultFileHasher), + } +} + +// All returns every collector, for manual registration. +func (c *Collectors) All() []prometheus.Collector { + return []prometheus.Collector{ + c.SqlStmtsSummary, + c.HotloadChangeTotal, + c.HotloadLastChangedTimestampSeconds, + c.HotloadModtimeLatencyHistogram, + c.HotloadPathChksumTimestampSeconds, + } +} + +// Hooks returns the hotload hooks that feed the collectors. Register them +// with hotload.RegisterHooks (EnablePrometheus does this for you). +func (c *Collectors) Hooks() hotload.Hooks { + return hotload.Hooks{ + OnConfigChange: func(ev hotload.ConfigChangeEvent) { + c.HotloadChangeTotal.WithLabelValues(ev.GroupName).Inc() + c.HotloadLastChangedTimestampSeconds.WithLabelValues(ev.GroupName).Set(float64(ev.At.Unix())) + }, + OnTxComplete: func(ev hotload.TxEvent) { + labels := hotload.GetExecLabelsFromContext(ev.Ctx) + service := labels[GRPCServiceKey] + method := labels[GRPCMethodKey] + c.SqlStmtsSummary.WithLabelValues(service, method, ExecStatement).Observe(float64(ev.ExecStmts)) + c.SqlStmtsSummary.WithLabelValues(service, method, QueryStatement).Observe(float64(ev.QueryStmts)) + }, + OnModTimeCheck: func(ev hotload.ModTimeEvent) { + c.HotloadModtimeLatencyHistogram.WithLabelValues(ev.Strategy, ev.Path).Observe(ev.Latency.Seconds()) + }, + OnWatch: func(ev hotload.WatchEvent) { + if !ev.Closed { + c.HotloadPathChksumTimestampSeconds.AddPath(ev.Path) + } + }, + } +} + +// EnablePrometheus creates the collectors, registers them with reg (the +// default prometheus registerer when reg is nil), and registers the hooks +// that feed them with hotload. Call it once during program initialization, +// before opening hotload connections. +func EnablePrometheus(reg prometheus.Registerer) (*Collectors, error) { + if reg == nil { + reg = prometheus.DefaultRegisterer + } + c := NewCollectors() + for _, collector := range c.All() { + if err := reg.Register(collector); err != nil { + return nil, err + } + } + hotload.RegisterHooks(c.Hooks()) + return c, nil +} + +// MustEnablePrometheus is EnablePrometheus, panicking on registration +// errors. +func MustEnablePrometheus(reg prometheus.Registerer) *Collectors { + c, err := EnablePrometheus(reg) + if err != nil { + panic(err) + } + return c +} diff --git a/observability/prometheus_test.go b/observability/prometheus_test.go new file mode 100644 index 0000000..43928c0 --- /dev/null +++ b/observability/prometheus_test.go @@ -0,0 +1,95 @@ +package observability + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/observability/promtest" +) + +func TestConfigChangeMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + at := time.Unix(1764000000, 0) + ev := hotload.ConfigChangeEvent{GroupName: "fsnotify://postgres/etc/dsn", At: at} + h.OnConfigChange(ev) + h.OnConfigChange(hotload.ConfigChangeEvent{GroupName: "fsnotify://postgres/etc/dsn", At: at.Add(time.Minute)}) + + if got := testutil.ToFloat64(c.HotloadChangeTotal.WithLabelValues(ev.GroupName)); got != 2 { + t.Errorf("hotload_change_total = %v, want 2", got) + } + want := float64(at.Add(time.Minute).Unix()) + if got := testutil.ToFloat64(c.HotloadLastChangedTimestampSeconds.WithLabelValues(ev.GroupName)); got != want { + t.Errorf("hotload_last_changed_timestamp_seconds = %v, want %v", got, want) + } +} + +func TestTxMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + ctx := hotload.ContextWithExecLabels(context.Background(), map[string]string{ + GRPCServiceKey: "svc", + GRPCMethodKey: "m", + }) + h.OnTxComplete(hotload.TxEvent{Ctx: ctx, ExecStmts: 3, QueryStmts: 1, Committed: true}) + + expect := ` +# HELP transaction_sql_stmts The number of sql stmts called in a transaction by statement type per grpc service and method +# TYPE transaction_sql_stmts summary +transaction_sql_stmts_sum\{grpc_method="m",grpc_service="svc",stmt="exec"\} 3 +transaction_sql_stmts_count\{grpc_method="m",grpc_service="svc",stmt="exec"\} 1 +transaction_sql_stmts_sum\{grpc_method="m",grpc_service="svc",stmt="query"\} 1 +transaction_sql_stmts_count\{grpc_method="m",grpc_service="svc",stmt="query"\} 1 +` + err := promtest.CollectAndRegexpCompare(c.SqlStmtsSummary, strings.NewReader(expect), SqlStmtsSummaryName) + if err != nil { + t.Errorf("unexpected metrics diff:\n%s", err) + } +} + +func TestModTimeMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + h.OnModTimeCheck(hotload.ModTimeEvent{Strategy: "fsnotify", Path: "/etc/dsn", Latency: 1000 * time.Second}) + + if got := testutil.CollectAndCount(c.HotloadModtimeLatencyHistogram, HotloadModtimeLatencyHistogramName); got != 1 { + t.Errorf("histogram series count = %d, want 1", got) + } +} + +func TestEnablePrometheus(t *testing.T) { + reg := prometheus.NewRegistry() + c, err := EnablePrometheus(reg) + if err != nil { + t.Fatal(err) + } + + c.Hooks().OnConfigChange(hotload.ConfigChangeEvent{GroupName: "g", At: time.Now()}) + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + found := false + for _, mf := range families { + if mf.GetName() == HotloadChangeTotalName { + found = true + } + } + if !found { + t.Errorf("%s not gatherable from registry", HotloadChangeTotalName) + } + + // Double registration must error, not panic. + if _, err := EnablePrometheus(reg); err == nil { + t.Error("second EnablePrometheus on same registry should error") + } +} diff --git a/observability/promtest/promtest.go b/observability/promtest/promtest.go new file mode 100644 index 0000000..64de3c0 --- /dev/null +++ b/observability/promtest/promtest.go @@ -0,0 +1,79 @@ +// Package promtest provides helpers for asserting prometheus metric output +// against regexp patterns. Hotload v1 shipped these in its internal +// package; they live here so the hotload core stays free of prometheus +// dependencies. +package promtest + +import ( + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/expfmt" +) + +// CollectAndRegexpCompare is similar to testutil.CollectAndCompare() +// but the expected lines are regexp patterns. +// Note that unlike testutil.CollectAndCompare(), +// the metricName MUST be specified to get any collected result. +func CollectAndRegexpCompare(colltor prometheus.Collector, expectRdr io.Reader, metricNames ...string) error { + expectBytes, err := io.ReadAll(expectRdr) + if err != nil { + return err + } + + collectBytes, err := testutil.CollectAndFormat(colltor, expfmt.TypeTextPlain, metricNames...) + if err != nil { + return err + } + + expectStr := strings.TrimSpace(string(expectBytes)) + collectStr := strings.TrimSpace(string(collectBytes)) + + expectSplit := strings.Split(expectStr, "\n") + collectSplit := strings.Split(collectStr, "\n") + + diffStr := strings.TrimSpace(SimpleRegexpLineDiff(expectSplit, collectSplit)) + if len(diffStr) > 0 { + return errors.New(diffStr) + } + return nil +} + +// SimpleRegexpLineDiff performs a simple/dumb line-by-line diff +// between two arrays of lines. The expected array of lines are regexp +// patterns. Returns line(s) which diff. Empty string is returned if there +// are no diffs. +func SimpleRegexpLineDiff(regexpLines []string, gotLines []string) string { + maxLen := max(len(regexpLines), len(gotLines)) + + for len(regexpLines) < maxLen { + regexpLines = append(regexpLines, "") + } + for len(gotLines) < maxLen { + gotLines = append(gotLines, "") + } + + var diffBuf strings.Builder + for k := 0; k < maxLen; k++ { + expStr := strings.TrimSpace(regexpLines[k]) + gotStr := strings.TrimSpace(gotLines[k]) + expPat := `^` + expStr + `$` + + matched, err := regexp.MatchString(expPat, gotStr) + if err != nil { + return err.Error() + } + + if !matched { + fmt.Fprintf(&diffBuf, "-%s\n", expStr) + fmt.Fprintf(&diffBuf, "+%s\n", gotStr) + } + } + + return diffBuf.String() +} From 4590b72009d94db5d5aa1945a5382e28e304cbaa Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 00:15:47 -0500 Subject: [PATCH 04/14] Add test/integration nested module with stdlib postgres tests Replace the ginkgo-based integrationtests package with a nested module (never tagged or imported) holding plain-testing scenarios against a real PostgreSQL via lib/pq and the fsnotify strategy: database switchover, password rotation during long Exec/ExecContext/Query/ QueryContext (graceful completes, forceKill cancels), transactions spanning a config change in both modes, caller context cancellation, prepared statements re-preparing across a change, and sequential long execs each overlapping one rotation. Each test writes its own DSN file under t.TempDir, so every test gets an isolated hotload group. Tests skip when no postgres is reachable; HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST/PORT are honored as before. The docker-compose setup and init SQL carry over unchanged. --- go.work | 1 + go.work.sum | 34 ++- test/integration/docker/docker-compose.yaml | 18 ++ test/integration/docker/intgtest_init.sql | 5 + test/integration/go.mod | 17 ++ test/integration/go.sum | 7 + test/integration/hotload_test.go | 303 ++++++++++++++++++++ test/integration/main_test.go | 175 +++++++++++ 8 files changed, 551 insertions(+), 9 deletions(-) create mode 100644 test/integration/docker/docker-compose.yaml create mode 100644 test/integration/docker/intgtest_init.sql create mode 100644 test/integration/go.mod create mode 100644 test/integration/go.sum create mode 100644 test/integration/hotload_test.go create mode 100644 test/integration/main_test.go diff --git a/go.work b/go.work index 677c5db..90b372f 100644 --- a/go.work +++ b/go.work @@ -3,4 +3,5 @@ go 1.23.0 use ( . ./observability + ./test/integration ) diff --git a/go.work.sum b/go.work.sum index e1e37ab..40f47aa 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,9 +1,25 @@ -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/integration/docker/docker-compose.yaml b/test/integration/docker/docker-compose.yaml new file mode 100644 index 0000000..3c4d594 --- /dev/null +++ b/test/integration/docker/docker-compose.yaml @@ -0,0 +1,18 @@ +version: '3.8' + +# postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml +services: + db: + image: postgres:10.3 + environment: + - POSTGRES_USER=admin + - POSTGRES_PASSWORD=test + - POSTGRES_DB=hldatabase + - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true + ports: + - '5432:5432' + volumes: + # https://github.com/felipewom/docker-compose-postgres + # https://geshan.com.np/blog/2021/12/docker-postgres/ + # https://github.com/docker-library/docs/tree/master/postgres#initialization-scripts + - ./intgtest_init.sql:/docker-entrypoint-initdb.d/intgtest_init.sql diff --git a/test/integration/docker/intgtest_init.sql b/test/integration/docker/intgtest_init.sql new file mode 100644 index 0000000..ff3b4b0 --- /dev/null +++ b/test/integration/docker/intgtest_init.sql @@ -0,0 +1,5 @@ +-- postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml +CREATE DATABASE hotload_test; +CREATE DATABASE hotload_test1; +GRANT ALL PRIVILEGES ON DATABASE hotload_test TO admin; +GRANT ALL PRIVILEGES ON DATABASE hotload_test1 TO admin; diff --git a/test/integration/go.mod b/test/integration/go.mod new file mode 100644 index 0000000..cc0ec19 --- /dev/null +++ b/test/integration/go.mod @@ -0,0 +1,17 @@ +module github.com/infobloxopen/hotload/test/integration + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + github.com/lib/pq v1.10.9 +) + +require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + golang.org/x/sys v0.35.0 // indirect +) + +// This module exists only to test the sibling modules in this repository; +// it is never tagged or imported, so the replace directive is permanent. +replace github.com/infobloxopen/hotload/v3 => ../../ diff --git a/test/integration/go.sum b/test/integration/go.sum new file mode 100644 index 0000000..0447aa2 --- /dev/null +++ b/test/integration/go.sum @@ -0,0 +1,7 @@ +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/test/integration/hotload_test.go b/test/integration/hotload_test.go new file mode 100644 index 0000000..8337d7e --- /dev/null +++ b/test/integration/hotload_test.go @@ -0,0 +1,303 @@ +package integration + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// TestSwitchDatabase: changing the watched DSN reroutes new connections to +// another database, observable via current_database(). +func TestSwitchDatabase(t *testing.T) { + requirePostgres(t) + + f := newDsnFile(t, adminDsn("hotload_test")) + db := openHotload(t, f, "") + + waitForQueryValue(t, db, "SELECT current_database()", "hotload_test", 5*time.Second) + + f.set(adminDsn("hotload_test1")) + waitForQueryValue(t, db, "SELECT current_database()", "hotload_test1", 10*time.Second) +} + +// TestPasswordRotationGraceful: rotating the database password and the DSN +// must transparently move the pool to fresh credentials; in-flight long +// operations on old connections complete undisturbed. +func TestPasswordRotationGraceful(t *testing.T) { + requirePostgres(t) + + ops := map[string]func(db *sql.DB) error{ + "Exec": func(db *sql.DB) error { + _, err := db.Exec("SELECT pg_sleep(2)") + return err + }, + "ExecContext": func(db *sql.DB) error { + _, err := db.ExecContext(context.Background(), "SELECT pg_sleep(2)") + return err + }, + "Query": func(db *sql.DB) error { + rows, err := db.Query("SELECT pg_sleep(2)") + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() + }, + "QueryContext": func(db *sql.DB) error { + rows, err := db.QueryContext(context.Background(), "SELECT pg_sleep(2)") + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() + }, + } + + for name, op := range ops { + t.Run(name, func(t *testing.T) { + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + + errCh := make(chan error, 1) + go func() { errCh <- op(db) }() + time.Sleep(300 * time.Millisecond) // let the long op reach the server + + newPass := rotatePassword(t) + f.set(userDsn(newPass)) + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("graceful mode must let the in-flight %s finish, got: %v", name, err) + } + case <-time.After(15 * time.Second): + t.Fatal("long operation did not return") + } + + // New connections must authenticate with the new password. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + }) + } +} + +// TestPasswordRotationForceKill: with forceKill the in-flight operation is +// cut when the DSN changes, and the pool recovers on the new credentials. +func TestPasswordRotationForceKill(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "forceKill=true") + + errCh := make(chan error, 1) + go func() { + _, err := db.Exec("SELECT pg_sleep(30)") + errCh <- err + }() + time.Sleep(300 * time.Millisecond) + + newPass := rotatePassword(t) + f.set(userDsn(newPass)) + + select { + case err := <-errCh: + if err == nil { + t.Fatal("forceKill mode must cancel the in-flight exec, got nil error") + } + t.Logf("in-flight exec canceled with: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("forceKill did not cancel the in-flight exec (30s sleep still running)") + } + + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) +} + +// TestLongTransactionAcrossChangeGraceful: a transaction spanning a config +// change completes on its original connection. +func TestLongTransactionAcrossChangeGraceful(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + + tag := fmt.Sprintf("txn-%d", time.Now().UnixNano()) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (1, $1)", tag); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("SELECT pg_sleep(1)"); err != nil { + t.Fatal(err) + } + + f.set(userDsn(rotatePassword(t))) + time.Sleep(300 * time.Millisecond) // let the change land mid-transaction + + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (2, $1)", tag); err != nil { + t.Fatalf("exec after change in graceful txn: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit after change in graceful txn: %v", err) + } + + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM test WHERE csource = $1", tag).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("committed rows = %d, want 2", n) + } +} + +// TestLongTransactionAcrossChangeForceKill: with forceKill a transaction +// holding a connection across a change is rolled back by the kill. +func TestLongTransactionAcrossChangeForceKill(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "forceKill=true") + + tag := fmt.Sprintf("txnfk-%d", time.Now().UnixNano()) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (1, $1)", tag); err != nil { + t.Fatal(err) + } + + f.set(userDsn(rotatePassword(t))) + time.Sleep(500 * time.Millisecond) // change lands; connection is killed + + err = func() error { + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (2, $1)", tag); err != nil { + return err + } + return tx.Commit() + }() + if err == nil { + t.Fatal("transaction across a forceKill change should fail") + } + t.Logf("transaction failed as expected: %v", err) + + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM test WHERE csource = $1", tag).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("rows from killed transaction = %d, want 0 (rolled back)", n) + } +} + +// TestContextCancellation: caller-driven cancellation behaves identically +// in both modes and leaves the pool healthy. +func TestContextCancellation(t *testing.T) { + requirePostgres(t) + + for _, params := range []string{"", "forceKill=true"} { + name := "graceful" + if params != "" { + name = "forceKill" + } + t.Run(name, func(t *testing.T) { + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, params) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + _, err := db.ExecContext(ctx, "SELECT pg_sleep(10)") + if err == nil { + t.Fatal("expected context cancellation error") + } + if !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(err.Error(), "cancel") { + t.Fatalf("err = %v, want context cancellation", err) + } + + // The pool must remain healthy after the cancellation. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + }) + } +} + +// TestPreparedStatementAcrossChange: a prepared statement survives a +// graceful change — database/sql re-prepares it on a new connection. +func TestPreparedStatementAcrossChange(t *testing.T) { + requirePostgres(t) + + f := newDsnFile(t, adminDsn("hotload_test")) + db := openHotload(t, f, "") + + stmt, err := db.Prepare("SELECT current_database()") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + var got string + if err := stmt.QueryRow().Scan(&got); err != nil { + t.Fatal(err) + } + if got != "hotload_test" { + t.Fatalf("before change: %q, want hotload_test", got) + } + + f.set(adminDsn("hotload_test1")) + deadline := time.Now().Add(10 * time.Second) + for { + if err := stmt.QueryRow().Scan(&got); err != nil { + t.Fatalf("prepared stmt after change: %v", err) + } + if got == "hotload_test1" { + break + } + if time.Now().After(deadline) { + t.Fatalf("prepared stmt still routed to %q", got) + } + time.Sleep(50 * time.Millisecond) + } +} + +// TestMultipleSequentialLongExecs: a sequence of long operations, each +// overlapping one password rotation, all complete in graceful mode (each +// in-flight operation stays within its generation's grace period; only the +// generation before the previous one is killed). +func TestMultipleSequentialLongExecs(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + db.SetMaxOpenConns(10) + + for i := 0; i < 5; i++ { + errCh := make(chan error, 1) + go func() { + _, err := db.Exec("SELECT pg_sleep(1)") + errCh <- err + }() + time.Sleep(150 * time.Millisecond) // let the exec reach the server + + f.set(userDsn(rotatePassword(t))) + + if err := <-errCh; err != nil { + t.Fatalf("round %d: long exec failed in graceful mode: %v", i, err) + } + // Confirm the swap landed before the next round dials; transient + // auth failures while the new DSN propagates are retried here. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + } +} diff --git a/test/integration/main_test.go b/test/integration/main_test.go new file mode 100644 index 0000000..7d224e1 --- /dev/null +++ b/test/integration/main_test.go @@ -0,0 +1,175 @@ +// Package integration tests hotload against a real PostgreSQL server using +// the lib/pq driver and the fsnotify strategy. +// +// Start the database with `make postgres-docker-compose-up` (or point the +// HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST/PORT environment variables at an +// existing server provisioned with docker/intgtest_init.sql). Tests skip +// themselves when no server is reachable. +package integration + +import ( + "database/sql" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + _ "github.com/infobloxopen/hotload/v3/fsnotify" + "github.com/lib/pq" +) + +var ( + postgresHost = "localhost" + postgresPort = "5432" + + adminUser = "admin" + adminPass = "test" + testDbUser = "uuser" + + // passSeq makes every password rotation unique across tests. + passSeq atomic.Int64 + + setupOnce sync.Once + setupErr error + adminDB *sql.DB +) + +func TestMain(m *testing.M) { + hotload.RegisterSQLDriver("postgres", &pq.Driver{}) + + if h := strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST")); h != "" { + postgresHost = h + } + if p := strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT")); p != "" { + postgresPort = p + } + + os.Exit(m.Run()) +} + +func adminDsn(database string) string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", + adminUser, adminPass, postgresHost, postgresPort, database) +} + +func userDsn(pass string) string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", + testDbUser, pass, postgresHost, postgresPort) +} + +// requirePostgres skips the test when no server is reachable and lazily +// provisions the test user and table. +func requirePostgres(t *testing.T) { + t.Helper() + addr := net.JoinHostPort(postgresHost, postgresPort) + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + t.Skipf("skipping: postgres not reachable at %s (start it with `make postgres-docker-compose-up`): %v", addr, err) + } + conn.Close() + + setupOnce.Do(func() { + adminDB, setupErr = sql.Open("postgres", adminDsn("hldatabase")) + if setupErr != nil { + return + } + stmts := []string{ + "DROP TABLE IF EXISTS test", + "DROP USER IF EXISTS " + testDbUser, + fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", testDbUser, nextPass(0)), + "CREATE TABLE IF NOT EXISTS test (cnum INT, csource TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)", + "GRANT ALL ON test TO PUBLIC", + } + for _, stmt := range stmts { + if _, setupErr = adminDB.Exec(stmt); setupErr != nil { + setupErr = fmt.Errorf("setup %q: %w", stmt, setupErr) + return + } + } + }) + if setupErr != nil { + t.Fatalf("postgres setup failed: %v", setupErr) + } +} + +// nextPass returns the password for rotation n (or the next unique one when +// n is 0). +func nextPass(n int64) string { + if n == 0 { + n = passSeq.Add(1) + } + return fmt.Sprintf("ppass%d", n) +} + +// rotatePassword sets a fresh password for the test user on the server and +// returns it. Existing sessions stay valid; only new dials use the new one. +func rotatePassword(t *testing.T) string { + t.Helper() + pass := nextPass(0) + if _, err := adminDB.Exec(fmt.Sprintf("ALTER USER %s WITH PASSWORD '%s'", testDbUser, pass)); err != nil { + t.Fatalf("rotatePassword: %v", err) + } + return pass +} + +// dsnFile manages the config file watched by the fsnotify strategy. Each +// test gets its own path, hence its own hotload group. +type dsnFile struct { + t *testing.T + path string +} + +func newDsnFile(t *testing.T, dsn string) *dsnFile { + t.Helper() + f := &dsnFile{t: t, path: filepath.Join(t.TempDir(), "dsn.txt")} + f.set(dsn) + return f +} + +func (f *dsnFile) set(dsn string) { + f.t.Helper() + if err := os.WriteFile(f.path, []byte(dsn), 0o644); err != nil { + f.t.Fatalf("writing dsn file: %v", err) + } +} + +// openHotload opens a hotload sql.DB watching f. +func openHotload(t *testing.T, f *dsnFile, params string) *sql.DB { + t.Helper() + url := "fsnotify://postgres" + f.path + if params != "" { + url += "?" + params + } + db, err := sql.Open("hotload", url) + if err != nil { + t.Fatalf("sql.Open(%q): %v", url, err) + } + t.Cleanup(func() { db.Close() }) + if err := db.Ping(); err != nil { + t.Fatalf("ping through hotload: %v", err) + } + return db +} + +// waitForQueryValue polls query until it returns want or the timeout +// elapses. +func waitForQueryValue(t *testing.T, db *sql.DB, query, want string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last string + var lastErr error + for time.Now().Before(deadline) { + lastErr = db.QueryRow(query).Scan(&last) + if lastErr == nil && last == want { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for %q to return %q (last value %q, last err %v)", query, want, last, lastErr) +} From 9d886f9eea5ef74f9b690800d0009e8c202e3357 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 00:19:43 -0500 Subject: [PATCH 05/14] Update Makefile, CI and docs for the multi-module v3 layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makefile: per-module fmt/vet/tidy/build/test loops, a generate target with a no-diff guard for the generated wrappers, and a dep-budget check asserting the root module's only direct dependency is fsnotify (internal/depbudget parses go mod edit -json with the stdlib). CI: replace the docker-build and kind/helm pipelines with native setup-go jobs — unit tests on a Go 1.23/1.24 matrix via make ci-test, and integration tests against docker-compose postgres. Remove the Dockerfiles, helm/kind targets and stale testdata. Docs: README rewritten for /v3 (corrects the long-stale Strategy.Watch signature, documents truthful capability mirroring, killWindow, connector lifecycle and opt-in metrics); MIGRATION.md covers the v1->v3 upgrade with the metrics call-out front and center; RELEASING.md documents the nested-module tag ordering and a consumer smoke check. --- .github/workflows/go.yml | 37 ++++++--- MIGRATION.md | 112 ++++++++++++++++++++++++++ Makefile | 102 ++++++++++-------------- README.md | 133 +++++++++++++++++++++++++------ RELEASING.md | 50 ++++++++++++ internal/depbudget/main.go | 31 +++++++ observability/prometheus_test.go | 2 +- test/integration/main_test.go | 22 ++++- 8 files changed, 390 insertions(+), 99 deletions(-) create mode 100644 MIGRATION.md create mode 100644 RELEASING.md create mode 100644 internal/depbudget/main.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 97e3e1c..1600965 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,21 +7,32 @@ on: branches: [ main ] jobs: - build: + unit: runs-on: ubuntu-latest + strategy: + matrix: + # database/sql's bad-conn retry behavior shifts subtly between Go + # releases; test the two most recent. + go-version: [ '1.23.x', '1.24.x' ] steps: - - uses: actions/checkout@v2 - - name: Unit Tests - run: make test-docker + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - name: Lint, generate-check, unit tests, dependency budget + run: make ci-test - integration-tests: + integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - uses: engineerd/setup-kind@v0.5.0 - with: - version: "v0.11.1" - - name: Integration Tests - run: | - kubectl cluster-info - make ci-integration-tests + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.24.x' + - name: Start postgres + run: make postgres-docker-compose-up + - name: Integration tests + run: make local-integration-tests + - name: Stop postgres + if: always() + run: make postgres-docker-compose-down diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..3517cf2 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,112 @@ +# Migrating from hotload v1 to v3 + +For most applications the upgrade is an import-path change. The driver name +(`"hotload"`), the DSN format (`strategy://driver/path?forceKill=...`), the +registration functions, the `Strategy` interface and the graceful/forceKill +semantics are unchanged. + +## 1. Import path + +```diff +-import "github.com/infobloxopen/hotload" +-import _ "github.com/infobloxopen/hotload/fsnotify" ++import "github.com/infobloxopen/hotload/v3" ++import _ "github.com/infobloxopen/hotload/v3/fsnotify" +``` + +```sh +go get github.com/infobloxopen/hotload/v3 +``` + +## 2. Metrics are now opt-in (action required if you scrape hotload metrics) + +**This is the most important behavioral change.** Hotload v1 registered its +prometheus metrics with the default registerer as a side effect of importing +the package. The v3 core has no prometheus dependency at all; if you do +nothing, you get no metrics. + +To keep your dashboards working, add the observability module: + +```sh +go get github.com/infobloxopen/hotload/observability +``` + +```go +import "github.com/infobloxopen/hotload/observability" + +func main() { + observability.MustEnablePrometheus(nil) // nil = prometheus.DefaultRegisterer + ... +} +``` + +Metric names, labels, and the `HOTLOAD_PATH_CHKSUM_METRICS_ENABLE` gate are +identical to v1: + +| v1 metric | v3 | +|---|---| +| `transaction_sql_stmts` | same name, via observability module | +| `hotload_change_total` | same name, via observability module | +| `hotload_last_changed_timestamp_seconds` | same name, via observability module | +| `hotload_modtime_latency_histogram` | same name, via observability module | +| `hotload_path_chksum_timestamp_seconds` | same name, via observability module | + +One accuracy improvement: statements executed through prepared statements now +count toward `transaction_sql_stmts` (v1 did not wrap `driver.Stmt`, so +prepared-statement traffic was invisible). + +The `hotload/metrics` package is gone. Its helper APIs map as follows: + +| v1 | v3 | +|---|---| +| `metrics.GetCollectors()` | `observability.NewCollectors().All()` or the return of `EnablePrometheus` | +| `metrics.GRPCServiceKey` etc. | `observability.GRPCServiceKey` etc. | +| `internal.CollectAndRegexpCompare` (not exported in v1) | `observability/promtest.CollectAndRegexpCompare` | + +`hotload.ContextWithExecLabels` / `hotload.GetExecLabelsFromContext` are +unchanged and remain in the core. + +## 3. Truthful driver capabilities + +v1 implemented a fixed set of optional `database/sql/driver` interfaces and +answered `driver.ErrSkip` for unsupported ones. v3 wraps connections and +statements so optional interfaces exist if and only if the underlying driver +supports them. For mainstream drivers (lib/pq, pgx stdlib, mysql) behavior is +unchanged; for minimal drivers, `database/sql` now takes its documented +fallback paths (e.g. prepared statements) instead of `ErrSkip` round-trips. + +Code that type-asserts on the raw driver conn (e.g. inside `sql.Conn.Raw`) +sees the new truthful method set. + +## 4. Lifecycle changes + +- **Errors at `sql.Open`:** the strategy watch starts when `sql.Open` is + called (hotload now implements `driver.DriverContext`), so a bad DSN, + unknown strategy/driver, or unreadable config file fails fast at `sql.Open` + instead of at first query. +- **Teardown:** closing the last `sql.DB` for a DSN now stops the strategy + watch and the background goroutine; v1 leaked both. +- **forceKill:** in-flight operations canceled by a config change fail with + an error matching `hotload.ErrHotSwap` (use `errors.Is`). The new + `killWindow` DSN parameter (default `100ms`) bounds how long hotload waits + for in-flight work to observe cancellation before force-closing + connections. +- **modtime:** `modtime.NewModTimeMonitor` API is unchanged but reports + latency through hotload hooks; enable the observability module to keep the + histogram. + +## 5. Removed/changed APIs + +| v1 | v3 | +|---|---| +| `hotload.WithLogger` / `hotload.GetLogger` | still present (deprecated); prefer `logger.WithLogger` / `logger.GetLogger` | +| `hotload/metrics` package | removed; see section 2 | +| `hotload.Register` (alias mentioned in old README) | was already `RegisterSQLDriver`; unchanged | + +## 6. Dependency diet + +The v3 core module depends only on `github.com/fsnotify/fsnotify`. The +following are gone from your module graph unless you import the +observability module: prometheus (client_golang, common, client_model, +procfs), pkg/errors, google/uuid, teivah/onecontext, colega/gaugefuncvec, +ginkgo/gomega, go-sqlmock, lib/pq. diff --git a/Makefile b/Makefile index 4684384..a83cf5f 100644 --- a/Makefile +++ b/Makefile @@ -1,76 +1,58 @@ -GIT_COMMIT ?= $(shell git describe --dirty=-unsupported --always --tags || echo pre-commit) -IMAGE_NAME ?= hotload-integration-tests:$(GIT_COMMIT) +# The repository holds three Go modules: the hotload core (.), the +# prometheus adapter (observability), and the postgres integration tests +# (test/integration). The committed go.work ties them together for +# development; most targets loop over all of them. +MODULES := . observability test/integration -get: - go get -t ./... +.PHONY: fmt vet tidy build test generate no-diff dep-budget ci-test \ + postgres-docker-compose-up postgres-docker-compose-down local-integration-tests -fmt: get - go fmt ./... +fmt: + @for m in $(MODULES); do (cd $$m && go fmt ./...) || exit 1; done -tidy: - go mod tidy - -# assert that there is no difference after running format -no-diff: - git diff --exit-code - -vet: fmt - go vet ./... - -build: vet - go build ./... - -get-ginkgo: - go get github.com/onsi/ginkgo/v2/ginkgo - -test: vet get-ginkgo go-test +vet: + @for m in $(MODULES); do (cd $$m && go vet ./...) || exit 1; done -go-test: - go test -race github.com/infobloxopen/hotload \ - github.com/infobloxopen/hotload/fsnotify \ - github.com/infobloxopen/hotload/internal \ - github.com/infobloxopen/hotload/metrics \ - github.com/infobloxopen/hotload/modtime - - -# test target which includes the no-diff fail condition -ci-test: fmt tidy no-diff test - -test-docker: - docker build -f Dockerfile.test . - -.integ-test-image-$(GIT_COMMIT): - docker build -f Dockerfile.integrationtest . -t $(IMAGE_NAME) - -integ-test-image: .integ-test-image-$(GIT_COMMIT) +tidy: + @for m in $(MODULES); do (cd $$m && go mod tidy) || exit 1; done -# this'll run outside of the build container -deploy-integration-tests: - helm upgrade hotload-integration-tests integrationtests/helm/hotload-integration-tests -i --set image.tag=$(GIT_COMMIT) +build: + @for m in $(MODULES); do (cd $$m && go build ./...) || exit 1; done -build-test: vet get-ginkgo - go test -c ./integrationtests +# Unit tests. The integration module skips itself when postgres is not +# reachable; use local-integration-tests to run it for real. +test: + @for m in $(MODULES); do (cd $$m && go test -race -timeout=5m -count=1 ./...) || exit 1; done -kind-create-cluster: - kind create cluster +# Regenerate the optional-interface combination wrappers (conn/stmt) and the +# dbfake capability views. +generate: + go generate ./... -kind-load: - kind load docker-image $(IMAGE_NAME) +# assert that there is no difference after running format/tidy/generate +no-diff: + git diff --exit-code -ci-integration-tests: integ-test-image kind-load deploy-integration-tests - (helm test --timeout=600s hotload-integration-tests || (kubectl logs hotload-integration-tests-job && exit 1)) && kubectl logs hotload-integration-tests-job +# The hotload core must stay near-stdlib-only: its sole direct dependency is +# fsnotify. Fails when dependency creep adds more. +dep-budget: + @reqs=$$(go mod edit -json | go run ./internal/depbudget); \ + if [ "$$reqs" != "github.com/fsnotify/fsnotify" ]; then \ + echo "dependency budget exceeded; direct requires of the root module:"; \ + echo "$$reqs"; \ + exit 1; \ + fi -delete-all: - helm uninstall hotload-integration-tests || true - kubectl delete pvc --all || true - kubectl delete pods --all || true +ci-test: fmt tidy generate no-diff vet test dep-budget postgres-docker-compose-up: - cd integrationtests/docker; docker compose up --detach + cd test/integration/docker; docker compose up --detach --wait postgres-docker-compose-down: - cd integrationtests/docker; docker compose down + cd test/integration/docker; docker compose down -# Requires postgres db, see target postgres-docker-compose-up +# Requires postgres, see target postgres-docker-compose-up local-integration-tests: - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true go test -v -race -timeout=3m -count=1 github.com/infobloxopen/hotload/integrationtests + cd test/integration && \ + HOTLOAD_INTEGRATION_TESTS=1 HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true \ + go test -v -race -timeout=5m -count=1 ./... diff --git a/README.md b/README.md index 847101e..19c14f3 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ -[![Go Reference](https://pkg.go.dev/badge/github.com/infobloxopen/hotload.svg)](https://pkg.go.dev/github.com/infobloxopen/hotload) +[![Go Reference](https://pkg.go.dev/badge/github.com/infobloxopen/hotload/v3.svg)](https://pkg.go.dev/github.com/infobloxopen/hotload/v3) # hotload Hotload is a Golang `database/sql` compatible package that supports dynamic reloading of database configuration. In the typical use of `sql.Open()`, users must close the returned DB object and recreate it to change the connection string. Hotload works by registering a driver that proxies -the [`Driver` interface](https://pkg.go.dev/database/sql/driver#Driver) +the [`Driver` interface](https://pkg.go.dev/database/sql/driver#Driver) to the real database driver. When config changes -are detected it closes connections in a manner that causes the `database/sql` +are detected it retires connections in a manner that causes the `database/sql` package to create new connections with the new connection parameters. +Upgrading from hotload v1? See [MIGRATION.md](MIGRATION.md). + ```go import ( // import the std lib sql package @@ -17,10 +19,10 @@ import ( log "github.com/sirupsen/logrus" // this import registers hotload with the sql package - "github.com/infobloxopen/hotload" + "github.com/infobloxopen/hotload/v3" // this import registers the fsnotify hotload strategy - _ "github.com/infobloxopen/hotload/fsnotify" + _ "github.com/infobloxopen/hotload/v3/fsnotify" // this import registers the postgres driver with the sql package "github.com/lib/pq" @@ -28,7 +30,7 @@ import ( func init() { // this function call registers the lib/pq postgres driver with hotload - hotload.Register("postgres", pq.Driver{}) + hotload.RegisterSQLDriver("postgres", &pq.Driver{}) } func main() { @@ -47,13 +49,29 @@ The above code: In the `main()` function, the `sql.Open` call uses the hotload driver. The URL for the connection string specifies `fsnotify` in the scheme. This is the hotload strategy. The -hostname in the URL specifies the real database driver (`postgres` in the example above). +hostname in the URL specifies the real database driver (`postgres` in the example above). Finally, the path and query parameters are left for the hotload strategy plugin to configure themselves. Below is an example of a `lib/pq` Postgres connection string that would have been stored at `/tmp/myconfig.txt` ``` user=pqgotest dbname=pqgotest sslmode=verify-full ``` +# Faithful driver capabilities + +`database/sql` discovers what a driver can do through type assertions on the +optional interfaces of `database/sql/driver` (`ExecerContext`, +`QueryerContext`, `Pinger`, `NamedValueChecker`, `StmtExecContext`, …). +Hotload wraps connections and statements so that a wrapped object implements +an optional interface **if and only if** the underlying driver supports the +capability. If your driver has no `ExecerContext` (or legacy `Execer`), +`database/sql` correctly falls back to prepared statements instead of +bouncing off `driver.ErrSkip`. + +The interfaces hotload needs for its own lifecycle (`ConnPrepareContext`, +`ConnBeginTx`, `SessionResetter`, `Validator`) are always implemented; when +the underlying driver lacks them, hotload replicates `database/sql`'s own +fallback behavior exactly, so the difference is unobservable. + # Strategies Hotload has an interface for adding reload strategies. The interface looks like this: @@ -63,44 +81,113 @@ type Strategy interface { // Watch returns back the contents of the resource as well as a channel // for subsequent updates (if the value has changed). If there is an error // getting the initial value, an error is returned. - Watch(ctx context.Context, pth string, options url.Values) (value string, values <-chan string, err error) + Watch(ctx context.Context, pth string, pathQry string) (value string, newValChan <-chan string, err error) + + // CloseWatch closes the specified watch. + CloseWatch(pth string, pathQry string) error + + // Close resets/closes strategy, in particular closes all the update channels. + Close() } ``` The strategies are loaded by calling the `RegisterStrategy` method in the hotload package. This is the same pattern the `database/sql` package uses for loading drivers. The strategy implements the `Watch` method. The context passed to the strategy should be used to shut -down any code watching the passed `pth`. Options are taken from the hotload connection -string query parameters. The strategy doesn't have to use a real file to load the config. -`pth` represents a unique string that makes sense to the strategy. For example, pth could -point to a path in etcd or a kind/id in k8s. +down any code watching the passed `pth`. `pathQry` carries the hotload connection +string's encoded query parameters. The strategy doesn't have to use a real file to load +the config. `pth` represents a unique string that makes sense to the strategy. For +example, pth could point to a path in etcd or a kind/id in k8s. The hotload project ships with one hotload strategy: `fsnotify`. -Note: In your project, if you do not implement your own `Strategy`, and instead choose to use the out-of-the-box -`fsnotify` strategy, you must import the `fsnotify` package in your project to register at least one strategy with +Note: In your project, if you do not implement your own `Strategy`, and instead choose to use the out-of-the-box +`fsnotify` strategy, you must import the `fsnotify` package in your project to register at least one strategy with hotload, otherwise an error will occur at runtime as the `database/sql` package will not be able to locate/load your intended hotload strategy as a recognizable driver. # Force Kill -By default, the hotload driver gracefully closes connections to the underlying driver. If your application holds connections open with long-running operations, this will prevent graceful switchover to new data sources. - -Adding `forceKill=true` to your DSN will cause the hotload driver to close the underlying connection manually when a -change to the connection information is detected. +By default, the hotload driver gracefully retires connections to the underlying driver: +in-flight work finishes on the old connection, and the pool replaces connections the +next time it reuses them. A retired generation gets one grace period — its connections +are force-closed when the *next* config change arrives. +Adding `forceKill=true` to your DSN will cause the hotload driver to cancel in-flight +operations and close the underlying connections as soon as a change to the connection +information is detected. Operations canceled this way fail with an error matching +`hotload.ErrHotSwap` (via `errors.Is`). For example: ``` db, err := sql.Open("hotload", "fsnotify://postgres/tmp/myconfig.txt?forceKill=true") ``` -# How To Run Integration Tests Locally +With `forceKill=true`, hotload waits a bounded window (default 100ms) for in-flight +operations to observe the cancellation before force-closing their connections, so a +driver that ignores context cancellation can never wedge config-change processing. +Tune it with the `killWindow` DSN parameter, e.g. `?forceKill=true&killWindow=250ms`. + +# Connection lifecycle + +`sql.Open("hotload", dsn)` starts the strategy watch for that DSN; multiple `sql.Open` +calls with the same DSN share one watch. Closing the last `sql.DB` for a DSN stops the +watch and its background goroutine. Note that this means configuration errors (unknown +strategy or driver, unreadable config file) surface at `sql.Open` instead of at first +use. + +# Metrics + +Hotload's core has no metrics dependency. It emits events through a small hooks API +(`hotload.RegisterHooks`); the +[`observability`](https://pkg.go.dev/github.com/infobloxopen/hotload/observability) +module adapts those events to prometheus, preserving the metric names of hotload v1: + +```go +import "github.com/infobloxopen/hotload/observability" + +func main() { + // nil registers with prometheus.DefaultRegisterer + observability.MustEnablePrometheus(nil) + ... +} +``` + +| Metric | Description | +|---|---| +| `transaction_sql_stmts` | sql statements per transaction, labeled by the exec labels carried via `hotload.ContextWithExecLabels` | +| `hotload_change_total` | config changes per hotload DSN | +| `hotload_last_changed_timestamp_seconds` | unix timestamp of the last change per DSN | +| `hotload_modtime_latency_histogram` | staleness of watched files (fed by the `modtime` package) | +| `hotload_path_chksum_timestamp_seconds` | when each watched file's checksum last changed (opt-in via `HOTLOAD_PATH_CHKSUM_METRICS_ENABLE`) | + +# How To Run Tests Locally + +Unit tests (no database needed): +``` +$ make test +``` + +Integration tests (real postgres via docker compose): ``` $ make postgres-docker-compose-up -$ cd integrationstests -$ go test -v -race -timeout=3m -$ vi ... -$ go test -v -race # this can be repeated in your edit-run-test cycle +$ make local-integration-tests $ make postgres-docker-compose-down ``` +The integration tests honor `HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST` and +`HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT` if your postgres lives elsewhere, and skip +themselves when no server is reachable. + +# Repository layout + +This repository holds three Go modules tied together by the committed `go.work`: + +| Module | Purpose | Dependencies | +|---|---|---| +| `github.com/infobloxopen/hotload/v3` | the driver, `fsnotify` strategy, `logger`, `modtime` | `fsnotify` only | +| `github.com/infobloxopen/hotload/observability` | prometheus adapter + test helpers | prometheus | +| `github.com/infobloxopen/hotload/test/integration` | postgres integration tests (never imported) | `lib/pq` | + +The combination wrapper types are generated; after changing `internal/gen`, +`conn_pieces.go` or `stmt_pieces.go`, run `make generate` and commit the result +(CI fails on a diff). diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..af3dd91 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,50 @@ +# Releasing + +This repository contains two released modules with independent tag +namespaces, plus an internal test module that is never released: + +| Module | Tag format | Example | +|---|---|---| +| `github.com/infobloxopen/hotload/v3` (repo root) | `vX.Y.Z` | `v3.0.0` | +| `github.com/infobloxopen/hotload/observability` | `observability/vX.Y.Z` | `observability/v1.0.0` | +| `github.com/infobloxopen/hotload/test/integration` | never tagged | — | + +## Order matters + +`observability/go.mod` requires `github.com/infobloxopen/hotload/v3` by +version. The `replace` directive in that file only affects building inside +this repository — **consumers resolve the `require` line**, so it must point +at a tag that exists. + +For a release that touches both modules: + +1. Tag the root module first: + ```sh + git tag v3.Y.Z && git push origin v3.Y.Z + ``` +2. Bump the require in `observability/go.mod` to the new tag: + ```sh + cd observability && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z + ``` + Commit, then tag: + ```sh + git tag observability/vA.B.C && git push origin observability/vA.B.C + ``` + +3. Sanity-check as a consumer (from any directory outside this repo): + ```sh + cd $(mktemp -d) && go mod init smoke + GOFLAGS=-mod=mod go get github.com/infobloxopen/hotload/v3@v3.Y.Z \ + github.com/infobloxopen/hotload/observability@vA.B.C + ``` + This catches the failure mode of a require pointing at a nonexistent + version (a `replace` masks it inside the repo). + +## Notes + +- The root module path carries the `/v3` suffix; tags below `v3.0.0` on the + root module are invalid for it. +- Do not tag `observability` with a version of the root module's tag series; + the namespaces are independent and need not be aligned. +- `test/integration` has a permanent `replace` and a nominal require + version; it is intentionally excluded from releases. diff --git a/internal/depbudget/main.go b/internal/depbudget/main.go new file mode 100644 index 0000000..8472df6 --- /dev/null +++ b/internal/depbudget/main.go @@ -0,0 +1,31 @@ +// Command depbudget reads `go mod edit -json` output on stdin and prints +// the module's direct (non-indirect) requirements, one per line. The +// Makefile's dep-budget target uses it to assert that the hotload core's +// only direct dependency is fsnotify. +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" +) + +type goMod struct { + Require []struct { + Path string + Indirect bool + } +} + +func main() { + var mod goMod + if err := json.NewDecoder(os.Stdin).Decode(&mod); err != nil { + log.Fatalf("depbudget: decoding go mod json: %v", err) + } + for _, req := range mod.Require { + if !req.Indirect { + fmt.Println(req.Path) + } + } +} diff --git a/observability/prometheus_test.go b/observability/prometheus_test.go index 43928c0..5d1f8bf 100644 --- a/observability/prometheus_test.go +++ b/observability/prometheus_test.go @@ -9,8 +9,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" - hotload "github.com/infobloxopen/hotload/v3" "github.com/infobloxopen/hotload/observability/promtest" + hotload "github.com/infobloxopen/hotload/v3" ) func TestConfigChangeMetrics(t *testing.T) { diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 7d224e1..37c446f 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -63,10 +63,28 @@ func userDsn(pass string) string { testDbUser, pass, postgresHost, postgresPort) } -// requirePostgres skips the test when no server is reachable and lazily -// provisions the test user and table. +// enabled reports whether the integration tests were explicitly requested: +// either HOTLOAD_INTEGRATION_TESTS is truthy (set by `make +// local-integration-tests`) or a postgres host/port was pointed at via the +// environment. An implicit "is something listening on 5432" probe is not +// enough — an unrelated local postgres would fail authentication instead of +// skipping. +func enabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TESTS"))) { + case "1", "true", "yes": + return true + } + return os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST") != "" || + os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT") != "" +} + +// requirePostgres skips the test unless integration testing was requested +// and the server is reachable; it lazily provisions the test user and table. func requirePostgres(t *testing.T) { t.Helper() + if !enabled() { + t.Skip("skipping: integration tests not requested (run via `make local-integration-tests`, or set HOTLOAD_INTEGRATION_TESTS=1)") + } addr := net.JoinHostPort(postgresHost, postgresPort) conn, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { From 5d623240f8d220b72965781723bf20a4d715ca25 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 00:31:13 -0500 Subject: [PATCH 06/14] Fix reopen-while-closing watch race; review cleanups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing the last sql.DB for a DSN released hdriver.mu before the group's strategy watch was torn down, so a concurrent sql.Open of the same DSN could create a new group that the strategy fed through the old group's about-to-be-closed update channel — the reopened handle then silently stopped receiving config changes. Strategy watch closure now happens inside the same hdriver.mu critical section that creates watches, serializing the two; the rest of the teardown (parent context cancel, hook emission) still runs outside all locks. TestReopenWhileClosing reproduces the race and fails on the previous ordering. Cleanups from review: drop the never-read baseConn.killed field, reuse the generation's cached redacted DSN when driver options don't modify the dial string (RedactUrl costs a mutex plus crypto/rand per call), and deduplicate the modtime test poll helper into internal/testutil. --- conn.go | 8 +++---- driver.go | 10 +++++++-- group.go | 28 ++++++++++++++++------- lifecycle_test.go | 50 +++++++++++++++++++++++++++++++++++++++++ modtime/modtime_test.go | 9 ++------ 5 files changed, 83 insertions(+), 22 deletions(-) diff --git a/conn.go b/conn.go index b9350bf..e52438b 100644 --- a/conn.go +++ b/conn.go @@ -25,7 +25,6 @@ type baseConn struct { dsn string redactDsn string closed atomic.Bool - killed atomic.Bool execStmts atomic.Int64 // exec statements since the last completed transaction queryStmts atomic.Int64 // query statements since the last completed transaction } @@ -163,14 +162,13 @@ func (c *baseConn) Close() error { } // closeConn closes the underlying conn exactly once, no matter how many -// paths race to close it (the pool, a generation kill, or both). +// paths race to close it (the pool, a generation kill, or both). killed +// records in the emitted hook event whether the close was caused by a +// config change rather than the pool retiring the conn. func (c *baseConn) closeConn(killed bool) error { if !c.closed.CompareAndSwap(false, true) { return nil } - if killed { - c.killed.Store(true) - } err := c.inner.Close() c.gen.remove(c) emitConnClose(ConnEvent{GroupName: c.group.name, RedactedDSN: c.redactDsn, Killed: killed}) diff --git a/driver.go b/driver.go index 36af2e8..264dd33 100644 --- a/driver.go +++ b/driver.go @@ -296,7 +296,12 @@ func (h *hdriver) getGroup(name string, pin bool) (*group, error) { } // releaseGroup drops one connector reference and shuts the group down when -// no references remain (unless a legacy Open pinned it). +// no references remain (unless a legacy Open pinned it). The strategy watch +// is closed while h.mu is still held: getGroup establishes watches under +// the same lock, so a dying group's watch is fully closed before a new +// group with the same DSN can watch the same path — otherwise the strategy +// could hand the new group the doomed update channel and the new group +// would silently stop receiving config changes. func (h *hdriver) releaseGroup(name string) { h.mu.Lock() g, ok := h.groups[name] @@ -306,12 +311,13 @@ func (h *hdriver) releaseGroup(name string) { g = nil } else { delete(h.groups, name) + g.closeWatch() } } h.mu.Unlock() if g != nil { - g.shutdown() + g.finishShutdown() } } diff --git a/group.go b/group.go index e61dbdc..c1acd11 100644 --- a/group.go +++ b/group.go @@ -227,6 +227,11 @@ func (g *group) conn(ctx context.Context) (driver.Conn, error) { if err != nil { return nil, err } + redactDsn := gen.redactDsn + if dsn != gen.dsn { + // Driver options changed the DSN; redact the actual dial string. + redactDsn = internal.RedactUrl(dsn) + } inner, err := g.dial(ctx, gen, dsn) if err != nil { return nil, err @@ -237,7 +242,7 @@ func (g *group) conn(ctx context.Context) (driver.Conn, error) { gen: gen, group: g, dsn: dsn, - redactDsn: internal.RedactUrl(dsn), + redactDsn: redactDsn, } gen.add(c) emitConnOpen(ConnEvent{GroupName: g.name, RedactedDSN: c.redactDsn}) @@ -263,11 +268,12 @@ func (g *group) dial(ctx context.Context, gen *generation, dsn string) (driver.C return gen.connector.Connect(ctx) } -// shutdown tears the group down: the strategy watch is closed and the -// parent context canceled, which terminates runLoop and cancels every -// generation context. Called once the last connector referencing the group -// closes. -func (g *group) shutdown() { +// closeWatch marks the group closed and stops its strategy watch. It is +// called by hdriver.releaseGroup with hdriver.mu held, which serializes it +// against getGroup's strategy.Watch calls (see releaseGroup). Strategy +// implementations must therefore never call back into hotload from +// CloseWatch. +func (g *group) closeWatch() { g.mu.Lock() if g.closed { g.mu.Unlock() @@ -277,11 +283,17 @@ func (g *group) shutdown() { g.mu.Unlock() if err := g.strategy.CloseWatch(g.path, g.pathQry); err != nil { - g.logf("group.shutdown", "CloseWatch error: %v", err) + g.logf("group.closeWatch", "CloseWatch error: %v", err) } +} + +// finishShutdown completes the teardown started by closeWatch: the parent +// context cancel terminates runLoop and cancels every generation context. +// Runs outside all locks so hook callbacks cannot deadlock. +func (g *group) finishShutdown() { g.parentCancel() EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) - g.logf("group.shutdown", "group closed") + g.logf("group.finishShutdown", "group closed") } func mergeConnStringOptions(dsn string, options map[string]string) (string, error) { diff --git a/lifecycle_test.go b/lifecycle_test.go index 93dec17..f1a1bc3 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "errors" "fmt" + "sync" "testing" "time" @@ -300,6 +301,55 @@ func TestGroupTeardownAndSharing(t *testing.T) { }) } +// TestReopenWhileClosing races the close of a DSN's last sql.DB against a +// fresh sql.Open of the same DSN. Whatever the interleaving, the surviving +// handle must keep receiving config changes: the dying group's watch +// teardown is serialized with new watch creation, so a new group can never +// be handed a doomed update channel. +func TestReopenWhileClosing(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{noDB: true}) + + db, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 20; i++ { + var ( + wg sync.WaitGroup + db2 *sql.DB + ) + wg.Add(2) + go func() { + defer wg.Done() + db.Close() + }() + go func() { + defer wg.Done() + var err error + db2, err = sql.Open("hotload", fx.dsn) + if err != nil { + t.Errorf("reopen: %v", err) + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // The surviving handle must observe a config change. + want := fmt.Sprintf("dsn-rw-%d", i) + fx.push(want) + testutil.WaitFor(t, 5*time.Second, "change to propagate to the reopened db", func() bool { + var got string + return db2.QueryRow("SELECT dsn").Scan(&got) == nil && got == want + }) + db = db2 + } + db.Close() +} + // TestStrategyChannelClose: when the strategy closes its update channel the // run loop exits, existing connections keep serving the last value, and a // later db.Close still tears down cleanly. diff --git a/modtime/modtime_test.go b/modtime/modtime_test.go index 77600aa..21d9273 100644 --- a/modtime/modtime_test.go +++ b/modtime/modtime_test.go @@ -10,6 +10,7 @@ import ( hotload "github.com/infobloxopen/hotload/v3" "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/internal/testutil" ) const testStrategy = "fsnotify" @@ -56,13 +57,7 @@ func (r *eventRecorder) forPath(path string) []hotload.ModTimeEvent { func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { t.Helper() - deadline := time.Now().Add(timeout) - for !cond() { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for %s", what) - } - time.Sleep(5 * time.Millisecond) - } + testutil.WaitFor(t, timeout, what, cond) } // TestAgainstUnixFS verifies the monitor works against the real filesystem. From cccf56274fe9a3d938cfd334ff51addd91187a1d Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 07:53:24 -0500 Subject: [PATCH 07/14] CI: test on Go 1.25 and 1.26 The go.mod directives stay at 1.23 (minimum supported version); the CI matrix tracks the two most recent Go releases. --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1600965..d9eff99 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,7 +13,7 @@ jobs: matrix: # database/sql's bad-conn retry behavior shifts subtly between Go # releases; test the two most recent. - go-version: [ '1.23.x', '1.24.x' ] + go-version: [ '1.25.x', '1.26.x' ] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.24.x' + go-version: '1.26.x' - name: Start postgres run: make postgres-docker-compose-up - name: Integration tests From d4de248c5d3bf0975b03a87a742216821b92b7b5 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 23:34:04 -0500 Subject: [PATCH 08/14] Add k8ssecret strategy module for cross-namespace Secret watching Port the k8ssecret strategy idea from PR #66 onto v3: a nested module (github.com/infobloxopen/hotload/k8ssecret, keeping client-go out of the core) implementing hotload.Strategy by watching a Kubernetes Secret through the API server, for deployments that need credentials from a Secret in another namespace, which Kubernetes cannot mount as a volume. DSN format matches the PR: k8ssecret:///?namespace=&dsn=, with the namespace defaulting to the pod's own and the key to dsn.txt. Beyond the port, this fixes four defects in the PR #66 implementation: - The hotload core passes uri.Path, so the secret name arrives with a leading slash; the PR used it verbatim and would look up a secret named "/myapp-db" (its tests called Watch directly with bare names, masking this). Covered here by an end-to-end test through sql.Open with a minimal fake driver. - Updates could be lost in the gap between the initial Get and the watch start, and while a dropped watch waited to reconnect (only Modified events were handled, watching from "now"). Every (re)connect now re-reads the Secret, delivers missed changes, and watches from that read's resource version; Added events (recreated secrets, post-reconnect snapshots) are handled too. - Slow subscribers had the newest value dropped when their buffer was full, leaving them permanently stale; delivery now drops the oldest queued value so subscribers converge on the latest. - Watches were keyed by namespace/name only, so two DSNs reading different data keys of the same Secret shared one value; the data key is now part of the watch identity. Tests use client-go's fake clientset: initial fetch, missing secret/key, defaults, update propagation, delete/recreate, reconnect catch-up (via a watch reactor whose first watcher is killed), slow subscriber convergence, independent keys, CloseWatch/Close semantics, and the sql.Open end-to-end path. The module joins go.work, the Makefile module loop (and therefore CI), and the release ordering docs. --- Makefile | 11 +- README.md | 24 ++- RELEASING.md | 22 ++- go.work | 1 + go.work.sum | 28 ++- k8ssecret/e2e_test.go | 151 +++++++++++++++ k8ssecret/go.mod | 52 ++++++ k8ssecret/go.sum | 154 +++++++++++++++ k8ssecret/strategy.go | 370 +++++++++++++++++++++++++++++++++++++ k8ssecret/strategy_test.go | 368 ++++++++++++++++++++++++++++++++++++ 10 files changed, 1154 insertions(+), 27 deletions(-) create mode 100644 k8ssecret/e2e_test.go create mode 100644 k8ssecret/go.mod create mode 100644 k8ssecret/go.sum create mode 100644 k8ssecret/strategy.go create mode 100644 k8ssecret/strategy_test.go diff --git a/Makefile b/Makefile index a83cf5f..2ed9c7f 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -# The repository holds three Go modules: the hotload core (.), the -# prometheus adapter (observability), and the postgres integration tests -# (test/integration). The committed go.work ties them together for -# development; most targets loop over all of them. -MODULES := . observability test/integration +# The repository holds four Go modules: the hotload core (.), the +# prometheus adapter (observability), the Kubernetes Secret strategy +# (k8ssecret), and the postgres integration tests (test/integration). The +# committed go.work ties them together for development; most targets loop +# over all of them. +MODULES := . k8ssecret observability test/integration .PHONY: fmt vet tidy build test generate no-diff dep-budget ci-test \ postgres-docker-compose-up postgres-docker-compose-down local-integration-tests diff --git a/README.md b/README.md index 19c14f3..39d9fbc 100644 --- a/README.md +++ b/README.md @@ -99,10 +99,27 @@ string's encoded query parameters. The strategy doesn't have to use a real file the config. `pth` represents a unique string that makes sense to the strategy. For example, pth could point to a path in etcd or a kind/id in k8s. -The hotload project ships with one hotload strategy: `fsnotify`. +The hotload project ships with two strategies: -Note: In your project, if you do not implement your own `Strategy`, and instead choose to use the out-of-the-box -`fsnotify` strategy, you must import the `fsnotify` package in your project to register at least one strategy with +* `fsnotify` (in the core module) watches a file on disk — the right choice when the + connection string is mounted into the pod (ConfigMap/Secret volumes included). +* `k8ssecret` (module [`github.com/infobloxopen/hotload/k8ssecret`](k8ssecret/)) watches a + Kubernetes Secret through the API server, for deployments that need credentials from a + Secret in **another namespace**, which Kubernetes cannot mount as a volume: + + ```go + import _ "github.com/infobloxopen/hotload/k8ssecret" + + db, err := sql.Open("hotload", "k8ssecret://pgx/myapp-db?namespace=prod&dsn=dsn.txt") + ``` + + The path component is the Secret name; `namespace` defaults to the pod's own namespace + and `dsn` (the data key holding the connection string) defaults to `dsn.txt`. The pod's + service account needs `get` and `watch` on the Secret. It lives in its own module so + `client-go` stays out of the hotload core. + +Note: In your project, if you do not implement your own `Strategy`, and instead choose to use an out-of-the-box +strategy, you must import its package in your project to register at least one strategy with hotload, otherwise an error will occur at runtime as the `database/sql` package will not be able to locate/load your intended hotload strategy as a recognizable driver. @@ -185,6 +202,7 @@ This repository holds three Go modules tied together by the committed `go.work`: | Module | Purpose | Dependencies | |---|---|---| | `github.com/infobloxopen/hotload/v3` | the driver, `fsnotify` strategy, `logger`, `modtime` | `fsnotify` only | +| `github.com/infobloxopen/hotload/k8ssecret` | Kubernetes Secret strategy (cross-namespace) | `client-go` | | `github.com/infobloxopen/hotload/observability` | prometheus adapter + test helpers | prometheus | | `github.com/infobloxopen/hotload/test/integration` | postgres integration tests (never imported) | `lib/pq` | diff --git a/RELEASING.md b/RELEASING.md index af3dd91..8271e3b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -6,36 +6,40 @@ namespaces, plus an internal test module that is never released: | Module | Tag format | Example | |---|---|---| | `github.com/infobloxopen/hotload/v3` (repo root) | `vX.Y.Z` | `v3.0.0` | +| `github.com/infobloxopen/hotload/k8ssecret` | `k8ssecret/vX.Y.Z` | `k8ssecret/v1.0.0` | | `github.com/infobloxopen/hotload/observability` | `observability/vX.Y.Z` | `observability/v1.0.0` | | `github.com/infobloxopen/hotload/test/integration` | never tagged | — | ## Order matters -`observability/go.mod` requires `github.com/infobloxopen/hotload/v3` by -version. The `replace` directive in that file only affects building inside -this repository — **consumers resolve the `require` line**, so it must point -at a tag that exists. +`observability/go.mod` and `k8ssecret/go.mod` require +`github.com/infobloxopen/hotload/v3` by version. The `replace` directives in +those files only affect building inside this repository — **consumers +resolve the `require` line**, so it must point at a tag that exists. -For a release that touches both modules: +For a release that touches the root and the satellite modules: 1. Tag the root module first: ```sh git tag v3.Y.Z && git push origin v3.Y.Z ``` -2. Bump the require in `observability/go.mod` to the new tag: +2. Bump the require in each satellite module to the new tag: ```sh - cd observability && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z + (cd observability && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z) + (cd k8ssecret && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z) ``` - Commit, then tag: + Commit, then tag each satellite: ```sh git tag observability/vA.B.C && git push origin observability/vA.B.C + git tag k8ssecret/vA.B.C && git push origin k8ssecret/vA.B.C ``` 3. Sanity-check as a consumer (from any directory outside this repo): ```sh cd $(mktemp -d) && go mod init smoke GOFLAGS=-mod=mod go get github.com/infobloxopen/hotload/v3@v3.Y.Z \ - github.com/infobloxopen/hotload/observability@vA.B.C + github.com/infobloxopen/hotload/observability@vA.B.C \ + github.com/infobloxopen/hotload/k8ssecret@vA.B.C ``` This catches the failure mode of a require pointing at a nonexistent version (a `replace` masks it inside the repo). diff --git a/go.work b/go.work index 90b372f..512edb2 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,7 @@ go 1.23.0 use ( . + ./k8ssecret ./observability ./test/integration ) diff --git a/go.work.sum b/go.work.sum index 40f47aa..8fca17f 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,25 +1,33 @@ +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= diff --git a/k8ssecret/e2e_test.go b/k8ssecret/e2e_test.go new file mode 100644 index 0000000..afe5692 --- /dev/null +++ b/k8ssecret/e2e_test.go @@ -0,0 +1,151 @@ +package k8ssecret + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "sync" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// TestEndToEndThroughSQLOpen drives the strategy through the real hotload +// core and database/sql: registration via this package's init, DSN parsing +// (including the leading slash the core puts on the path component), watch +// establishment at sql.Open, and update propagation into the pool. +func TestEndToEndThroughSQLOpen(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + + // Swap the package-registered strategy instance for one bound to the + // fake clientset; restore a pristine instance afterwards. + hotload.UnregisterStrategy("k8ssecret") + s := NewStrategyWithClientset(cs) + s.backoff = 20 * time.Millisecond + hotload.RegisterStrategy("k8ssecret", s) + t.Cleanup(func() { + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategy()) + }) + + registerFakeDriverOnce() + + db, err := sql.Open("hotload", "k8ssecret://k8sfake/mydb?namespace=prod&dsn=dsn") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + defer db.Close() + + queryDSN := func() (string, error) { + var dsn string + err := db.QueryRow("SELECT dsn").Scan(&dsn) + return dsn, err + } + + got, err := queryDSN() + if err != nil { + t.Fatal(err) + } + if got != "dsn-1" { + t.Fatalf("initial query routed to %q, want dsn-1", got) + } + + // Rotate the secret; the pool must converge on the new DSN. + if _, err := cs.CoreV1().Secrets("prod").Update(context.Background(), + makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(5 * time.Second) + for { + got, err := queryDSN() + if err == nil && got == "dsn-2" { + return + } + if time.Now().After(deadline) { + t.Fatalf("pool did not converge on dsn-2 (last: %q, err: %v)", got, err) + } + time.Sleep(10 * time.Millisecond) + } +} + +// TestSQLOpenErrorsOnMissingSecret: configuration problems surface at +// sql.Open (the hotload core starts the watch there). +func TestSQLOpenErrorsOnMissingSecret(t *testing.T) { + cs := fakeClientset(t) + + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategyWithClientset(cs)) + t.Cleanup(func() { + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategy()) + }) + + registerFakeDriverOnce() + + if _, err := sql.Open("hotload", "k8ssecret://k8sfake/nosuchsecret?namespace=prod"); err == nil { + t.Fatal("expected sql.Open to fail for a missing secret") + } +} + +// A minimal driver whose conns answer "SELECT dsn" with the DSN they were +// opened with, so tests can observe which generation served a query. (The +// hotload core's richer fake lives in its internal packages, which this +// module cannot import.) +var registerFakeDriver sync.Once + +func registerFakeDriverOnce() { + registerFakeDriver.Do(func() { + hotload.RegisterSQLDriver("k8sfake", fakeDriver{}) + }) +} + +type fakeDriver struct{} + +func (fakeDriver) Open(dsn string) (driver.Conn, error) { return &fakeConn{dsn: dsn}, nil } + +type fakeConn struct{ dsn string } + +func (c *fakeConn) Prepare(query string) (driver.Stmt, error) { return &fakeStmt{conn: c}, nil } +func (c *fakeConn) Close() error { return nil } +func (c *fakeConn) Begin() (driver.Tx, error) { return fakeTx{}, nil } + +func (c *fakeConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return &fakeRows{dsn: c.dsn}, nil +} + +type fakeStmt struct{ conn *fakeConn } + +func (s *fakeStmt) Close() error { return nil } +func (s *fakeStmt) NumInput() int { return -1 } +func (s *fakeStmt) Exec(args []driver.Value) (driver.Result, error) { + return driver.RowsAffected(1), nil +} +func (s *fakeStmt) Query(args []driver.Value) (driver.Rows, error) { + return &fakeRows{dsn: s.conn.dsn}, nil +} + +type fakeTx struct{} + +func (fakeTx) Commit() error { return nil } +func (fakeTx) Rollback() error { return nil } + +type fakeRows struct { + dsn string + done bool +} + +func (r *fakeRows) Columns() []string { return []string{"dsn"} } +func (r *fakeRows) Close() error { return nil } +func (r *fakeRows) Next(dest []driver.Value) error { + if r.done { + return io.EOF + } + dest[0] = r.dsn + r.done = true + return nil +} diff --git a/k8ssecret/go.mod b/k8ssecret/go.mod new file mode 100644 index 0000000..fc8de02 --- /dev/null +++ b/k8ssecret/go.mod @@ -0,0 +1,52 @@ +module github.com/infobloxopen/hotload/k8ssecret + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + k8s.io/api v0.32.3 + k8s.io/apimachinery v0.32.3 + k8s.io/client-go v0.32.3 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace github.com/infobloxopen/hotload/v3 => ../ diff --git a/k8ssecret/go.sum b/k8ssecret/go.sum new file mode 100644 index 0000000..da2dc2f --- /dev/null +++ b/k8ssecret/go.sum @@ -0,0 +1,154 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= +k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/k8ssecret/strategy.go b/k8ssecret/strategy.go new file mode 100644 index 0000000..ae6921c --- /dev/null +++ b/k8ssecret/strategy.go @@ -0,0 +1,370 @@ +// Package k8ssecret implements a hotload strategy that watches Kubernetes +// Secrets for database connection string changes. +// +// In multi-namespace deployments a service sometimes needs credentials from +// a Secret in another team's namespace. Kubernetes does not allow mounting +// Secrets across namespaces, so the volume-based fsnotify strategy cannot +// see them; this strategy watches the Secret through the Kubernetes API +// instead. +// +// This package is a separate Go module +// (github.com/infobloxopen/hotload/k8ssecret) so client-go and its +// transitive dependencies stay out of the hotload core. +// +// # DSN format +// +// k8ssecret:///?namespace=&dsn= +// +// Parameters: +// - secret-name: the Kubernetes Secret name (the path component) +// - namespace: the namespace containing the Secret (default: the pod's +// namespace from the service account mount, else "default") +// - dsn: the data key within the Secret holding the connection string +// (default: "dsn.txt") +// +// The hotload parameters (forceKill, killWindow) may appear in the same +// query string. +// +// # Usage +// +// Import the package for its side effect of registering the strategy: +// +// import _ "github.com/infobloxopen/hotload/k8ssecret" +// +// db, err := sql.Open("hotload", "k8ssecret://pgx/myapp-db?namespace=prod&dsn=dsn.txt") +// +// The pod's service account needs get and watch permissions on the Secret. +package k8ssecret + +import ( + "context" + "fmt" + "net/url" + "os" + "path" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/logger" +) + +func init() { + hotload.RegisterStrategy("k8ssecret", NewStrategy()) +} + +const ( + defaultKey = "dsn.txt" + defaultBackoff = 2 * time.Second +) + +// ClientsetFunc constructs the Kubernetes clientset used by strategies that +// were not given one explicitly (including the instance registered by this +// package's init). It is consulted lazily on the first Watch; replace it +// before opening connections to use out-of-cluster config or fakes. +var ClientsetFunc = defaultClientset + +func defaultClientset() (kubernetes.Interface, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("k8ssecret: in-cluster config: %w", err) + } + return kubernetes.NewForConfig(cfg) +} + +// Strategy implements hotload.Strategy by watching Kubernetes Secrets. +type Strategy struct { + mu sync.Mutex + clientset kubernetes.Interface + watches map[watchKey]*secretWatch + backoff time.Duration +} + +// watchKey identifies one watched value. The data key is part of the +// identity: two DSNs reading different keys of the same Secret carry +// different values and must not share state. +type watchKey struct { + namespace string + name string + key string +} + +// secretWatch fans one watched Secret value out to its subscribers (one per +// distinct pathQry). Subscriber channels have capacity 1 and are written +// with drop-oldest semantics, so a subscriber always converges on the +// latest value and a slow subscriber can never block delivery. All channel +// sends and closes happen under Strategy.mu, so they cannot race. +type secretWatch struct { + cancel context.CancelFunc + value string + queries map[string]chan string +} + +// NewStrategy creates a strategy that builds its clientset lazily from +// ClientsetFunc on first use. +func NewStrategy() *Strategy { + return &Strategy{ + watches: make(map[watchKey]*secretWatch), + backoff: defaultBackoff, + } +} + +// NewStrategyWithClientset creates a strategy using the given clientset; +// useful for tests (client-go's fake clientset) and out-of-cluster use. +func NewStrategyWithClientset(cs kubernetes.Interface) *Strategy { + s := NewStrategy() + s.clientset = cs + return s +} + +// secretName extracts the Secret name from the path component the hotload +// core passes to Watch. For "k8ssecret://pgx/myapp-db" that component is +// "/myapp-db" — with a leading slash that is not part of the name. +func secretName(pth string) string { + return strings.TrimPrefix(path.Clean(strings.TrimSpace(pth)), "/") +} + +// parseParams extracts the namespace and data key from the encoded query +// parameters of the hotload DSN. +func parseParams(pathQry string) (namespace, key string, err error) { + params, err := url.ParseQuery(strings.TrimSpace(pathQry)) + if err != nil { + return "", "", fmt.Errorf("k8ssecret: parse query %q: %w", pathQry, err) + } + namespace = params.Get("namespace") + if namespace == "" { + namespace = podNamespace() + } + key = params.Get("dsn") + if key == "" { + key = defaultKey + } + return namespace, key, nil +} + +// Watch implements hotload.Strategy. pth is the Secret name; pathQry +// carries the namespace and dsn parameters (see the package documentation). +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, <-chan string, error) { + name := secretName(pth) + pathQry = strings.TrimSpace(pathQry) + namespace, key, err := parseParams(pathQry) + if err != nil { + return "", nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.clientset == nil { + cs, err := ClientsetFunc() + if err != nil { + return "", nil, err + } + s.clientset = cs + } + if s.watches == nil { + // Re-initialize after Close. + s.watches = make(map[watchKey]*secretWatch) + } + + wk := watchKey{namespace: namespace, name: name, key: key} + sw, exists := s.watches[wk] + if !exists { + secret, err := s.clientset.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", nil, fmt.Errorf("k8ssecret: get secret %s/%s: %w", namespace, name, err) + } + val, ok := secret.Data[key] + if !ok { + return "", nil, fmt.Errorf("k8ssecret: secret %s/%s has no key %q", namespace, name, key) + } + + watchCtx, cancel := context.WithCancel(context.Background()) + sw = &secretWatch{ + cancel: cancel, + value: string(val), + queries: make(map[string]chan string), + } + s.watches[wk] = sw + go s.runWatch(watchCtx, wk) + } + + ch, exists := sw.queries[pathQry] + if !exists { + ch = make(chan string, 1) + sw.queries[pathQry] = ch + } + return sw.value, ch, nil +} + +// CloseWatch implements hotload.Strategy. When the last subscriber of a +// Secret value is closed, the API watch is stopped. +func (s *Strategy) CloseWatch(pth string, pathQry string) error { + name := secretName(pth) + pathQry = strings.TrimSpace(pathQry) + namespace, key, err := parseParams(pathQry) + if err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + wk := watchKey{namespace: namespace, name: name, key: key} + sw, ok := s.watches[wk] + if !ok { + return nil + } + if ch, ok := sw.queries[pathQry]; ok { + close(ch) + delete(sw.queries, pathQry) + } + if len(sw.queries) == 0 { + sw.cancel() + delete(s.watches, wk) + } + return nil +} + +// Close implements hotload.Strategy: stops every watch and closes every +// update channel. +func (s *Strategy) Close() { + s.mu.Lock() + defer s.mu.Unlock() + for wk, sw := range s.watches { + sw.cancel() + for q, ch := range sw.queries { + close(ch) + delete(sw.queries, q) + } + delete(s.watches, wk) + } + s.watches = nil +} + +// runWatch maintains the API watch for one Secret value until its context +// is canceled. Every (re)connect first re-reads the Secret and delivers any +// value missed while disconnected, then watches from that read's resource +// version — so no modification is lost between the read and the watch, or +// while a dropped watch was reconnecting. +func (s *Strategy) runWatch(ctx context.Context, wk watchKey) { + for { + rv, err := s.catchUp(ctx, wk) + if err == nil { + err = s.consumeWatch(ctx, wk, rv) + } + if ctx.Err() != nil { + return + } + logger.ErrLogf("k8ssecret.runWatch:", "watch %s/%s interrupted: %v, reconnecting in %s", + wk.namespace, wk.name, err, s.backoff) + select { + case <-ctx.Done(): + return + case <-time.After(s.backoff): + } + } +} + +// catchUp reads the Secret's current value, delivers it if it changed, and +// returns the resource version to start the watch from. +func (s *Strategy) catchUp(ctx context.Context, wk watchKey) (string, error) { + secret, err := s.clientset.CoreV1().Secrets(wk.namespace).Get(ctx, wk.name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("get: %w", err) + } + if val, ok := secret.Data[wk.key]; ok { + s.deliver(wk, string(val)) + } + return secret.ResourceVersion, nil +} + +// consumeWatch processes watch events until the watch drops or the context +// is canceled. +func (s *Strategy) consumeWatch(ctx context.Context, wk watchKey, rv string) error { + watcher, err := s.clientset.CoreV1().Secrets(wk.namespace).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + wk.name, + ResourceVersion: rv, + }) + if err != nil { + return fmt.Errorf("start watch: %w", err) + } + defer watcher.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-watcher.ResultChan(): + if !ok { + return fmt.Errorf("watch channel closed") + } + switch event.Type { + case watch.Added, watch.Modified: + // Added matters: a Secret deleted and recreated comes back + // as Added, and so does the first event after a reconnect. + case watch.Error: + return fmt.Errorf("watch error event: %v", event.Object) + default: + // Deleted: keep serving the last known value, exactly like + // the fsnotify strategy when the watched file disappears; + // the recreate arrives as Added. + continue + } + secret, ok := event.Object.(*corev1.Secret) + if !ok || secret.Name != wk.name { + // The fake clientset (and old API servers) can ignore field + // selectors, so filter by name here too. + continue + } + if val, ok := secret.Data[wk.key]; ok { + s.deliver(wk, string(val)) + } + } + } +} + +// deliver pushes a changed value to every subscriber of the watch. +// Channels have capacity 1; when full, the stale queued value is dropped so +// the subscriber always converges on the latest one (dropping the new value +// instead would leave a slow subscriber permanently stale). +func (s *Strategy) deliver(wk watchKey, val string) { + s.mu.Lock() + defer s.mu.Unlock() + sw, ok := s.watches[wk] + if !ok || sw.value == val { + return + } + sw.value = val + for _, ch := range sw.queries { + select { + case ch <- val: + continue + default: + } + select { + case <-ch: // drop the stale queued value + default: + } + select { + case ch <- val: + default: + } + } +} + +// podNamespace returns the namespace of the current pod from the service +// account mount, or "default" when not running in-cluster. +func podNamespace() string { + if ns, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + return strings.TrimSpace(string(ns)) + } + return "default" +} diff --git a/k8ssecret/strategy_test.go b/k8ssecret/strategy_test.go new file mode 100644 index 0000000..ecfcd31 --- /dev/null +++ b/k8ssecret/strategy_test.go @@ -0,0 +1,368 @@ +package k8ssecret + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func makeSecret(namespace, name string, data map[string]string) *corev1.Secret { + bs := make(map[string][]byte, len(data)) + for k, v := range data { + bs[k] = []byte(v) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: bs, + } +} + +func fakeClientset(t *testing.T, secrets ...*corev1.Secret) kubernetes.Interface { + t.Helper() + cs := fake.NewClientset() + for _, sec := range secrets { + if _, err := cs.CoreV1().Secrets(sec.Namespace).Create(context.Background(), sec, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + return cs +} + +// newTestStrategy returns a strategy on a fake clientset with a short +// reconnect backoff, closed when the test ends. +func newTestStrategy(t *testing.T, cs kubernetes.Interface) *Strategy { + t.Helper() + s := NewStrategyWithClientset(cs) + s.backoff = 20 * time.Millisecond + t.Cleanup(s.Close) + return s +} + +func awaitValue(t *testing.T, ch <-chan string, want string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case got, ok := <-ch: + if !ok { + t.Fatalf("update channel closed while waiting for %q", want) + } + t.Logf("update: %q", got) + if got == want { + return + } + case <-deadline: + t.Fatalf("timed out waiting for value %q", want) + } + } +} + +func awaitClosed(t *testing.T, ch <-chan string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("timed out waiting for channel close") + } + } +} + +func updateSecret(t *testing.T, cs kubernetes.Interface, sec *corev1.Secret) { + t.Helper() + if _, err := cs.CoreV1().Secrets(sec.Namespace).Update(context.Background(), sec, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } +} + +func TestWatchInitialValue(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "postgres://host/db"})) + s := newTestStrategy(t, cs) + + // The hotload core passes uri.Path, which has a leading slash. + val, ch, err := s.Watch(context.Background(), "/mydb", "dsn=dsn&namespace=prod") + if err != nil { + t.Fatalf("Watch: %v", err) + } + if val != "postgres://host/db" { + t.Errorf("initial value = %q, want postgres://host/db", val) + } + if ch == nil { + t.Fatal("expected non-nil channel") + } +} + +func TestWatchSecretNotFound(t *testing.T) { + s := newTestStrategy(t, fakeClientset(t)) + if _, _, err := s.Watch(context.Background(), "/missing", "namespace=prod"); err == nil { + t.Fatal("expected error for missing secret") + } +} + +func TestWatchKeyNotFound(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"password": "hunter2"})) + s := newTestStrategy(t, cs) + if _, _, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn"); err == nil { + t.Fatal("expected error for missing key") + } +} + +func TestWatchDefaults(t *testing.T) { + // Default key is dsn.txt; default namespace outside a pod is "default". + cs := fakeClientset(t, makeSecret("default", "mydb", map[string]string{"dsn.txt": "postgres://host/db"})) + s := newTestStrategy(t, cs) + + val, _, err := s.Watch(context.Background(), "/mydb", "") + if err != nil { + t.Fatalf("Watch: %v", err) + } + if val != "postgres://host/db" { + t.Errorf("value = %q, want postgres://host/db", val) + } +} + +func TestWatchSeesUpdates(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, ch, "dsn-2") + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-3"})) + awaitValue(t, ch, "dsn-3") +} + +// TestWatchSeesDeleteAndRecreate: a deleted Secret keeps serving the last +// value; the recreate arrives as an Added event and propagates. +func TestWatchSeesDeleteAndRecreate(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + if err := cs.CoreV1().Secrets("prod").Delete(context.Background(), "mydb", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + if _, err := cs.CoreV1().Secrets("prod").Create(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + awaitValue(t, ch, "dsn-2") +} + +// TestWatchCatchesUpAfterReconnect: a change made while the API watch is +// down must be delivered by the reconnect's catch-up read — the gap that +// loses updates when reconnects only resume watching from "now". +func TestWatchCatchesUpAfterReconnect(t *testing.T) { + cs := fake.NewClientset() + if _, err := cs.CoreV1().Secrets("prod").Create(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"}), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + // Hand the strategy a controllable watcher, then kill it. + firstWatch := watch.NewFake() + watchCalls := make(chan struct{}, 10) + first := true + cs.PrependWatchReactor("secrets", func(action k8stesting.Action) (bool, watch.Interface, error) { + watchCalls <- struct{}{} + if first { + first = false + return true, firstWatch, nil + } + return false, nil, nil // fall through to the default tracker watch + }) + + s := newTestStrategy(t, cs) + _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + <-watchCalls // first watch established + + // Change the secret while the (about to die) first watch sees nothing, + // then drop the watch. + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + firstWatch.Stop() + + // The reconnect's catch-up Get must deliver the missed value. + awaitValue(t, ch, "dsn-2") + <-watchCalls // and a second watch was established +} + +// TestSlowSubscriberConvergesOnLatest: when a subscriber is not draining +// its channel, intermediate values may drop but the latest must win. +func TestSlowSubscriberConvergesOnLatest(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-0"})) + s := newTestStrategy(t, cs) + + _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + // Push several updates without reading; the channel has capacity 1. + for i := 1; i <= 5; i++ { + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-x"})) + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-final"})) + } + // Now drain: the last value seen must be dsn-final, not a stale one + // stuck in the buffer. + awaitValue(t, ch, "dsn-final") +} + +func TestMultipleSubscribersOneSecret(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, ch1, "dsn-2") + awaitValue(t, ch2, "dsn-2") +} + +// TestDistinctKeysAreIndependent: two watchers reading different data keys +// of the same Secret must each get their own key's value. +func TestDistinctKeysAreIndependent(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{ + "primary": "dsn-primary-1", + "replica": "dsn-replica-1", + })) + s := newTestStrategy(t, cs) + + ctx := context.Background() + vp, chP, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=primary") + if err != nil { + t.Fatal(err) + } + vr, chR, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=replica") + if err != nil { + t.Fatal(err) + } + if vp != "dsn-primary-1" || vr != "dsn-replica-1" { + t.Fatalf("initial values = %q / %q, want dsn-primary-1 / dsn-replica-1", vp, vr) + } + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{ + "primary": "dsn-primary-2", + "replica": "dsn-replica-2", + })) + awaitValue(t, chP, "dsn-primary-2") + awaitValue(t, chR, "dsn-replica-2") +} + +func TestCloseWatch(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx := context.Background() + qry := "namespace=prod&dsn=dsn" + _, ch, err := s.Watch(ctx, "/mydb", qry) + if err != nil { + t.Fatal(err) + } + if err := s.CloseWatch("/mydb", qry); err != nil { + t.Fatal(err) + } + awaitClosed(t, ch) + + // Closing an unknown watch is a no-op. + if err := s.CloseWatch("/other", qry); err != nil { + t.Fatal(err) + } +} + +// TestCloseWatchKeepsOtherSubscribers: closing one subscriber leaves the +// other one live. +func TestCloseWatchKeepsOtherSubscribers(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx := context.Background() + _, ch1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") + if err != nil { + t.Fatal(err) + } + _, ch2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + if err := s.CloseWatch("/mydb", "namespace=prod&dsn=dsn&forceKill=true"); err != nil { + t.Fatal(err) + } + awaitClosed(t, ch1) + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, ch2, "dsn-2") +} + +func TestStrategyClose(t *testing.T) { + cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := NewStrategyWithClientset(cs) + s.backoff = 20 * time.Millisecond + + _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + s.Close() + awaitClosed(t, ch) + + // Watch after Close re-initializes (matches the fsnotify strategy and + // keeps the package-registered instance usable after UnregisterStrategy + // + re-register cycles). + val, _, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatalf("Watch after Close: %v", err) + } + if val != "dsn-1" { + t.Errorf("value after reopen = %q, want dsn-1", val) + } + s.Close() +} + +// TestSecretName covers the path normalization between the hotload DSN and +// the Kubernetes Secret name. +func TestSecretName(t *testing.T) { + cases := map[string]string{ + "/mydb": "mydb", + "mydb": "mydb", + " /mydb ": "mydb", + "/a/b": "a/b", // invalid as a Secret name; surfaces as a Get error + } + for in, want := range cases { + if got := secretName(in); got != want { + t.Errorf("secretName(%q) = %q, want %q", in, got, want) + } + } +} + +var _ runtime.Object = (*corev1.Secret)(nil) From 6222d523ac0e65d91baf7df3efb65ad56b62c8da Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 23:39:02 -0500 Subject: [PATCH 09/14] Make integration postgres startup deterministic in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker compose up --wait only waits for the container to be running unless a healthcheck is defined, and the postgres entrypoint restarts the server after running init scripts — so the tests could connect during initialization and fail with connection-reset errors. Add a pg_isready healthcheck (targeting TCP, since the init-phase temporary server only listens on the unix socket) and retry the setup ping for up to a minute as defense for environments without the healthcheck. --- test/integration/docker/docker-compose.yaml | 16 ++++++++++------ test/integration/main_test.go | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/test/integration/docker/docker-compose.yaml b/test/integration/docker/docker-compose.yaml index 3c4d594..8544d64 100644 --- a/test/integration/docker/docker-compose.yaml +++ b/test/integration/docker/docker-compose.yaml @@ -1,6 +1,3 @@ -version: '3.8' - -# postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml services: db: image: postgres:10.3 @@ -11,8 +8,15 @@ services: - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true ports: - '5432:5432' - volumes: - # https://github.com/felipewom/docker-compose-postgres - # https://geshan.com.np/blog/2021/12/docker-postgres/ + # `docker compose up --wait` blocks on this. The TCP host target + # matters: the postgres entrypoint runs init scripts against a + # temporary unix-socket-only server and then restarts; a plain + # pg_isready would report ready during that phase. + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U admin -d hldatabase"] + interval: 2s + timeout: 3s + retries: 30 + volumes: # https://github.com/docker-library/docs/tree/master/postgres#initialization-scripts - ./intgtest_init.sql:/docker-entrypoint-initdb.d/intgtest_init.sql diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 37c446f..2d2dfad 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -97,6 +97,21 @@ func requirePostgres(t *testing.T) { if setupErr != nil { return } + // The TCP probe above only proves something is listening; postgres + // may still be initializing (its docker entrypoint restarts the + // server after running init scripts). Retry until it answers. + deadline := time.Now().Add(60 * time.Second) + for { + setupErr = adminDB.Ping() + if setupErr == nil { + break + } + if time.Now().After(deadline) { + setupErr = fmt.Errorf("postgres never became ready: %w", setupErr) + return + } + time.Sleep(500 * time.Millisecond) + } stmts := []string{ "DROP TABLE IF EXISTS test", "DROP USER IF EXISTS " + testDbUser, From 62c85db6ae44ccc68628aa5cb465ebc0ac51d83e Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Wed, 10 Jun 2026 23:45:59 -0500 Subject: [PATCH 10/14] Deterministically order secret mutations after watch registration in k8ssecret tests client-go's fake clientset ignores the resource version in watch options and only delivers events to watchers already registered with its tracker, so a test mutating a Secret while the strategy's watch goroutine was still establishing its watch lost the event and timed out (flaked in CI under load; a real API server replays from the resource version, so the strategy itself has no such gap). The fake clientset helper now wraps the tracker watch in a reactor that signals after registration, and every mutating test waits for that signal first. --- k8ssecret/e2e_test.go | 9 ++++-- k8ssecret/strategy_test.go | 62 +++++++++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/k8ssecret/e2e_test.go b/k8ssecret/e2e_test.go index afe5692..17e49d0 100644 --- a/k8ssecret/e2e_test.go +++ b/k8ssecret/e2e_test.go @@ -19,7 +19,7 @@ import ( // (including the leading slash the core puts on the path component), watch // establishment at sql.Open, and update propagation into the pool. func TestEndToEndThroughSQLOpen(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) // Swap the package-registered strategy instance for one bound to the // fake clientset; restore a pristine instance afterwards. @@ -54,6 +54,11 @@ func TestEndToEndThroughSQLOpen(t *testing.T) { t.Fatalf("initial query routed to %q, want dsn-1", got) } + // The fake clientset cannot replay events from before a watch is + // registered (a real API server can, via the resource version), so wait + // for the strategy's watch before rotating the secret. + awaitWatchReady(t, ready) + // Rotate the secret; the pool must converge on the new DSN. if _, err := cs.CoreV1().Secrets("prod").Update(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.UpdateOptions{}); err != nil { @@ -76,7 +81,7 @@ func TestEndToEndThroughSQLOpen(t *testing.T) { // TestSQLOpenErrorsOnMissingSecret: configuration problems surface at // sql.Open (the hotload core starts the watch there). func TestSQLOpenErrorsOnMissingSecret(t *testing.T) { - cs := fakeClientset(t) + cs, _ := fakeClientset(t) hotload.UnregisterStrategy("k8ssecret") hotload.RegisterStrategy("k8ssecret", NewStrategyWithClientset(cs)) diff --git a/k8ssecret/strategy_test.go b/k8ssecret/strategy_test.go index ecfcd31..8607007 100644 --- a/k8ssecret/strategy_test.go +++ b/k8ssecret/strategy_test.go @@ -25,7 +25,15 @@ func makeSecret(namespace, name string, data map[string]string) *corev1.Secret { } } -func fakeClientset(t *testing.T, secrets ...*corev1.Secret) kubernetes.Interface { +// fakeClientset returns a fake clientset pre-populated with secrets, plus a +// channel that signals every time a secret watch has been registered with +// the tracker. Unlike a real API server, the fake ignores the resource +// version in watch options and only delivers events to already-registered +// watchers — so tests MUST receive from ready before mutating a secret, or +// the mutation can race the strategy's watch establishment and be lost (in +// production the watch replays from the resource version and no such gap +// exists). +func fakeClientset(t *testing.T, secrets ...*corev1.Secret) (kubernetes.Interface, <-chan struct{}) { t.Helper() cs := fake.NewClientset() for _, sec := range secrets { @@ -33,7 +41,25 @@ func fakeClientset(t *testing.T, secrets ...*corev1.Secret) kubernetes.Interface t.Fatal(err) } } - return cs + ready := make(chan struct{}, 16) + cs.PrependWatchReactor("secrets", func(action k8stesting.Action) (bool, watch.Interface, error) { + w, err := cs.Tracker().Watch(action.GetResource(), action.GetNamespace()) + if err != nil { + return false, nil, err + } + ready <- struct{}{} // the watcher is registered; mutations are now visible to it + return true, w, nil + }) + return cs, ready +} + +func awaitWatchReady(t *testing.T, ready <-chan struct{}) { + t.Helper() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the strategy to establish its watch") + } } // newTestStrategy returns a strategy on a fake clientset with a short @@ -88,7 +114,7 @@ func updateSecret(t *testing.T, cs kubernetes.Interface, sec *corev1.Secret) { } func TestWatchInitialValue(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "postgres://host/db"})) + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "postgres://host/db"})) s := newTestStrategy(t, cs) // The hotload core passes uri.Path, which has a leading slash. @@ -105,14 +131,15 @@ func TestWatchInitialValue(t *testing.T) { } func TestWatchSecretNotFound(t *testing.T) { - s := newTestStrategy(t, fakeClientset(t)) + cs, _ := fakeClientset(t) + s := newTestStrategy(t, cs) if _, _, err := s.Watch(context.Background(), "/missing", "namespace=prod"); err == nil { t.Fatal("expected error for missing secret") } } func TestWatchKeyNotFound(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"password": "hunter2"})) + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"password": "hunter2"})) s := newTestStrategy(t, cs) if _, _, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn"); err == nil { t.Fatal("expected error for missing key") @@ -121,7 +148,7 @@ func TestWatchKeyNotFound(t *testing.T) { func TestWatchDefaults(t *testing.T) { // Default key is dsn.txt; default namespace outside a pod is "default". - cs := fakeClientset(t, makeSecret("default", "mydb", map[string]string{"dsn.txt": "postgres://host/db"})) + cs, _ := fakeClientset(t, makeSecret("default", "mydb", map[string]string{"dsn.txt": "postgres://host/db"})) s := newTestStrategy(t, cs) val, _, err := s.Watch(context.Background(), "/mydb", "") @@ -134,13 +161,14 @@ func TestWatchDefaults(t *testing.T) { } func TestWatchSeesUpdates(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } + awaitWatchReady(t, ready) updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) awaitValue(t, ch, "dsn-2") @@ -152,13 +180,14 @@ func TestWatchSeesUpdates(t *testing.T) { // TestWatchSeesDeleteAndRecreate: a deleted Secret keeps serving the last // value; the recreate arrives as an Added event and propagates. func TestWatchSeesDeleteAndRecreate(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } + awaitWatchReady(t, ready) if err := cs.CoreV1().Secrets("prod").Delete(context.Background(), "mydb", metav1.DeleteOptions{}); err != nil { t.Fatal(err) @@ -211,13 +240,14 @@ func TestWatchCatchesUpAfterReconnect(t *testing.T) { // TestSlowSubscriberConvergesOnLatest: when a subscriber is not draining // its channel, intermediate values may drop but the latest must win. func TestSlowSubscriberConvergesOnLatest(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-0"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-0"})) s := newTestStrategy(t, cs) _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } + awaitWatchReady(t, ready) // Push several updates without reading; the channel has capacity 1. for i := 1; i <= 5; i++ { @@ -230,7 +260,7 @@ func TestSlowSubscriberConvergesOnLatest(t *testing.T) { } func TestMultipleSubscribersOneSecret(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) ctx := context.Background() @@ -242,6 +272,7 @@ func TestMultipleSubscribersOneSecret(t *testing.T) { if err != nil { t.Fatal(err) } + awaitWatchReady(t, ready) updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) awaitValue(t, ch1, "dsn-2") @@ -251,7 +282,7 @@ func TestMultipleSubscribersOneSecret(t *testing.T) { // TestDistinctKeysAreIndependent: two watchers reading different data keys // of the same Secret must each get their own key's value. func TestDistinctKeysAreIndependent(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{ + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{ "primary": "dsn-primary-1", "replica": "dsn-replica-1", })) @@ -269,6 +300,8 @@ func TestDistinctKeysAreIndependent(t *testing.T) { if vp != "dsn-primary-1" || vr != "dsn-replica-1" { t.Fatalf("initial values = %q / %q, want dsn-primary-1 / dsn-replica-1", vp, vr) } + awaitWatchReady(t, ready) // primary key watch + awaitWatchReady(t, ready) // replica key watch updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{ "primary": "dsn-primary-2", @@ -279,7 +312,7 @@ func TestDistinctKeysAreIndependent(t *testing.T) { } func TestCloseWatch(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) ctx := context.Background() @@ -302,7 +335,7 @@ func TestCloseWatch(t *testing.T) { // TestCloseWatchKeepsOtherSubscribers: closing one subscriber leaves the // other one live. func TestCloseWatchKeepsOtherSubscribers(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) ctx := context.Background() @@ -315,6 +348,7 @@ func TestCloseWatchKeepsOtherSubscribers(t *testing.T) { t.Fatal(err) } + awaitWatchReady(t, ready) if err := s.CloseWatch("/mydb", "namespace=prod&dsn=dsn&forceKill=true"); err != nil { t.Fatal(err) } @@ -325,7 +359,7 @@ func TestCloseWatchKeepsOtherSubscribers(t *testing.T) { } func TestStrategyClose(t *testing.T) { - cs := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := NewStrategyWithClientset(cs) s.backoff = 20 * time.Millisecond From 212865472d98e544e3850c3cfdf9361dfbf94523 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Thu, 11 Jun 2026 08:45:34 -0500 Subject: [PATCH 11/14] CI: add aggregate build gate for branch protection Branch protection on main requires a status check named build, the job name in the pre-v3 workflow; the rewritten workflow's unit/integration jobs never report it, leaving PRs stuck on "expected". Add an aggregate gate job with that name needing both real jobs. It runs even when they fail (a skipped required check counts as satisfied) and keeps the required-check name stable while the unit matrix's Go versions change. --- .github/workflows/go.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d9eff99..8f51a81 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -36,3 +36,18 @@ jobs: - name: Stop postgres if: always() run: make postgres-docker-compose-down + + # Aggregate merge gate. Branch protection on main requires a check named + # "build" (the job name in the pre-v3 workflow); this job reports it and + # keeps the required-check name stable while the unit matrix's Go versions + # change. It must run on failure too: a skipped required check would count + # as satisfied. + build: + runs-on: ubuntu-latest + needs: [ unit, integration ] + if: always() + steps: + - name: Require unit and integration success + run: | + test "${{ needs.unit.result }}" = "success" + test "${{ needs.integration.result }}" = "success" From efb28ef7520e9708452d5475257c9e44f8f1bbb1 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Sun, 5 Jul 2026 12:59:31 -0500 Subject: [PATCH 12/14] Rework Strategy interface around per-watch Watchable handles Collapse Strategy to a single Watch method returning the current value plus a Watchable (Values/Close), per the PR #68 interface discussion. CloseWatch and Close are gone: per-watch lifetime is the returned handle or the Watch context, whichever ends first, so strategies no longer re-parse path/query to find the watch to close. Every Watch call now establishes an independent subscription with its own channel (previously two DSNs mapping to the same path+query silently shared one channel and split updates between them). The fsnotify strategy also releases its OS watcher when the last watch closes. The fake strategy moves out of internal/dbfake into the test packages: the interface now names hotload.Watchable, so a fake must import hotload, which dbfake cannot (internal tests import dbfake). --- MIGRATION.md | 38 ++++++++- README.md | 37 +++++--- driver.go | 65 +++++++------- driver_internal_test.go | 56 +++++++++++- fixture_test.go | 4 +- fsnotify/filewatcher.go | 134 ++++++++++++++--------------- fsnotify/filewatcher_test.go | 161 ++++++++++++++++------------------- group.go | 18 ++-- internal/dbfake/dbfake.go | 92 +------------------- k8ssecret/strategy.go | 135 ++++++++++++++--------------- k8ssecret/strategy_test.go | 126 ++++++++++++++------------- lifecycle_test.go | 6 +- strategy_fake_test.go | 109 ++++++++++++++++++++++++ 13 files changed, 547 insertions(+), 434 deletions(-) create mode 100644 strategy_fake_test.go diff --git a/MIGRATION.md b/MIGRATION.md index 3517cf2..e815f77 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -2,8 +2,9 @@ For most applications the upgrade is an import-path change. The driver name (`"hotload"`), the DSN format (`strategy://driver/path?forceKill=...`), the -registration functions, the `Strategy` interface and the graceful/forceKill -semantics are unchanged. +registration functions and the graceful/forceKill semantics are unchanged. +Only authors of custom strategies are affected by the reworked `Strategy` +interface (section 5). ## 1. Import path @@ -102,6 +103,39 @@ sees the new truthful method set. | `hotload.WithLogger` / `hotload.GetLogger` | still present (deprecated); prefer `logger.WithLogger` / `logger.GetLogger` | | `hotload/metrics` package | removed; see section 2 | | `hotload.Register` (alias mentioned in old README) | was already `RegisterSQLDriver`; unchanged | +| `Strategy.CloseWatch` / `Strategy.Close` | removed; `Watch` returns a per-watch `Watchable` handle (see below) | + +**Custom strategy authors:** `Strategy` is now a single-method interface. +`Watch` still receives `(ctx, pth, pathQry)` and still returns the current +value synchronously, but the update channel is wrapped in a per-watch +`Watchable` handle: + +```go +type Strategy interface { + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} + +type Watchable interface { + Values() <-chan string + Close() error +} +``` + +Lifecycle rules: + +- Each `Watch` call establishes an independent watch with its own channel, + even for a path/query pair already being watched (share the underlying + resource watch internally if you like). +- The watch ends when its `Watchable` is closed **or** the `Watch` context + is canceled; either way the strategy releases the watch's resources and + closes the `Values` channel. `Close` must be idempotent and must not call + back into hotload. +- There is no strategy-wide `Close` anymore: a registered strategy lives for + the process. Tests wanting isolation construct fresh strategy instances. + +This removes the identity bookkeeping v1 forced on strategies: `CloseWatch` +had to re-parse `pth`/`pathQry` to find the watch to close; now the handle +*is* the watch. ## 6. Dependency diet diff --git a/README.md b/README.md index 39d9fbc..04b28ed 100644 --- a/README.md +++ b/README.md @@ -76,28 +76,37 @@ fallback behavior exactly, so the difference is unobservable. Hotload has an interface for adding reload strategies. The interface looks like this: ```go -// Strategy is the plugin interface for hotload. +// Strategy is the plugin interface for hotload: given a resource, watch it +// and stream its values. type Strategy interface { - // Watch returns back the contents of the resource as well as a channel - // for subsequent updates (if the value has changed). If there is an error - // getting the initial value, an error is returned. - Watch(ctx context.Context, pth string, pathQry string) (value string, newValChan <-chan string, err error) + // Watch begins watching the resource identified by pth (pathQry carries + // the hotload DSN's encoded query parameters). It returns the resource's + // current value and a Watchable streaming subsequent values. Each call + // establishes an independent watch. The watch lives until its Watchable + // is closed or ctx is canceled. + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} - // CloseWatch closes the specified watch. - CloseWatch(pth string, pathQry string) error +// Watchable is one active watch established by Strategy.Watch. +type Watchable interface { + // Values returns the channel on which changed values of the watched + // resource are delivered. The strategy closes the channel when the + // watch ends. + Values() <-chan string - // Close resets/closes strategy, in particular closes all the update channels. - Close() + // Close releases the watch and closes its Values channel. Close is + // idempotent. + Close() error } ``` The strategies are loaded by calling the `RegisterStrategy` method in the hotload package. This is the same pattern the `database/sql` package uses for loading drivers. The strategy -implements the `Watch` method. The context passed to the strategy should be used to shut -down any code watching the passed `pth`. `pathQry` carries the hotload connection -string's encoded query parameters. The strategy doesn't have to use a real file to load -the config. `pth` represents a unique string that makes sense to the strategy. For -example, pth could point to a path in etcd or a kind/id in k8s. +implements the `Watch` method; each watch it hands out lives until the returned `Watchable` +is closed or the `Watch` context is canceled, whichever comes first. `pathQry` carries the +hotload connection string's encoded query parameters. The strategy doesn't have to use a +real file to load the config. `pth` represents a unique string that makes sense to the +strategy. For example, pth could point to a path in etcd or a kind/id in k8s. The hotload project ships with two strategies: diff --git a/driver.go b/driver.go index 264dd33..a59735c 100644 --- a/driver.go +++ b/driver.go @@ -62,18 +62,31 @@ import ( "github.com/infobloxopen/hotload/v3/logger" ) -// Strategy is the plugin interface for hotload. +// Strategy is the plugin interface for hotload: given a resource, watch it +// and stream its values. type Strategy interface { - // Watch returns back the contents of the resource as well as a channel - // for subsequent updates (if the value has changed). If there is an error - // getting the initial value, an error is returned. - Watch(ctx context.Context, pth string, pathQry string) (value string, newValChan <-chan string, err error) - - // CloseWatch closes the specified watch. - CloseWatch(pth string, pathQry string) error + // Watch begins watching the resource identified by pth (pathQry carries + // the hotload DSN's encoded query parameters). It returns the resource's + // current value and a Watchable streaming subsequent values. Each call + // establishes an independent watch, even for a path and query already + // being watched. The watch lives until its Watchable is closed or ctx + // is canceled. If the current value cannot be obtained, an error is + // returned and nothing is watched. + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} - // Close resets/closes strategy, in particular closes all the update channels. - Close() +// Watchable is one active watch established by Strategy.Watch. Values may +// carry secrets — they must not be logged unredacted. +type Watchable interface { + // Values returns the channel on which changed values of the watched + // resource are delivered. The strategy closes the channel when the + // watch ends. + Values() <-chan string + + // Close releases the watch: the strategy frees the resources backing it + // and closes the Values channel. Close is idempotent. It must not call + // back into hotload (the core calls it while holding internal locks). + Close() error } const forceKillParam = "forceKill" @@ -154,18 +167,13 @@ func RegisterStrategy(name string, strategy Strategy) { } // UnregisterStrategy unregisters the named driver strategy. -// Does nothing if strategy does not exist. -// Intended for internal unit-testing. +// Does nothing if strategy does not exist. Watches already established +// through the strategy are unaffected; they end when their groups close +// them. Intended for internal unit-testing. func UnregisterStrategy(name string) { mu.Lock() defer mu.Unlock() - strategy, ok := strategies[name] - if ok { - if strategy != nil { - strategy.Close() - } - delete(strategies, name) - } + delete(strategies, name) } // Strategies returns a sorted list of the names of the registered @@ -260,25 +268,25 @@ func (h *hdriver) getGroup(name string, pin bool) (*group, error) { return nil, err } - pathQry := queryParams.Encode() - value, newValChan, err := strategy.Watch(h.ctx, uri.Path, pathQry) + // The watch is scoped to the group's parent context: canceling it + // (group teardown) ends the watch even if Close were never called. + parentCtx, parentCancel := context.WithCancel(h.ctx) + value, watch, err := strategy.Watch(parentCtx, uri.Path, queryParams.Encode()) if err != nil { + parentCancel() return nil, err } - parentCtx, parentCancel := context.WithCancel(h.ctx) g = &group{ name: name, strategyName: uri.Scheme, - strategy: strategy, path: uri.Path, - pathQry: pathQry, sqlDriver: sqlDriver, forceKill: forceKill, killWindow: killWindow, parentCtx: parentCtx, parentCancel: parentCancel, - newValChan: newValChan, + watch: watch, } g.cur = newGeneration(parentCtx, value) h.groups[name] = g @@ -298,10 +306,9 @@ func (h *hdriver) getGroup(name string, pin bool) (*group, error) { // releaseGroup drops one connector reference and shuts the group down when // no references remain (unless a legacy Open pinned it). The strategy watch // is closed while h.mu is still held: getGroup establishes watches under -// the same lock, so a dying group's watch is fully closed before a new -// group with the same DSN can watch the same path — otherwise the strategy -// could hand the new group the doomed update channel and the new group -// would silently stop receiving config changes. +// the same lock, so a dying group's watch has fully released its strategy +// resources before a new group for the same DSN can establish its own. +// This is why Watchable.Close must never call back into hotload. func (h *hdriver) releaseGroup(name string) { h.mu.Lock() g, ok := h.groups[name] diff --git a/driver_internal_test.go b/driver_internal_test.go index 182ed66..0b39fcc 100644 --- a/driver_internal_test.go +++ b/driver_internal_test.go @@ -5,12 +5,62 @@ import ( "database/sql/driver" "fmt" "net/url" + "sync" "testing" "time" "github.com/infobloxopen/hotload/v3/internal/dbfake" ) +// testStrategy is a minimal in-package fake Strategy. The richer fake in +// the external test package cannot be used here (it imports hotload), and +// dbfake cannot host one (the Strategy interface names hotload.Watchable). +type testStrategy struct { + mu sync.Mutex + initial map[string]string + watches int +} + +func newTestStrategy(initial map[string]string) *testStrategy { + return &testStrategy{initial: initial} +} + +func (s *testStrategy) Watch(ctx context.Context, pth string, pathQry string) (string, Watchable, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.initial[pth] + if !ok { + return "", nil, fmt.Errorf("testStrategy: no initial value for path %q", pth) + } + s.watches++ + return value, &testWatch{strat: s, ch: make(chan string)}, nil +} + +func (s *testStrategy) Watches() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.watches +} + +type testWatch struct { + strat *testStrategy + ch chan string + closed bool // guarded by strat.mu +} + +func (w *testWatch) Values() <-chan string { return w.ch } + +func (w *testWatch) Close() error { + w.strat.mu.Lock() + defer w.strat.mu.Unlock() + if !w.closed { + w.closed = true + close(w.ch) + w.strat.watches-- + } + return nil +} + func Test_mergeConnStringOptions(t *testing.T) { tests := []struct { name string @@ -132,7 +182,7 @@ func TestRegisterPanics(t *testing.T) { }) mustPanic("nil strategy", func() { RegisterStrategy("internal-test-nilstrat", nil) }) mustPanic("dup strategy", func() { - s := dbfake.NewStrategy(nil) + s := newTestStrategy(nil) RegisterStrategy("internal-test-dupstrat", s) defer UnregisterStrategy("internal-test-dupstrat") RegisterStrategy("internal-test-dupstrat", s) @@ -142,7 +192,7 @@ func TestRegisterPanics(t *testing.T) { func TestRegistryLists(t *testing.T) { RegisterSQLDriver("internal-test-list-b", &dbfake.Driver{}) RegisterSQLDriver("internal-test-list-a", &dbfake.Driver{}) - RegisterStrategy("internal-test-list-strat", dbfake.NewStrategy(nil)) + RegisterStrategy("internal-test-list-strat", newTestStrategy(nil)) defer UnregisterStrategy("internal-test-list-strat") drivers := SQLDrivers() @@ -182,7 +232,7 @@ func TestLegacyOpenPath(t *testing.T) { h := newHdriver(ctx) drv := &dbfake.Driver{Caps: dbfake.CapsModern} - strat := dbfake.NewStrategy(map[string]string{"/cfg": "dsn-1"}) + strat := newTestStrategy(map[string]string{"/cfg": "dsn-1"}) RegisterSQLDriver("internal-test-legacy-drv", drv) RegisterStrategy("internal-test-legacy-strat", strat) defer UnregisterStrategy("internal-test-legacy-strat") diff --git a/fixture_test.go b/fixture_test.go index bdd909e..ed23bc8 100644 --- a/fixture_test.go +++ b/fixture_test.go @@ -33,7 +33,7 @@ type fixture struct { t *testing.T db *sql.DB drv *dbfake.Driver - strat *dbfake.Strategy + strat *fakeStrategy dsn string // the hotload DSN changes chan hotload.ConfigChangeEvent } @@ -61,7 +61,7 @@ func newFixture(t *testing.T, cfg fxCfg) *fixture { fx := &fixture{ t: t, drv: &dbfake.Driver{Caps: caps, ExecFn: cfg.execFn, QueryFn: cfg.queryFn}, - strat: dbfake.NewStrategy(map[string]string{fixturePath: initial}), + strat: newFakeStrategy(map[string]string{fixturePath: initial}), changes: make(chan hotload.ConfigChangeEvent, 100), } diff --git a/fsnotify/filewatcher.go b/fsnotify/filewatcher.go index ad62b96..93c16d7 100644 --- a/fsnotify/filewatcher.go +++ b/fsnotify/filewatcher.go @@ -46,25 +46,27 @@ type update struct { redactDsn string } -// queryWatch fans one path's updates out to one subscriber (the hotload -// group watching pth+pathQry). Its opLoop goroutine is the only writer of -// updateChan: it forwards queued updates and closes the channel when -// operChan closes, so channel operations never race. done is closed (under -// the strategy lock) when the watch is closed, unblocking an opLoop stuck -// sending to a subscriber that stopped receiving. +// queryWatch is one active watch handed out by Watch (it implements +// hotload.Watchable). It fans one path's updates out to one subscriber. Its +// opLoop goroutine is the only writer of updateChan: it forwards queued +// updates and closes the channel when operChan closes, so channel +// operations never race. done is closed (under the strategy lock) when the +// watch is closed, unblocking an opLoop stuck sending to a subscriber that +// stopped receiving. type queryWatch struct { parentPathW *pathWatch - pathQuery string + pathQuery string // for logging only updateChan chan string operChan chan update done chan struct{} + stopAfter func() bool // detaches the ctx-cancel hook installed by Watch } type pathWatch struct { parentStrat *Strategy watchPath string value string - queries map[string]*queryWatch + queries map[*queryWatch]struct{} } func (s *Strategy) readConfigFile(path string) (v []byte, err error) { @@ -169,7 +171,7 @@ func (s *Strategy) setVal(pth string, val string) { } pathW.value = val redactDsn := internal.RedactUrl(val) - for _, qryW := range pathW.queries { + for qryW := range pathW.queries { qryW.enqueue(update{dsn: val, redactDsn: redactDsn}) } } @@ -198,13 +200,14 @@ func (qw *queryWatch) enqueue(up update) { } } -// Watch implements the hotload.Strategy interface. -func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value string, values <-chan string, err error) { +// Watch implements the hotload.Strategy interface. Every call returns an +// independent watch; watches on the same path share one underlying file +// watch. +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { pth = path.Clean(pth) pathQry = strings.TrimSpace(pathQry) s.mu.Lock() defer s.mu.Unlock() - // Re-initialize after Close (which nils the maps out). if s.paths == nil { s.paths = make(map[string]*pathWatch) } @@ -223,93 +226,86 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value } else { s.logf("fsnotify.Watch", "new path to be watched: '%s'", pth) if err := s.watcher.Add(pth); err != nil { + s.closeWatcherIfIdle() return "", nil, err } bs, err := s.readConfigFile(pth) if err != nil { s.watcher.Remove(pth) + s.closeWatcherIfIdle() return "", nil, err } pathW = &pathWatch{ parentStrat: s, watchPath: pth, value: string(bs), - queries: make(map[string]*queryWatch), + queries: make(map[*queryWatch]struct{}), } s.paths[pth] = pathW } - qryW, found := pathW.queries[pathQry] - if found { - qryW.logf("fsnotify.Watch", "query already being watched") - } else { - pathW.logf("fsnotify.Watch", "new query to be watched: '%s'", pathQry) - qryW = &queryWatch{ - parentPathW: pathW, - pathQuery: pathQry, - updateChan: make(chan string), - operChan: make(chan update, 30), - done: make(chan struct{}), - } - pathW.queries[pathQry] = qryW - go qryW.opLoop() + qryW := &queryWatch{ + parentPathW: pathW, + pathQuery: pathQry, + updateChan: make(chan string), + operChan: make(chan update, 30), + done: make(chan struct{}), } + pathW.queries[qryW] = struct{}{} + go qryW.opLoop() + qryW.stopAfter = context.AfterFunc(ctx, func() { qryW.Close() }) + qryW.logf("fsnotify.Watch", "new watch") - return pathW.value, qryW.updateChan, nil + return pathW.value, qryW, nil } -// CloseWatch implements the hotload.Strategy interface. -// Closes the specified watch; its update channel is closed (asynchronously, -// by the watch's delivery goroutine) and, if this was the last watch on the -// path, the path is removed from the file watcher. -func (s *Strategy) CloseWatch(pth string, pathQry string) error { - pth = path.Clean(pth) - pathQry = strings.TrimSpace(pathQry) +// Values implements hotload.Watchable. +func (qw *queryWatch) Values() <-chan string { + return qw.updateChan +} + +// Close implements hotload.Watchable. The update channel is closed +// (asynchronously, by the watch's delivery goroutine); closing the last +// watch on a path removes the path from the file watcher, and closing the +// last watch on the strategy closes the underlying watcher. +func (qw *queryWatch) Close() error { + pathW := qw.parentPathW + s := pathW.parentStrat s.mu.Lock() defer s.mu.Unlock() - pathW, found := s.paths[pth] - if !found { - return nil + select { + case <-qw.done: + return nil // already closed + default: } - qryW, found := pathW.queries[pathQry] - if !found { + qw.stopAfter() + qw.shutdown() + delete(pathW.queries, qw) + qw.logf("fsnotify.Close", "closed watch") + + if len(pathW.queries) > 0 { return nil } - - delete(pathW.queries, pathQry) - qryW.shutdown() - qryW.logf("fsnotify.CloseWatch", "closed watch") - - if len(pathW.queries) == 0 { - delete(s.paths, pth) - if err := s.watcher.Remove(pth); err != nil { - s.errlogf("fsnotify.CloseWatch", "failed to remove '%s' from watcher, err=%v", pth, err) - return err - } - s.logf("fsnotify.CloseWatch", "removed path from being watched '%s'", pth) + delete(s.paths, pathW.watchPath) + var err error + if err = s.watcher.Remove(pathW.watchPath); err != nil { + s.errlogf("fsnotify.Close", "failed to remove '%s' from watcher, err=%v", pathW.watchPath, err) + } else { + s.logf("fsnotify.Close", "removed path from being watched '%s'", pathW.watchPath) } - return nil + s.closeWatcherIfIdle() + return err } -// Close implements the hotload.Strategy interface. -// Closes this strategy by closing the internal watcher -// and closing all the update channels. -func (s *Strategy) Close() { - s.mu.Lock() - defer s.mu.Unlock() - if s.watcher != nil { +// closeWatcherIfIdle closes the internal watcher when no paths remain +// watched, so an idle strategy holds no OS resources; the next Watch +// re-initializes it. Callers must hold the strategy lock. +func (s *Strategy) closeWatcherIfIdle() { + if len(s.paths) == 0 && s.watcher != nil { s.watcher.Close() - s.logf("fsnotify.Close", "closed internal watcher") s.watcher = nil + s.logf("fsnotify", "no paths watched; closed internal watcher") } - for _, pathW := range s.paths { - for _, qryW := range pathW.queries { - qryW.shutdown() - qryW.logf("fsnotify.Close", "closed watch") - } - pathW.queries = nil - } - s.paths = nil } // shutdown stops the watch's delivery goroutine. Callers must hold the diff --git a/fsnotify/filewatcher_test.go b/fsnotify/filewatcher_test.go index ada16cb..6a06957 100644 --- a/fsnotify/filewatcher_test.go +++ b/fsnotify/filewatcher_test.go @@ -9,15 +9,24 @@ import ( ) // newTestStrategy returns a strategy with a short resync period (the -// recovery path after failed re-watches) that is closed when the test ends. +// recovery path after failed re-watches). Watches are tied to per-test +// contexts (see testCtx), so cleanup happens by cancellation. func newTestStrategy(t *testing.T) *Strategy { t.Helper() s := NewStrategy() s.resyncPeriod = 50 * time.Millisecond - t.Cleanup(s.Close) return s } +// testCtx returns a context canceled when the test ends, closing every +// watch established with it. +func testCtx(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.WriteFile(path, []byte(content), 0o644); err != nil { @@ -64,7 +73,7 @@ func awaitClosed(t *testing.T, ch <-chan string) { func TestWatchMissingFile(t *testing.T) { s := newTestStrategy(t) - _, _, err := s.Watch(context.Background(), filepath.Join(t.TempDir(), "missing"), "") + _, _, err := s.Watch(testCtx(t), filepath.Join(t.TempDir(), "missing"), "") if err == nil { t.Fatal("expected an error watching a missing file") } @@ -75,7 +84,7 @@ func TestWatchInitialValueTrimmed(t *testing.T) { p := filepath.Join(t.TempDir(), "config") writeFile(t, p, " dsn-1 \n") - value, _, err := s.Watch(context.Background(), p, "") + value, _, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } @@ -89,16 +98,16 @@ func TestWatchSeesWrites(t *testing.T) { p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - _, updates, err := s.Watch(context.Background(), p, "") + _, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } writeFile(t, p, "dsn-2") - awaitValue(t, updates, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") writeFile(t, p, "dsn-3") - awaitValue(t, updates, "dsn-3") + awaitValue(t, w.Values(), "dsn-3") } // TestWatchSeesAtomicRename covers the write-then-rename pattern used by @@ -109,7 +118,7 @@ func TestWatchSeesAtomicRename(t *testing.T) { p := filepath.Join(dir, "config") writeFile(t, p, "dsn-1") - _, updates, err := s.Watch(context.Background(), p, "") + _, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } @@ -119,7 +128,7 @@ func TestWatchSeesAtomicRename(t *testing.T) { if err := os.Rename(tmp, p); err != nil { t.Fatal(err) } - awaitValue(t, updates, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") } // TestWatchRecoversFromRemoveAndRecreate: deleting the file fails the @@ -130,7 +139,7 @@ func TestWatchRecoversFromRemoveAndRecreate(t *testing.T) { p := filepath.Join(dir, "config") writeFile(t, p, "dsn-1") - _, updates, err := s.Watch(context.Background(), p, "") + _, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } @@ -140,7 +149,7 @@ func TestWatchRecoversFromRemoveAndRecreate(t *testing.T) { } time.Sleep(20 * time.Millisecond) writeFile(t, p, "dsn-2") - awaitValue(t, updates, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") } // TestWatchSeesKubernetesConfigMapSwap reproduces the kubelet's ConfigMap @@ -165,7 +174,7 @@ func TestWatchSeesKubernetesConfigMapSwap(t *testing.T) { t.Fatal(err) } - value, updates, err := s.Watch(context.Background(), p, "") + value, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } @@ -192,156 +201,136 @@ func TestWatchSeesKubernetesConfigMapSwap(t *testing.T) { t.Fatal(err) } - awaitValue(t, updates, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") } -// TestMultipleWatchersOnePath: watchers with different query strings get -// independent channels fed from one underlying file watch. +// TestMultipleWatchersOnePath: watches established by separate Watch calls +// get independent channels fed from one underlying file watch — even for an +// identical path and query (two hotload DSNs can differ only in the driver +// component). func TestMultipleWatchersOnePath(t *testing.T) { s := newTestStrategy(t) p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - ctx := context.Background() - _, ch1, err := s.Watch(ctx, p, "forceKill=true") + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, p, "forceKill=true") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, p, "") + _, w2, err := s.Watch(ctx, p, "forceKill=true") if err != nil { t.Fatal(err) } writeFile(t, p, "dsn-2") - awaitValue(t, ch1, "dsn-2") - awaitValue(t, ch2, "dsn-2") + awaitValue(t, w1.Values(), "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } -// TestWatchSamePathAndQueryShared: the same path+query pair returns the -// same channel rather than a second watch. -func TestWatchSamePathAndQueryShared(t *testing.T) { +func TestCloseClosesChannel(t *testing.T) { s := newTestStrategy(t) p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - ctx := context.Background() - _, ch1, err := s.Watch(ctx, p, "q=1") + _, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, p, "q=1") - if err != nil { + if err := w.Close(); err != nil { t.Fatal(err) } - if ch1 != ch2 { - t.Error("same path+query should share one update channel") - } -} + awaitClosed(t, w.Values()) -func TestCloseWatchClosesChannel(t *testing.T) { - s := newTestStrategy(t) - p := filepath.Join(t.TempDir(), "config") - writeFile(t, p, "dsn-1") - - _, updates, err := s.Watch(context.Background(), p, "") - if err != nil { - t.Fatal(err) + // Close is idempotent. + if err := w.Close(); err != nil { + t.Fatalf("second Close: %v", err) } - if err := s.CloseWatch(p, ""); err != nil { - t.Fatal(err) - } - awaitClosed(t, updates) } -// TestCloseWatchKeepsOtherQueries: closing one query watch leaves the other -// subscriber of the same path working. -func TestCloseWatchKeepsOtherQueries(t *testing.T) { +// TestCloseKeepsOtherWatches: closing one watch leaves the other subscriber +// of the same path working. +func TestCloseKeepsOtherWatches(t *testing.T) { s := newTestStrategy(t) p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - ctx := context.Background() - _, ch1, err := s.Watch(ctx, p, "q=1") + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, p, "q=1") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, p, "q=2") + _, w2, err := s.Watch(ctx, p, "q=2") if err != nil { t.Fatal(err) } - if err := s.CloseWatch(p, "q=1"); err != nil { + if err := w1.Close(); err != nil { t.Fatal(err) } - awaitClosed(t, ch1) + awaitClosed(t, w1.Values()) writeFile(t, p, "dsn-2") - awaitValue(t, ch2, "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } -func TestStrategyCloseClosesAllChannels(t *testing.T) { +// TestCtxCancelClosesWatch: canceling the Watch context releases the watch, +// exactly like Close. +func TestCtxCancelClosesWatch(t *testing.T) { s := newTestStrategy(t) - dir := t.TempDir() - p1 := filepath.Join(dir, "a") - p2 := filepath.Join(dir, "b") - writeFile(t, p1, "dsn-a") - writeFile(t, p2, "dsn-b") + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") - ctx := context.Background() - _, ch1, err := s.Watch(ctx, p1, "") + ctx, cancel := context.WithCancel(context.Background()) + _, w, err := s.Watch(ctx, p, "") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, p2, "") - if err != nil { - t.Fatal(err) - } - - s.Close() - awaitClosed(t, ch1) - awaitClosed(t, ch2) + cancel() + awaitClosed(t, w.Values()) } -// TestWatchAfterClose: the strategy re-initializes itself after Close (the -// registered global instance must survive UnregisterStrategy+re-register -// cycles). -func TestWatchAfterClose(t *testing.T) { +// TestWatchAfterAllClosed: closing the last watch releases the internal +// watcher; a later Watch must re-initialize it (the registered global +// instance lives for the process). +func TestWatchAfterAllClosed(t *testing.T) { s := newTestStrategy(t) p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - ctx := context.Background() - _, _, err := s.Watch(ctx, p, "") + _, w, err := s.Watch(testCtx(t), p, "") if err != nil { t.Fatal(err) } - s.Close() + if err := w.Close(); err != nil { + t.Fatal(err) + } - value, updates, err := s.Watch(ctx, p, "") + value, w2, err := s.Watch(testCtx(t), p, "") if err != nil { - t.Fatalf("Watch after Close: %v", err) + t.Fatalf("Watch after last Close: %v", err) } if value != "dsn-1" { t.Errorf("value = %q, want dsn-1", value) } writeFile(t, p, "dsn-2") - awaitValue(t, updates, "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } // TestAbandonedSubscriberDoesNotWedgeStrategy: if a subscriber stops // receiving (its hotload group is gone), pending updates must not block the -// strategy's delivery or its CloseWatch path. +// strategy's delivery or the watch's Close path. func TestAbandonedSubscriberDoesNotWedgeStrategy(t *testing.T) { s := newTestStrategy(t) p := filepath.Join(t.TempDir(), "config") writeFile(t, p, "dsn-1") - ctx := context.Background() + ctx := testCtx(t) _, abandoned, err := s.Watch(ctx, p, "q=abandoned") if err != nil { t.Fatal(err) } - _ = abandoned // never read from + // abandoned is never read from. _, live, err := s.Watch(ctx, p, "q=live") if err != nil { @@ -353,20 +342,20 @@ func TestAbandonedSubscriberDoesNotWedgeStrategy(t *testing.T) { writeFile(t, p, "dsn-x") writeFile(t, p, "dsn-2") } - awaitValue(t, live, "dsn-2") + awaitValue(t, live.Values(), "dsn-2") // Closing the abandoned watch must not deadlock. doneCh := make(chan struct{}) go func() { defer close(doneCh) - if err := s.CloseWatch(p, "q=abandoned"); err != nil { - t.Errorf("CloseWatch: %v", err) + if err := abandoned.Close(); err != nil { + t.Errorf("Close: %v", err) } }() select { case <-doneCh: case <-time.After(5 * time.Second): - t.Fatal("CloseWatch deadlocked on an abandoned subscriber") + t.Fatal("Close deadlocked on an abandoned subscriber") } - awaitClosed(t, abandoned) + awaitClosed(t, abandoned.Values()) } diff --git a/group.go b/group.go index c1acd11..d42daef 100644 --- a/group.go +++ b/group.go @@ -128,15 +128,13 @@ func (gen *generation) awaitIdle(window time.Duration) { type group struct { name string // the full hotload DSN strategyName string - strategy Strategy path string - pathQry string sqlDriver *driverInstance forceKill bool killWindow time.Duration parentCtx context.Context parentCancel context.CancelFunc - newValChan <-chan string + watch Watchable // refs and pinned are guarded by hdriver.mu. refs counts live // connectors; pinned marks groups created through the legacy @@ -154,12 +152,13 @@ type group struct { // group is shut down or the strategy closes the channel. It is the only // goroutine that swaps generations. func (g *group) runLoop() { + updates := g.watch.Values() for { select { case <-g.parentCtx.Done(): g.logf("group.runLoop", "parent context done, terminating") return - case newValue, ok := <-g.newValChan: + case newValue, ok := <-updates: if !ok { g.logf("group.runLoop", "strategy channel closed, terminating") EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) @@ -268,11 +267,10 @@ func (g *group) dial(ctx context.Context, gen *generation, dsn string) (driver.C return gen.connector.Connect(ctx) } -// closeWatch marks the group closed and stops its strategy watch. It is +// closeWatch marks the group closed and closes its strategy watch. It is // called by hdriver.releaseGroup with hdriver.mu held, which serializes it -// against getGroup's strategy.Watch calls (see releaseGroup). Strategy -// implementations must therefore never call back into hotload from -// CloseWatch. +// against getGroup's strategy.Watch calls (see releaseGroup). Watchable +// implementations must therefore never call back into hotload from Close. func (g *group) closeWatch() { g.mu.Lock() if g.closed { @@ -282,8 +280,8 @@ func (g *group) closeWatch() { g.closed = true g.mu.Unlock() - if err := g.strategy.CloseWatch(g.path, g.pathQry); err != nil { - g.logf("group.closeWatch", "CloseWatch error: %v", err) + if err := g.watch.Close(); err != nil { + g.logf("group.closeWatch", "watch close error: %v", err) } } diff --git a/internal/dbfake/dbfake.go b/internal/dbfake/dbfake.go index 4ead095..df13595 100644 --- a/internal/dbfake/dbfake.go +++ b/internal/dbfake/dbfake.go @@ -12,8 +12,10 @@ // observe which generation served a query. // // dbfake deliberately does not import the hotload package (hotload's -// internal tests import dbfake, so an import in the other direction would be -// a cycle). Strategy matches hotload.Strategy structurally. +// internal tests import dbfake, so an import in the other direction would +// be a cycle). For the same reason it contains no fake hotload.Strategy: +// the Strategy interface names the hotload.Watchable type, so a fake +// satisfying it must import hotload — each test package defines its own. package dbfake import ( @@ -427,89 +429,3 @@ func (l *CallLog) Reset() { defer l.mu.Unlock() l.calls = nil } - -// Strategy is a fake hotload strategy driven by tests. It matches -// hotload.Strategy structurally. -type Strategy struct { - mu sync.Mutex - initial map[string]string - chans map[string]chan string - watches int -} - -// NewStrategy returns a Strategy that answers Watch with the given initial -// values keyed by path. -func NewStrategy(initial map[string]string) *Strategy { - cp := make(map[string]string, len(initial)) - for k, v := range initial { - cp[k] = v - } - return &Strategy{initial: cp, chans: make(map[string]chan string)} -} - -// Push delivers a new value to the watcher of path. It blocks until the -// hotload run loop receives it, which makes change injection deterministic. -func (s *Strategy) Push(path, value string) { - s.mu.Lock() - ch := s.chans[path] - s.mu.Unlock() - if ch == nil { - panic("dbfake: Push on unwatched path " + path) - } - ch <- value -} - -// CloseChan closes the update channel for path, simulating a strategy that -// stops watching. -func (s *Strategy) CloseChan(path string) { - s.mu.Lock() - defer s.mu.Unlock() - if ch := s.chans[path]; ch != nil { - close(ch) - delete(s.chans, path) - } -} - -// Watches reports how many watches are currently open. -func (s *Strategy) Watches() int { - s.mu.Lock() - defer s.mu.Unlock() - return s.watches -} - -func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, <-chan string, error) { - s.mu.Lock() - defer s.mu.Unlock() - value, ok := s.initial[pth] - if !ok { - return "", nil, fmt.Errorf("dbfake: no initial value for path %q", pth) - } - ch, ok := s.chans[pth] - if !ok { - ch = make(chan string) - s.chans[pth] = ch - } - s.watches++ - return value, ch, nil -} - -func (s *Strategy) CloseWatch(pth string, pathQry string) error { - s.mu.Lock() - defer s.mu.Unlock() - s.watches-- - if ch := s.chans[pth]; ch != nil { - close(ch) - delete(s.chans, pth) - } - return nil -} - -func (s *Strategy) Close() { - s.mu.Lock() - defer s.mu.Unlock() - for p, ch := range s.chans { - close(ch) - delete(s.chans, p) - } - s.watches = 0 -} diff --git a/k8ssecret/strategy.go b/k8ssecret/strategy.go index ae6921c..9045ba9 100644 --- a/k8ssecret/strategy.go +++ b/k8ssecret/strategy.go @@ -96,15 +96,54 @@ type watchKey struct { key string } -// secretWatch fans one watched Secret value out to its subscribers (one per -// distinct pathQry). Subscriber channels have capacity 1 and are written -// with drop-oldest semantics, so a subscriber always converges on the -// latest value and a slow subscriber can never block delivery. All channel -// sends and closes happen under Strategy.mu, so they cannot race. +// secretWatch fans one watched Secret value out to its subscriptions. +// Subscription channels have capacity 1 and are written with drop-oldest +// semantics, so a subscriber always converges on the latest value and a +// slow subscriber can never block delivery. All channel sends and closes +// happen under Strategy.mu, so they cannot race. type secretWatch struct { - cancel context.CancelFunc - value string - queries map[string]chan string + cancel context.CancelFunc + value string + subs map[*subscription]struct{} +} + +// subscription is one active watch handed out by Watch (it implements +// hotload.Watchable). Closing the last subscription of a Secret value stops +// the underlying API watch. +type subscription struct { + strat *Strategy + wk watchKey + ch chan string + closed bool // guarded by strat.mu + stopAfter func() bool // detaches the ctx-cancel hook installed by Watch +} + +// Values implements hotload.Watchable. +func (sub *subscription) Values() <-chan string { + return sub.ch +} + +// Close implements hotload.Watchable. +func (sub *subscription) Close() error { + s := sub.strat + s.mu.Lock() + defer s.mu.Unlock() + if sub.closed { + return nil + } + sub.closed = true + sub.stopAfter() + close(sub.ch) + sw, ok := s.watches[sub.wk] + if !ok { + return nil + } + delete(sw.subs, sub) + if len(sw.subs) == 0 { + sw.cancel() + delete(s.watches, sub.wk) + } + return nil } // NewStrategy creates a strategy that builds its clientset lazily from @@ -151,10 +190,11 @@ func parseParams(pathQry string) (namespace, key string, err error) { // Watch implements hotload.Strategy. pth is the Secret name; pathQry // carries the namespace and dsn parameters (see the package documentation). -func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, <-chan string, error) { +// Every call returns an independent watch; watches on the same Secret value +// share one underlying API watch. +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { name := secretName(pth) - pathQry = strings.TrimSpace(pathQry) - namespace, key, err := parseParams(pathQry) + namespace, key, err := parseParams(strings.TrimSpace(pathQry)) if err != nil { return "", nil, err } @@ -170,7 +210,7 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (strin s.clientset = cs } if s.watches == nil { - // Re-initialize after Close. + // A zero-value Strategy works too. s.watches = make(map[watchKey]*secretWatch) } @@ -188,65 +228,18 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (strin watchCtx, cancel := context.WithCancel(context.Background()) sw = &secretWatch{ - cancel: cancel, - value: string(val), - queries: make(map[string]chan string), + cancel: cancel, + value: string(val), + subs: make(map[*subscription]struct{}), } s.watches[wk] = sw go s.runWatch(watchCtx, wk) } - ch, exists := sw.queries[pathQry] - if !exists { - ch = make(chan string, 1) - sw.queries[pathQry] = ch - } - return sw.value, ch, nil -} - -// CloseWatch implements hotload.Strategy. When the last subscriber of a -// Secret value is closed, the API watch is stopped. -func (s *Strategy) CloseWatch(pth string, pathQry string) error { - name := secretName(pth) - pathQry = strings.TrimSpace(pathQry) - namespace, key, err := parseParams(pathQry) - if err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - wk := watchKey{namespace: namespace, name: name, key: key} - sw, ok := s.watches[wk] - if !ok { - return nil - } - if ch, ok := sw.queries[pathQry]; ok { - close(ch) - delete(sw.queries, pathQry) - } - if len(sw.queries) == 0 { - sw.cancel() - delete(s.watches, wk) - } - return nil -} - -// Close implements hotload.Strategy: stops every watch and closes every -// update channel. -func (s *Strategy) Close() { - s.mu.Lock() - defer s.mu.Unlock() - for wk, sw := range s.watches { - sw.cancel() - for q, ch := range sw.queries { - close(ch) - delete(sw.queries, q) - } - delete(s.watches, wk) - } - s.watches = nil + sub := &subscription{strat: s, wk: wk, ch: make(chan string, 1)} + sw.subs[sub] = struct{}{} + sub.stopAfter = context.AfterFunc(ctx, func() { sub.Close() }) + return sw.value, sub, nil } // runWatch maintains the API watch for one Secret value until its context @@ -331,7 +324,7 @@ func (s *Strategy) consumeWatch(ctx context.Context, wk watchKey, rv string) err } } -// deliver pushes a changed value to every subscriber of the watch. +// deliver pushes a changed value to every subscription of the watch. // Channels have capacity 1; when full, the stale queued value is dropped so // the subscriber always converges on the latest one (dropping the new value // instead would leave a slow subscriber permanently stale). @@ -343,18 +336,18 @@ func (s *Strategy) deliver(wk watchKey, val string) { return } sw.value = val - for _, ch := range sw.queries { + for sub := range sw.subs { select { - case ch <- val: + case sub.ch <- val: continue default: } select { - case <-ch: // drop the stale queued value + case <-sub.ch: // drop the stale queued value default: } select { - case ch <- val: + case sub.ch <- val: default: } } diff --git a/k8ssecret/strategy_test.go b/k8ssecret/strategy_test.go index 8607007..87aeec0 100644 --- a/k8ssecret/strategy_test.go +++ b/k8ssecret/strategy_test.go @@ -63,15 +63,24 @@ func awaitWatchReady(t *testing.T, ready <-chan struct{}) { } // newTestStrategy returns a strategy on a fake clientset with a short -// reconnect backoff, closed when the test ends. +// reconnect backoff. Watches are tied to per-test contexts (see testCtx), +// so cleanup happens by cancellation. func newTestStrategy(t *testing.T, cs kubernetes.Interface) *Strategy { t.Helper() s := NewStrategyWithClientset(cs) s.backoff = 20 * time.Millisecond - t.Cleanup(s.Close) return s } +// testCtx returns a context canceled when the test ends, closing every +// watch established with it. +func testCtx(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + func awaitValue(t *testing.T, ch <-chan string, want string) { t.Helper() deadline := time.After(5 * time.Second) @@ -118,22 +127,22 @@ func TestWatchInitialValue(t *testing.T) { s := newTestStrategy(t, cs) // The hotload core passes uri.Path, which has a leading slash. - val, ch, err := s.Watch(context.Background(), "/mydb", "dsn=dsn&namespace=prod") + val, w, err := s.Watch(testCtx(t), "/mydb", "dsn=dsn&namespace=prod") if err != nil { t.Fatalf("Watch: %v", err) } if val != "postgres://host/db" { t.Errorf("initial value = %q, want postgres://host/db", val) } - if ch == nil { - t.Fatal("expected non-nil channel") + if w == nil || w.Values() == nil { + t.Fatal("expected non-nil watch and channel") } } func TestWatchSecretNotFound(t *testing.T) { cs, _ := fakeClientset(t) s := newTestStrategy(t, cs) - if _, _, err := s.Watch(context.Background(), "/missing", "namespace=prod"); err == nil { + if _, _, err := s.Watch(testCtx(t), "/missing", "namespace=prod"); err == nil { t.Fatal("expected error for missing secret") } } @@ -141,7 +150,7 @@ func TestWatchSecretNotFound(t *testing.T) { func TestWatchKeyNotFound(t *testing.T) { cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"password": "hunter2"})) s := newTestStrategy(t, cs) - if _, _, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn"); err == nil { + if _, _, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn"); err == nil { t.Fatal("expected error for missing key") } } @@ -151,7 +160,7 @@ func TestWatchDefaults(t *testing.T) { cs, _ := fakeClientset(t, makeSecret("default", "mydb", map[string]string{"dsn.txt": "postgres://host/db"})) s := newTestStrategy(t, cs) - val, _, err := s.Watch(context.Background(), "/mydb", "") + val, _, err := s.Watch(testCtx(t), "/mydb", "") if err != nil { t.Fatalf("Watch: %v", err) } @@ -164,17 +173,17 @@ func TestWatchSeesUpdates(t *testing.T) { cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) - _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } awaitWatchReady(t, ready) updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) - awaitValue(t, ch, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-3"})) - awaitValue(t, ch, "dsn-3") + awaitValue(t, w.Values(), "dsn-3") } // TestWatchSeesDeleteAndRecreate: a deleted Secret keeps serving the last @@ -183,7 +192,7 @@ func TestWatchSeesDeleteAndRecreate(t *testing.T) { cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) - _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } @@ -195,7 +204,7 @@ func TestWatchSeesDeleteAndRecreate(t *testing.T) { if _, err := cs.CoreV1().Secrets("prod").Create(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.CreateOptions{}); err != nil { t.Fatal(err) } - awaitValue(t, ch, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") } // TestWatchCatchesUpAfterReconnect: a change made while the API watch is @@ -221,7 +230,7 @@ func TestWatchCatchesUpAfterReconnect(t *testing.T) { }) s := newTestStrategy(t, cs) - _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } @@ -233,7 +242,7 @@ func TestWatchCatchesUpAfterReconnect(t *testing.T) { firstWatch.Stop() // The reconnect's catch-up Get must deliver the missed value. - awaitValue(t, ch, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") <-watchCalls // and a second watch was established } @@ -243,7 +252,7 @@ func TestSlowSubscriberConvergesOnLatest(t *testing.T) { cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-0"})) s := newTestStrategy(t, cs) - _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } @@ -256,27 +265,30 @@ func TestSlowSubscriberConvergesOnLatest(t *testing.T) { } // Now drain: the last value seen must be dsn-final, not a stale one // stuck in the buffer. - awaitValue(t, ch, "dsn-final") + awaitValue(t, w.Values(), "dsn-final") } +// TestMultipleSubscribersOneSecret: watches established by separate Watch +// calls get independent channels fed from one underlying API watch — even +// for an identical path and query. func TestMultipleSubscribersOneSecret(t *testing.T) { cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) - ctx := context.Background() - _, ch1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + _, w2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } awaitWatchReady(t, ready) updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) - awaitValue(t, ch1, "dsn-2") - awaitValue(t, ch2, "dsn-2") + awaitValue(t, w1.Values(), "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } // TestDistinctKeysAreIndependent: two watchers reading different data keys @@ -288,12 +300,12 @@ func TestDistinctKeysAreIndependent(t *testing.T) { })) s := newTestStrategy(t, cs) - ctx := context.Background() - vp, chP, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=primary") + ctx := testCtx(t) + vp, wP, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=primary") if err != nil { t.Fatal(err) } - vr, chR, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=replica") + vr, wR, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=replica") if err != nil { t.Fatal(err) } @@ -307,80 +319,80 @@ func TestDistinctKeysAreIndependent(t *testing.T) { "primary": "dsn-primary-2", "replica": "dsn-replica-2", })) - awaitValue(t, chP, "dsn-primary-2") - awaitValue(t, chR, "dsn-replica-2") + awaitValue(t, wP.Values(), "dsn-primary-2") + awaitValue(t, wR.Values(), "dsn-replica-2") } -func TestCloseWatch(t *testing.T) { +func TestCloseClosesChannel(t *testing.T) { cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) - ctx := context.Background() - qry := "namespace=prod&dsn=dsn" - _, ch, err := s.Watch(ctx, "/mydb", qry) + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } - if err := s.CloseWatch("/mydb", qry); err != nil { + if err := w.Close(); err != nil { t.Fatal(err) } - awaitClosed(t, ch) + awaitClosed(t, w.Values()) - // Closing an unknown watch is a no-op. - if err := s.CloseWatch("/other", qry); err != nil { - t.Fatal(err) + // Close is idempotent. + if err := w.Close(); err != nil { + t.Fatalf("second Close: %v", err) } } -// TestCloseWatchKeepsOtherSubscribers: closing one subscriber leaves the -// other one live. -func TestCloseWatchKeepsOtherSubscribers(t *testing.T) { +// TestCloseKeepsOtherSubscribers: closing one watch leaves the other one +// live. +func TestCloseKeepsOtherSubscribers(t *testing.T) { cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) s := newTestStrategy(t, cs) - ctx := context.Background() - _, ch1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") if err != nil { t.Fatal(err) } - _, ch2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + _, w2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } awaitWatchReady(t, ready) - if err := s.CloseWatch("/mydb", "namespace=prod&dsn=dsn&forceKill=true"); err != nil { + if err := w1.Close(); err != nil { t.Fatal(err) } - awaitClosed(t, ch1) + awaitClosed(t, w1.Values()) updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) - awaitValue(t, ch2, "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } -func TestStrategyClose(t *testing.T) { +// TestCtxCancelClosesWatch: canceling the Watch context releases the watch, +// exactly like Close; the strategy stays usable and re-establishes the API +// watch for a subsequent Watch. +func TestCtxCancelClosesWatch(t *testing.T) { cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) - s := NewStrategyWithClientset(cs) - s.backoff = 20 * time.Millisecond + s := newTestStrategy(t, cs) - _, ch, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + ctx, cancel := context.WithCancel(context.Background()) + _, w, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") if err != nil { t.Fatal(err) } - s.Close() - awaitClosed(t, ch) + cancel() + awaitClosed(t, w.Values()) - // Watch after Close re-initializes (matches the fsnotify strategy and - // keeps the package-registered instance usable after UnregisterStrategy - // + re-register cycles). - val, _, err := s.Watch(context.Background(), "/mydb", "namespace=prod&dsn=dsn") + val, w2, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") if err != nil { - t.Fatalf("Watch after Close: %v", err) + t.Fatalf("Watch after cancel: %v", err) } if val != "dsn-1" { t.Errorf("value after reopen = %q, want dsn-1", val) } - s.Close() + if err := w2.Close(); err != nil { + t.Fatal(err) + } } // TestSecretName covers the path normalization between the hotload DSN and diff --git a/lifecycle_test.go b/lifecycle_test.go index f1a1bc3..13ee944 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -303,9 +303,9 @@ func TestGroupTeardownAndSharing(t *testing.T) { // TestReopenWhileClosing races the close of a DSN's last sql.DB against a // fresh sql.Open of the same DSN. Whatever the interleaving, the surviving -// handle must keep receiving config changes: the dying group's watch -// teardown is serialized with new watch creation, so a new group can never -// be handed a doomed update channel. +// handle must keep receiving config changes: every Watch call gets its own +// update channel, and the dying group's watch teardown is serialized with +// new watch creation under the driver lock. func TestReopenWhileClosing(t *testing.T) { testutil.NoLeaks(t) fx := newFixture(t, fxCfg{noDB: true}) diff --git a/strategy_fake_test.go b/strategy_fake_test.go new file mode 100644 index 0000000..83c4fba --- /dev/null +++ b/strategy_fake_test.go @@ -0,0 +1,109 @@ +package hotload_test + +import ( + "context" + "fmt" + "sync" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// fakeStrategy is a hotload.Strategy driven by tests. It lives in the test +// package rather than in internal/dbfake because the Strategy interface +// names the hotload.Watchable type, and dbfake must not import hotload (the +// internal tests import dbfake). +type fakeStrategy struct { + mu sync.Mutex + initial map[string]string + subs map[string][]*fakeWatch + watches int +} + +// newFakeStrategy returns a fakeStrategy that answers Watch with the given +// initial values keyed by path. +func newFakeStrategy(initial map[string]string) *fakeStrategy { + cp := make(map[string]string, len(initial)) + for k, v := range initial { + cp[k] = v + } + return &fakeStrategy{initial: cp, subs: make(map[string][]*fakeWatch)} +} + +// Watch implements hotload.Strategy; every call opens an independent watch. +func (s *fakeStrategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.initial[pth] + if !ok { + return "", nil, fmt.Errorf("fakeStrategy: no initial value for path %q", pth) + } + w := &fakeWatch{strat: s, path: pth, ch: make(chan string)} + s.subs[pth] = append(s.subs[pth], w) + s.watches++ + context.AfterFunc(ctx, func() { w.Close() }) + return value, w, nil +} + +// Push delivers a new value to every watcher of path. It blocks until each +// hotload run loop receives it, which makes change injection deterministic. +func (s *fakeStrategy) Push(path, value string) { + s.mu.Lock() + watches := append([]*fakeWatch(nil), s.subs[path]...) + s.mu.Unlock() + if len(watches) == 0 { + panic("fakeStrategy: Push on unwatched path " + path) + } + for _, w := range watches { + w.ch <- value + } +} + +// CloseChan closes the update channels of every watch on path, simulating a +// strategy that stops watching on its own. +func (s *fakeStrategy) CloseChan(path string) { + s.mu.Lock() + defer s.mu.Unlock() + for _, w := range s.subs[path] { + w.closed = true + close(w.ch) + s.watches-- + } + delete(s.subs, path) +} + +// Watches reports how many watches are currently open. +func (s *fakeStrategy) Watches() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.watches +} + +// fakeWatch is the hotload.Watchable handed out by fakeStrategy.Watch. +type fakeWatch struct { + strat *fakeStrategy + path string + ch chan string + closed bool // guarded by strat.mu +} + +func (w *fakeWatch) Values() <-chan string { return w.ch } + +func (w *fakeWatch) Close() error { + s := w.strat + s.mu.Lock() + defer s.mu.Unlock() + if w.closed { + return nil + } + w.closed = true + close(w.ch) + s.watches-- + watches := s.subs[w.path] + for i, other := range watches { + if other == w { + s.subs[w.path] = append(watches[:i], watches[i+1:]...) + break + } + } + return nil +} From 0775ca1797c717944468e4eb7b2bf09de75312cb Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Sun, 5 Jul 2026 13:43:56 -0500 Subject: [PATCH 13/14] Add v1-migration safety nets for metrics enablement Two changes to soften the sharpest v1-to-v3 uptake hazards without reintroducing a metrics dependency in the core: EnablePrometheus with a nil (or default) registerer is now idempotent: the first call registers collectors and hooks, later calls return the same collectors. v1 enabled metrics as an import side effect, so ported code may reasonably enable defensively in both an application and a shared library; previously the second call panicked on duplicate registration. Explicit registerers keep per-call fresh collectors for tests. The core logs a one-time notice through the error logger when the first watch starts with no hooks registered. Without it, a service that ports the import paths and nothing else loses its hotload prometheus metrics with no signal anywhere; the notice points at the observability module and MIGRATION.md. Registering hooks or replacing the error logger silences it. --- MIGRATION.md | 11 ++++++ driver.go | 19 +++++++++ driver_internal_test.go | 68 ++++++++++++++++++++++++++++++++ observability/prometheus.go | 45 ++++++++++++++++++--- observability/prometheus_test.go | 15 +++++++ observer.go | 7 ++++ 6 files changed, 159 insertions(+), 6 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index e815f77..fa3cdfa 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -41,6 +41,17 @@ func main() { } ``` +With a nil (default) registerer the call is idempotent: the first call wins +and later calls return the same collectors, so an application and a shared +library can both enable metrics defensively without a duplicate-registration +panic. Explicit registerers register fresh collectors on every call. + +As a safety net, if the first watch starts with no hooks registered at all, +hotload logs a one-time notice through its error logger (visible by default) +pointing at this section — so forgetting the call above shows up in logs +instead of as silently empty dashboards. Registering any hooks, or replacing +the error logger via `logger.WithErrLogger`, silences it. + Metric names, labels, and the `HOTLOAD_PATH_CHKSUM_METRICS_ENABLE` gate are identical to v1: diff --git a/driver.go b/driver.go index a59735c..6db6a32 100644 --- a/driver.go +++ b/driver.go @@ -205,6 +205,9 @@ type hdriver struct { ctx context.Context mu sync.Mutex groups map[string]*group + + // hooksNote emits the one-time no-hooks notice (see warnIfNoHooks). + hooksNote sync.Once } var ( @@ -290,6 +293,7 @@ func (h *hdriver) getGroup(name string, pin bool) (*group, error) { } g.cur = newGeneration(parentCtx, value) h.groups[name] = g + h.hooksNote.Do(warnIfNoHooks) h.logf("hotload", "new group: '%s'", name) EmitWatchEvent(WatchEvent{GroupName: name, Strategy: uri.Scheme, Path: uri.Path}) go g.runLoop() @@ -328,6 +332,21 @@ func (h *hdriver) releaseGroup(name string) { } } +// warnIfNoHooks logs a one-time notice when the first watch starts with no +// hooks registered. Hotload v1 exported prometheus metrics as an import +// side effect; a v3 consumer that ports the import paths and nothing else +// would lose those metrics with no other signal. The notice goes to the +// error logger because, unlike the info logger, it is visible by default; +// silence it by registering hooks (e.g. the observability module) or via +// logger.WithErrLogger. +func warnIfNoHooks() { + if hooksRegistered() { + return + } + logger.ErrLogf("hotload:", "no hooks registered; unlike hotload v1, v3 does not export prometheus metrics unless enabled — "+ + "import github.com/infobloxopen/hotload/observability and call observability.MustEnablePrometheus(nil) before opening connections (see MIGRATION.md)") +} + func parseGroupParams(vs url.Values) (forceKill bool, killWindow time.Duration, err error) { killWindow = DefaultKillWindow if v, ok := vs[forceKillParam]; ok && len(v) > 0 { diff --git a/driver_internal_test.go b/driver_internal_test.go index 0b39fcc..22f830d 100644 --- a/driver_internal_test.go +++ b/driver_internal_test.go @@ -5,11 +5,13 @@ import ( "database/sql/driver" "fmt" "net/url" + "strings" "sync" "testing" "time" "github.com/infobloxopen/hotload/v3/internal/dbfake" + "github.com/infobloxopen/hotload/v3/logger" ) // testStrategy is a minimal in-package fake Strategy. The richer fake in @@ -259,6 +261,72 @@ func TestLegacyOpenPath(t *testing.T) { } } +// TestNoHooksNotice: creating the first group on a driver with no hooks +// registered logs a one-time notice pointing at the observability module +// (v1 exported prometheus metrics as an import side effect; without the +// notice a ported service loses them silently). With hooks registered, +// nothing is logged. +func TestNoHooksNotice(t *testing.T) { + resetHooks() + defer resetHooks() + + var mu sync.Mutex + notices := 0 + logger.WithErrLogger(func(args ...any) { + mu.Lock() + defer mu.Unlock() + if strings.Contains(fmt.Sprint(args...), "no hooks registered") { + notices++ + } + }) + defer logger.WithErrLogger(nil) + count := func() int { + mu.Lock() + defer mu.Unlock() + return notices + } + + RegisterSQLDriver("internal-test-notice-drv", &dbfake.Driver{Caps: dbfake.CapsModern}) + RegisterStrategy("internal-test-notice-strat", newTestStrategy(map[string]string{"/cfg": "dsn-1"})) + defer UnregisterStrategy("internal-test-notice-strat") + base := "internal-test-notice-strat://internal-test-notice-drv/cfg" + + // Cancelable driver contexts: legacy Open pins groups, so their run + // loops must be terminated via the parent context or they trip the + // goroutine-leak checks of later tests. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h := newHdriver(ctx) + conn, err := h.Open(base) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + conn2, err := h.Open(base + "?forceKill=true") // second group, same driver + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + + if got := count(); got != 1 { + t.Errorf("notices = %d, want exactly 1 per driver", got) + } + + // With hooks registered, a fresh driver stays quiet. + RegisterHooks(Hooks{OnConfigChange: func(ConfigChangeEvent) {}}) + h2 := newHdriver(ctx) + conn3, err := h2.Open(base + "?killWindow=200ms") + if err != nil { + t.Fatal(err) + } + defer conn3.Close() + + if got := count(); got != 1 { + t.Errorf("notices after hooks registered = %d, want still 1", got) + } +} + // TestEmitHelpersNilSafe ensures hooks with nil fields are skipped. func TestEmitHelpersNilSafe(t *testing.T) { resetHooks() diff --git a/observability/prometheus.go b/observability/prometheus.go index 6c8b85b..484b854 100644 --- a/observability/prometheus.go +++ b/observability/prometheus.go @@ -11,9 +11,14 @@ // observability.MustEnablePrometheus(nil) // nil = prometheus.DefaultRegisterer // ... // } +// +// With a nil registerer the call is idempotent, so a library and its caller +// can both enable metrics without coordinating. package observability import ( + "sync" + "github.com/prometheus/client_golang/prometheus" hotload "github.com/infobloxopen/hotload/v3" @@ -126,14 +131,42 @@ func (c *Collectors) Hooks() hotload.Hooks { } } -// EnablePrometheus creates the collectors, registers them with reg (the -// default prometheus registerer when reg is nil), and registers the hooks -// that feed them with hotload. Call it once during program initialization, -// before opening hotload connections. +// defaultMu guards the idempotent default-registerer path of +// EnablePrometheus; defaultCollectors holds its result. +var ( + defaultMu sync.Mutex + defaultCollectors *Collectors +) + +// EnablePrometheus creates the collectors, registers them with reg, and +// registers the hooks that feed them with hotload. Call it during program +// initialization, before opening hotload connections. +// +// When reg is nil (or prometheus.DefaultRegisterer) the call is idempotent: +// the first call registers collectors and hooks with the default registerer +// and later calls return that same *Collectors — so an application and a +// library it uses can both enable hotload metrics defensively without +// tripping duplicate-registration errors. Calls with any other registerer +// create and register fresh collectors every time; managing their lifetime +// is the caller's job (this is the path tests use with throwaway +// registries). func EnablePrometheus(reg prometheus.Registerer) (*Collectors, error) { - if reg == nil { - reg = prometheus.DefaultRegisterer + if reg != nil && reg != prometheus.DefaultRegisterer { + return enablePrometheus(reg) } + defaultMu.Lock() + defer defaultMu.Unlock() + if defaultCollectors == nil { + c, err := enablePrometheus(prometheus.DefaultRegisterer) + if err != nil { + return nil, err + } + defaultCollectors = c + } + return defaultCollectors, nil +} + +func enablePrometheus(reg prometheus.Registerer) (*Collectors, error) { c := NewCollectors() for _, collector := range c.All() { if err := reg.Register(collector); err != nil { diff --git a/observability/prometheus_test.go b/observability/prometheus_test.go index 5d1f8bf..3ce1abb 100644 --- a/observability/prometheus_test.go +++ b/observability/prometheus_test.go @@ -93,3 +93,18 @@ func TestEnablePrometheus(t *testing.T) { t.Error("second EnablePrometheus on same registry should error") } } + +// TestEnablePrometheusDefaultIdempotent: hotload v1 enabled metrics as an +// import side effect, so migrated code may enable defensively in more than +// one place; with the default registerer the second call must return the +// same collectors instead of a duplicate-registration panic. +func TestEnablePrometheusDefaultIdempotent(t *testing.T) { + c1 := MustEnablePrometheus(nil) + c2 := MustEnablePrometheus(nil) + if c1 != c2 { + t.Error("second MustEnablePrometheus(nil) returned different collectors") + } + if c3 := MustEnablePrometheus(prometheus.DefaultRegisterer); c3 != c1 { + t.Error("MustEnablePrometheus(DefaultRegisterer) should take the idempotent default path") + } +} diff --git a/observer.go b/observer.go index e9482e3..bf5fd5b 100644 --- a/observer.go +++ b/observer.go @@ -90,6 +90,13 @@ func resetHooks() { hooks = nil } +// hooksRegistered reports whether any hooks have been registered. +func hooksRegistered() bool { + hooksMu.RLock() + defer hooksMu.RUnlock() + return len(hooks) != 0 +} + func snapshotHooks() []Hooks { hooksMu.RLock() defer hooksMu.RUnlock() From 2dda84fa6f23ac81c6a6509b8fa8bba684e254d7 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Sun, 5 Jul 2026 15:36:33 -0500 Subject: [PATCH 14/14] Track chksum paths only for fsnotify; document branch strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The path-chksum collector hashes local files at scrape time, but the generic watch event feeds it paths from every strategy — a k8ssecret watch would register its Secret name as a path, producing a bogus zero-valued series and a hashing error on every scrape. Filter to the fsnotify strategy, which is exactly what v1 tracked (only the fsnotify strategy called AddToDefaultPathChksum); custom file-backed strategies can call AddPath directly. Also document the branch layout near the top of the README: main is v3, release-1.x is the v1 maintenance line. --- README.md | 13 ++++++++++++- observability/pathchksum.go | 7 ++++--- observability/prometheus.go | 6 +++++- observability/prometheus_test.go | 26 ++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 04b28ed..307349d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,18 @@ to the real database driver. When config changes are detected it retires connections in a manner that causes the `database/sql` package to create new connections with the new connection parameters. -Upgrading from hotload v1? See [MIGRATION.md](MIGRATION.md). +## Versions and branches + +| Branch | Module path | Status | +|---|---|---| +| [`main`](https://github.com/infobloxopen/hotload/tree/main) | `github.com/infobloxopen/hotload/v3` | current — all new development | +| [`release-1.x`](https://github.com/infobloxopen/hotload/tree/release-1.x) | `github.com/infobloxopen/hotload` | maintenance — security and critical fixes only | + +Existing v1 imports keep working: the un-suffixed module path always resolves +to the newest `v1.x` tag, and `go get -u` never moves a v1 consumer onto v3. +Upgrading to v3 is an explicit import-path change — see +[MIGRATION.md](MIGRATION.md). File v1 fixes as PRs against `release-1.x`; +they are released as `v1.7.x` tags from that branch. ```go import ( diff --git a/observability/pathchksum.go b/observability/pathchksum.go index 4b538a5..e4a5f2c 100644 --- a/observability/pathchksum.go +++ b/observability/pathchksum.go @@ -78,9 +78,10 @@ func NewPathChksumCollector(hasher FileHasher) *PathChksumCollector { } } -// AddPath starts reporting the checksum timestamp of pathStr. Duplicate adds -// are ignored. Fed automatically from hotload's watch events when the -// collector is wired through Collectors.Hooks. +// AddPath starts reporting the checksum timestamp of pathStr, which must be +// a local file path. Duplicate adds are ignored. Fed automatically for +// fsnotify watches when the collector is wired through Collectors.Hooks; +// custom file-backed strategies should call it directly. func (p *PathChksumCollector) AddPath(pathStr string) { if !p.enabled { return diff --git a/observability/prometheus.go b/observability/prometheus.go index 484b854..8a386e7 100644 --- a/observability/prometheus.go +++ b/observability/prometheus.go @@ -124,7 +124,11 @@ func (c *Collectors) Hooks() hotload.Hooks { c.HotloadModtimeLatencyHistogram.WithLabelValues(ev.Strategy, ev.Path).Observe(ev.Latency.Seconds()) }, OnWatch: func(ev hotload.WatchEvent) { - if !ev.Closed { + // Only fsnotify watches local files; other strategies' paths + // (Kubernetes Secret names, etcd keys, ...) are not hashable. + // File-backed strategies outside this repo can call + // Collectors.HotloadPathChksumTimestampSeconds.AddPath directly. + if !ev.Closed && ev.Strategy == "fsnotify" { c.HotloadPathChksumTimestampSeconds.AddPath(ev.Path) } }, diff --git a/observability/prometheus_test.go b/observability/prometheus_test.go index 3ce1abb..c40ed34 100644 --- a/observability/prometheus_test.go +++ b/observability/prometheus_test.go @@ -94,6 +94,32 @@ func TestEnablePrometheus(t *testing.T) { } } +// TestPathChksumOnlyFsnotify: watch events register chksum paths only for +// the fsnotify strategy — other strategies' paths (Secret names, etcd keys) +// are not local files and must not be hashed at scrape time. +func TestPathChksumOnlyFsnotify(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "true") + c := NewCollectors() + h := c.Hooks() + + h.OnWatch(hotload.WatchEvent{Strategy: "fsnotify", Path: "/etc/dsn"}) + h.OnWatch(hotload.WatchEvent{Strategy: "k8ssecret", Path: "/mydb"}) + h.OnWatch(hotload.WatchEvent{Strategy: "fsnotify", Path: "/etc/other", Closed: true}) + + col := c.HotloadPathChksumTimestampSeconds + col.mu.Lock() + defer col.mu.Unlock() + if _, ok := col.paths["/etc/dsn"]; !ok { + t.Error("fsnotify watch path not registered for chksum") + } + if _, ok := col.paths["/mydb"]; ok { + t.Error("k8ssecret path must not be registered for chksum") + } + if _, ok := col.paths["/etc/other"]; ok { + t.Error("closed watch event must not register a path") + } +} + // TestEnablePrometheusDefaultIdempotent: hotload v1 enabled metrics as an // import side effect, so migrated code may enable defensively in more than // one place; with the default registerer the second call must return the