diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 97e3e1c..8f51a81 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,21 +7,47 @@ on: branches: [ main ] jobs: - build: + unit: runs-on: ubuntu-latest + strategy: + matrix: + # database/sql's bad-conn retry behavior shifts subtly between Go + # releases; test the two most recent. + go-version: [ '1.25.x', '1.26.x' ] steps: - - uses: actions/checkout@v2 - - name: Unit Tests - run: make test-docker + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - name: Lint, generate-check, unit tests, dependency budget + run: make ci-test - integration-tests: + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + - name: Start postgres + run: make postgres-docker-compose-up + - name: Integration tests + run: make local-integration-tests + - name: Stop postgres + if: always() + run: make postgres-docker-compose-down + + # Aggregate merge gate. Branch protection on main requires a check named + # "build" (the job name in the pre-v3 workflow); this job reports it and + # keeps the required-check name stable while the unit matrix's Go versions + # change. It must run on failure too: a skipped required check would count + # as satisfied. + build: runs-on: ubuntu-latest + needs: [ unit, integration ] + if: always() steps: - - uses: actions/checkout@master - - uses: engineerd/setup-kind@v0.5.0 - with: - version: "v0.11.1" - - name: Integration Tests - run: | - kubectl cluster-info - make ci-integration-tests + - name: Require unit and integration success + run: | + test "${{ needs.unit.result }}" = "success" + test "${{ needs.integration.result }}" = "success" diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..fa3cdfa --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,157 @@ +# Migrating from hotload v1 to v3 + +For most applications the upgrade is an import-path change. The driver name +(`"hotload"`), the DSN format (`strategy://driver/path?forceKill=...`), the +registration functions and the graceful/forceKill semantics are unchanged. +Only authors of custom strategies are affected by the reworked `Strategy` +interface (section 5). + +## 1. Import path + +```diff +-import "github.com/infobloxopen/hotload" +-import _ "github.com/infobloxopen/hotload/fsnotify" ++import "github.com/infobloxopen/hotload/v3" ++import _ "github.com/infobloxopen/hotload/v3/fsnotify" +``` + +```sh +go get github.com/infobloxopen/hotload/v3 +``` + +## 2. Metrics are now opt-in (action required if you scrape hotload metrics) + +**This is the most important behavioral change.** Hotload v1 registered its +prometheus metrics with the default registerer as a side effect of importing +the package. The v3 core has no prometheus dependency at all; if you do +nothing, you get no metrics. + +To keep your dashboards working, add the observability module: + +```sh +go get github.com/infobloxopen/hotload/observability +``` + +```go +import "github.com/infobloxopen/hotload/observability" + +func main() { + observability.MustEnablePrometheus(nil) // nil = prometheus.DefaultRegisterer + ... +} +``` + +With a nil (default) registerer the call is idempotent: the first call wins +and later calls return the same collectors, so an application and a shared +library can both enable metrics defensively without a duplicate-registration +panic. Explicit registerers register fresh collectors on every call. + +As a safety net, if the first watch starts with no hooks registered at all, +hotload logs a one-time notice through its error logger (visible by default) +pointing at this section — so forgetting the call above shows up in logs +instead of as silently empty dashboards. Registering any hooks, or replacing +the error logger via `logger.WithErrLogger`, silences it. + +Metric names, labels, and the `HOTLOAD_PATH_CHKSUM_METRICS_ENABLE` gate are +identical to v1: + +| v1 metric | v3 | +|---|---| +| `transaction_sql_stmts` | same name, via observability module | +| `hotload_change_total` | same name, via observability module | +| `hotload_last_changed_timestamp_seconds` | same name, via observability module | +| `hotload_modtime_latency_histogram` | same name, via observability module | +| `hotload_path_chksum_timestamp_seconds` | same name, via observability module | + +One accuracy improvement: statements executed through prepared statements now +count toward `transaction_sql_stmts` (v1 did not wrap `driver.Stmt`, so +prepared-statement traffic was invisible). + +The `hotload/metrics` package is gone. Its helper APIs map as follows: + +| v1 | v3 | +|---|---| +| `metrics.GetCollectors()` | `observability.NewCollectors().All()` or the return of `EnablePrometheus` | +| `metrics.GRPCServiceKey` etc. | `observability.GRPCServiceKey` etc. | +| `internal.CollectAndRegexpCompare` (not exported in v1) | `observability/promtest.CollectAndRegexpCompare` | + +`hotload.ContextWithExecLabels` / `hotload.GetExecLabelsFromContext` are +unchanged and remain in the core. + +## 3. Truthful driver capabilities + +v1 implemented a fixed set of optional `database/sql/driver` interfaces and +answered `driver.ErrSkip` for unsupported ones. v3 wraps connections and +statements so optional interfaces exist if and only if the underlying driver +supports them. For mainstream drivers (lib/pq, pgx stdlib, mysql) behavior is +unchanged; for minimal drivers, `database/sql` now takes its documented +fallback paths (e.g. prepared statements) instead of `ErrSkip` round-trips. + +Code that type-asserts on the raw driver conn (e.g. inside `sql.Conn.Raw`) +sees the new truthful method set. + +## 4. Lifecycle changes + +- **Errors at `sql.Open`:** the strategy watch starts when `sql.Open` is + called (hotload now implements `driver.DriverContext`), so a bad DSN, + unknown strategy/driver, or unreadable config file fails fast at `sql.Open` + instead of at first query. +- **Teardown:** closing the last `sql.DB` for a DSN now stops the strategy + watch and the background goroutine; v1 leaked both. +- **forceKill:** in-flight operations canceled by a config change fail with + an error matching `hotload.ErrHotSwap` (use `errors.Is`). The new + `killWindow` DSN parameter (default `100ms`) bounds how long hotload waits + for in-flight work to observe cancellation before force-closing + connections. +- **modtime:** `modtime.NewModTimeMonitor` API is unchanged but reports + latency through hotload hooks; enable the observability module to keep the + histogram. + +## 5. Removed/changed APIs + +| v1 | v3 | +|---|---| +| `hotload.WithLogger` / `hotload.GetLogger` | still present (deprecated); prefer `logger.WithLogger` / `logger.GetLogger` | +| `hotload/metrics` package | removed; see section 2 | +| `hotload.Register` (alias mentioned in old README) | was already `RegisterSQLDriver`; unchanged | +| `Strategy.CloseWatch` / `Strategy.Close` | removed; `Watch` returns a per-watch `Watchable` handle (see below) | + +**Custom strategy authors:** `Strategy` is now a single-method interface. +`Watch` still receives `(ctx, pth, pathQry)` and still returns the current +value synchronously, but the update channel is wrapped in a per-watch +`Watchable` handle: + +```go +type Strategy interface { + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} + +type Watchable interface { + Values() <-chan string + Close() error +} +``` + +Lifecycle rules: + +- Each `Watch` call establishes an independent watch with its own channel, + even for a path/query pair already being watched (share the underlying + resource watch internally if you like). +- The watch ends when its `Watchable` is closed **or** the `Watch` context + is canceled; either way the strategy releases the watch's resources and + closes the `Values` channel. `Close` must be idempotent and must not call + back into hotload. +- There is no strategy-wide `Close` anymore: a registered strategy lives for + the process. Tests wanting isolation construct fresh strategy instances. + +This removes the identity bookkeeping v1 forced on strategies: `CloseWatch` +had to re-parse `pth`/`pathQry` to find the watch to close; now the handle +*is* the watch. + +## 6. Dependency diet + +The v3 core module depends only on `github.com/fsnotify/fsnotify`. The +following are gone from your module graph unless you import the +observability module: prometheus (client_golang, common, client_model, +procfs), pkg/errors, google/uuid, teivah/onecontext, colega/gaugefuncvec, +ginkgo/gomega, go-sqlmock, lib/pq. diff --git a/Makefile b/Makefile index 4684384..2ed9c7f 100644 --- a/Makefile +++ b/Makefile @@ -1,76 +1,59 @@ -GIT_COMMIT ?= $(shell git describe --dirty=-unsupported --always --tags || echo pre-commit) -IMAGE_NAME ?= hotload-integration-tests:$(GIT_COMMIT) +# The repository holds four Go modules: the hotload core (.), the +# prometheus adapter (observability), the Kubernetes Secret strategy +# (k8ssecret), and the postgres integration tests (test/integration). The +# committed go.work ties them together for development; most targets loop +# over all of them. +MODULES := . k8ssecret observability test/integration -get: - go get -t ./... +.PHONY: fmt vet tidy build test generate no-diff dep-budget ci-test \ + postgres-docker-compose-up postgres-docker-compose-down local-integration-tests -fmt: get - go fmt ./... +fmt: + @for m in $(MODULES); do (cd $$m && go fmt ./...) || exit 1; done -tidy: - go mod tidy - -# assert that there is no difference after running format -no-diff: - git diff --exit-code - -vet: fmt - go vet ./... - -build: vet - go build ./... - -get-ginkgo: - go get github.com/onsi/ginkgo/v2/ginkgo - -test: vet get-ginkgo go-test +vet: + @for m in $(MODULES); do (cd $$m && go vet ./...) || exit 1; done -go-test: - go test -race github.com/infobloxopen/hotload \ - github.com/infobloxopen/hotload/fsnotify \ - github.com/infobloxopen/hotload/internal \ - github.com/infobloxopen/hotload/metrics \ - github.com/infobloxopen/hotload/modtime - - -# test target which includes the no-diff fail condition -ci-test: fmt tidy no-diff test - -test-docker: - docker build -f Dockerfile.test . - -.integ-test-image-$(GIT_COMMIT): - docker build -f Dockerfile.integrationtest . -t $(IMAGE_NAME) - -integ-test-image: .integ-test-image-$(GIT_COMMIT) +tidy: + @for m in $(MODULES); do (cd $$m && go mod tidy) || exit 1; done -# this'll run outside of the build container -deploy-integration-tests: - helm upgrade hotload-integration-tests integrationtests/helm/hotload-integration-tests -i --set image.tag=$(GIT_COMMIT) +build: + @for m in $(MODULES); do (cd $$m && go build ./...) || exit 1; done -build-test: vet get-ginkgo - go test -c ./integrationtests +# Unit tests. The integration module skips itself when postgres is not +# reachable; use local-integration-tests to run it for real. +test: + @for m in $(MODULES); do (cd $$m && go test -race -timeout=5m -count=1 ./...) || exit 1; done -kind-create-cluster: - kind create cluster +# Regenerate the optional-interface combination wrappers (conn/stmt) and the +# dbfake capability views. +generate: + go generate ./... -kind-load: - kind load docker-image $(IMAGE_NAME) +# assert that there is no difference after running format/tidy/generate +no-diff: + git diff --exit-code -ci-integration-tests: integ-test-image kind-load deploy-integration-tests - (helm test --timeout=600s hotload-integration-tests || (kubectl logs hotload-integration-tests-job && exit 1)) && kubectl logs hotload-integration-tests-job +# The hotload core must stay near-stdlib-only: its sole direct dependency is +# fsnotify. Fails when dependency creep adds more. +dep-budget: + @reqs=$$(go mod edit -json | go run ./internal/depbudget); \ + if [ "$$reqs" != "github.com/fsnotify/fsnotify" ]; then \ + echo "dependency budget exceeded; direct requires of the root module:"; \ + echo "$$reqs"; \ + exit 1; \ + fi -delete-all: - helm uninstall hotload-integration-tests || true - kubectl delete pvc --all || true - kubectl delete pods --all || true +ci-test: fmt tidy generate no-diff vet test dep-budget postgres-docker-compose-up: - cd integrationtests/docker; docker compose up --detach + cd test/integration/docker; docker compose up --detach --wait postgres-docker-compose-down: - cd integrationtests/docker; docker compose down + cd test/integration/docker; docker compose down -# Requires postgres db, see target postgres-docker-compose-up +# Requires postgres, see target postgres-docker-compose-up local-integration-tests: - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true go test -v -race -timeout=3m -count=1 github.com/infobloxopen/hotload/integrationtests + cd test/integration && \ + HOTLOAD_INTEGRATION_TESTS=1 HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true \ + go test -v -race -timeout=5m -count=1 ./... diff --git a/README.md b/README.md index 847101e..307349d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,27 @@ -[![Go Reference](https://pkg.go.dev/badge/github.com/infobloxopen/hotload.svg)](https://pkg.go.dev/github.com/infobloxopen/hotload) +[![Go Reference](https://pkg.go.dev/badge/github.com/infobloxopen/hotload/v3.svg)](https://pkg.go.dev/github.com/infobloxopen/hotload/v3) # hotload Hotload is a Golang `database/sql` compatible package that supports dynamic reloading of database configuration. In the typical use of `sql.Open()`, users must close the returned DB object and recreate it to change the connection string. Hotload works by registering a driver that proxies -the [`Driver` interface](https://pkg.go.dev/database/sql/driver#Driver) +the [`Driver` interface](https://pkg.go.dev/database/sql/driver#Driver) to the real database driver. When config changes -are detected it closes connections in a manner that causes the `database/sql` +are detected it retires connections in a manner that causes the `database/sql` package to create new connections with the new connection parameters. +## Versions and branches + +| Branch | Module path | Status | +|---|---|---| +| [`main`](https://github.com/infobloxopen/hotload/tree/main) | `github.com/infobloxopen/hotload/v3` | current — all new development | +| [`release-1.x`](https://github.com/infobloxopen/hotload/tree/release-1.x) | `github.com/infobloxopen/hotload` | maintenance — security and critical fixes only | + +Existing v1 imports keep working: the un-suffixed module path always resolves +to the newest `v1.x` tag, and `go get -u` never moves a v1 consumer onto v3. +Upgrading to v3 is an explicit import-path change — see +[MIGRATION.md](MIGRATION.md). File v1 fixes as PRs against `release-1.x`; +they are released as `v1.7.x` tags from that branch. + ```go import ( // import the std lib sql package @@ -17,10 +30,10 @@ import ( log "github.com/sirupsen/logrus" // this import registers hotload with the sql package - "github.com/infobloxopen/hotload" + "github.com/infobloxopen/hotload/v3" // this import registers the fsnotify hotload strategy - _ "github.com/infobloxopen/hotload/fsnotify" + _ "github.com/infobloxopen/hotload/v3/fsnotify" // this import registers the postgres driver with the sql package "github.com/lib/pq" @@ -28,7 +41,7 @@ import ( func init() { // this function call registers the lib/pq postgres driver with hotload - hotload.Register("postgres", pq.Driver{}) + hotload.RegisterSQLDriver("postgres", &pq.Driver{}) } func main() { @@ -47,60 +60,172 @@ The above code: In the `main()` function, the `sql.Open` call uses the hotload driver. The URL for the connection string specifies `fsnotify` in the scheme. This is the hotload strategy. The -hostname in the URL specifies the real database driver (`postgres` in the example above). +hostname in the URL specifies the real database driver (`postgres` in the example above). Finally, the path and query parameters are left for the hotload strategy plugin to configure themselves. Below is an example of a `lib/pq` Postgres connection string that would have been stored at `/tmp/myconfig.txt` ``` user=pqgotest dbname=pqgotest sslmode=verify-full ``` +# Faithful driver capabilities + +`database/sql` discovers what a driver can do through type assertions on the +optional interfaces of `database/sql/driver` (`ExecerContext`, +`QueryerContext`, `Pinger`, `NamedValueChecker`, `StmtExecContext`, …). +Hotload wraps connections and statements so that a wrapped object implements +an optional interface **if and only if** the underlying driver supports the +capability. If your driver has no `ExecerContext` (or legacy `Execer`), +`database/sql` correctly falls back to prepared statements instead of +bouncing off `driver.ErrSkip`. + +The interfaces hotload needs for its own lifecycle (`ConnPrepareContext`, +`ConnBeginTx`, `SessionResetter`, `Validator`) are always implemented; when +the underlying driver lacks them, hotload replicates `database/sql`'s own +fallback behavior exactly, so the difference is unobservable. + # Strategies Hotload has an interface for adding reload strategies. The interface looks like this: ```go -// Strategy is the plugin interface for hotload. +// Strategy is the plugin interface for hotload: given a resource, watch it +// and stream its values. type Strategy interface { - // Watch returns back the contents of the resource as well as a channel - // for subsequent updates (if the value has changed). If there is an error - // getting the initial value, an error is returned. - Watch(ctx context.Context, pth string, options url.Values) (value string, values <-chan string, err error) + // Watch begins watching the resource identified by pth (pathQry carries + // the hotload DSN's encoded query parameters). It returns the resource's + // current value and a Watchable streaming subsequent values. Each call + // establishes an independent watch. The watch lives until its Watchable + // is closed or ctx is canceled. + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} + +// Watchable is one active watch established by Strategy.Watch. +type Watchable interface { + // Values returns the channel on which changed values of the watched + // resource are delivered. The strategy closes the channel when the + // watch ends. + Values() <-chan string + + // Close releases the watch and closes its Values channel. Close is + // idempotent. + Close() error } ``` The strategies are loaded by calling the `RegisterStrategy` method in the hotload package. This is the same pattern the `database/sql` package uses for loading drivers. The strategy -implements the `Watch` method. The context passed to the strategy should be used to shut -down any code watching the passed `pth`. Options are taken from the hotload connection -string query parameters. The strategy doesn't have to use a real file to load the config. -`pth` represents a unique string that makes sense to the strategy. For example, pth could -point to a path in etcd or a kind/id in k8s. +implements the `Watch` method; each watch it hands out lives until the returned `Watchable` +is closed or the `Watch` context is canceled, whichever comes first. `pathQry` carries the +hotload connection string's encoded query parameters. The strategy doesn't have to use a +real file to load the config. `pth` represents a unique string that makes sense to the +strategy. For example, pth could point to a path in etcd or a kind/id in k8s. -The hotload project ships with one hotload strategy: `fsnotify`. +The hotload project ships with two strategies: -Note: In your project, if you do not implement your own `Strategy`, and instead choose to use the out-of-the-box -`fsnotify` strategy, you must import the `fsnotify` package in your project to register at least one strategy with +* `fsnotify` (in the core module) watches a file on disk — the right choice when the + connection string is mounted into the pod (ConfigMap/Secret volumes included). +* `k8ssecret` (module [`github.com/infobloxopen/hotload/k8ssecret`](k8ssecret/)) watches a + Kubernetes Secret through the API server, for deployments that need credentials from a + Secret in **another namespace**, which Kubernetes cannot mount as a volume: + + ```go + import _ "github.com/infobloxopen/hotload/k8ssecret" + + db, err := sql.Open("hotload", "k8ssecret://pgx/myapp-db?namespace=prod&dsn=dsn.txt") + ``` + + The path component is the Secret name; `namespace` defaults to the pod's own namespace + and `dsn` (the data key holding the connection string) defaults to `dsn.txt`. The pod's + service account needs `get` and `watch` on the Secret. It lives in its own module so + `client-go` stays out of the hotload core. + +Note: In your project, if you do not implement your own `Strategy`, and instead choose to use an out-of-the-box +strategy, you must import its package in your project to register at least one strategy with hotload, otherwise an error will occur at runtime as the `database/sql` package will not be able to locate/load your intended hotload strategy as a recognizable driver. # Force Kill -By default, the hotload driver gracefully closes connections to the underlying driver. If your application holds connections open with long-running operations, this will prevent graceful switchover to new data sources. - -Adding `forceKill=true` to your DSN will cause the hotload driver to close the underlying connection manually when a -change to the connection information is detected. +By default, the hotload driver gracefully retires connections to the underlying driver: +in-flight work finishes on the old connection, and the pool replaces connections the +next time it reuses them. A retired generation gets one grace period — its connections +are force-closed when the *next* config change arrives. +Adding `forceKill=true` to your DSN will cause the hotload driver to cancel in-flight +operations and close the underlying connections as soon as a change to the connection +information is detected. Operations canceled this way fail with an error matching +`hotload.ErrHotSwap` (via `errors.Is`). For example: ``` db, err := sql.Open("hotload", "fsnotify://postgres/tmp/myconfig.txt?forceKill=true") ``` -# How To Run Integration Tests Locally +With `forceKill=true`, hotload waits a bounded window (default 100ms) for in-flight +operations to observe the cancellation before force-closing their connections, so a +driver that ignores context cancellation can never wedge config-change processing. +Tune it with the `killWindow` DSN parameter, e.g. `?forceKill=true&killWindow=250ms`. + +# Connection lifecycle + +`sql.Open("hotload", dsn)` starts the strategy watch for that DSN; multiple `sql.Open` +calls with the same DSN share one watch. Closing the last `sql.DB` for a DSN stops the +watch and its background goroutine. Note that this means configuration errors (unknown +strategy or driver, unreadable config file) surface at `sql.Open` instead of at first +use. + +# Metrics + +Hotload's core has no metrics dependency. It emits events through a small hooks API +(`hotload.RegisterHooks`); the +[`observability`](https://pkg.go.dev/github.com/infobloxopen/hotload/observability) +module adapts those events to prometheus, preserving the metric names of hotload v1: + +```go +import "github.com/infobloxopen/hotload/observability" + +func main() { + // nil registers with prometheus.DefaultRegisterer + observability.MustEnablePrometheus(nil) + ... +} +``` + +| Metric | Description | +|---|---| +| `transaction_sql_stmts` | sql statements per transaction, labeled by the exec labels carried via `hotload.ContextWithExecLabels` | +| `hotload_change_total` | config changes per hotload DSN | +| `hotload_last_changed_timestamp_seconds` | unix timestamp of the last change per DSN | +| `hotload_modtime_latency_histogram` | staleness of watched files (fed by the `modtime` package) | +| `hotload_path_chksum_timestamp_seconds` | when each watched file's checksum last changed (opt-in via `HOTLOAD_PATH_CHKSUM_METRICS_ENABLE`) | + +# How To Run Tests Locally + +Unit tests (no database needed): +``` +$ make test +``` + +Integration tests (real postgres via docker compose): ``` $ make postgres-docker-compose-up -$ cd integrationstests -$ go test -v -race -timeout=3m -$ vi ... -$ go test -v -race # this can be repeated in your edit-run-test cycle +$ make local-integration-tests $ make postgres-docker-compose-down ``` +The integration tests honor `HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST` and +`HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT` if your postgres lives elsewhere, and skip +themselves when no server is reachable. + +# Repository layout + +This repository holds three Go modules tied together by the committed `go.work`: + +| Module | Purpose | Dependencies | +|---|---|---| +| `github.com/infobloxopen/hotload/v3` | the driver, `fsnotify` strategy, `logger`, `modtime` | `fsnotify` only | +| `github.com/infobloxopen/hotload/k8ssecret` | Kubernetes Secret strategy (cross-namespace) | `client-go` | +| `github.com/infobloxopen/hotload/observability` | prometheus adapter + test helpers | prometheus | +| `github.com/infobloxopen/hotload/test/integration` | postgres integration tests (never imported) | `lib/pq` | + +The combination wrapper types are generated; after changing `internal/gen`, +`conn_pieces.go` or `stmt_pieces.go`, run `make generate` and commit the result +(CI fails on a diff). diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..8271e3b --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,54 @@ +# Releasing + +This repository contains two released modules with independent tag +namespaces, plus an internal test module that is never released: + +| Module | Tag format | Example | +|---|---|---| +| `github.com/infobloxopen/hotload/v3` (repo root) | `vX.Y.Z` | `v3.0.0` | +| `github.com/infobloxopen/hotload/k8ssecret` | `k8ssecret/vX.Y.Z` | `k8ssecret/v1.0.0` | +| `github.com/infobloxopen/hotload/observability` | `observability/vX.Y.Z` | `observability/v1.0.0` | +| `github.com/infobloxopen/hotload/test/integration` | never tagged | — | + +## Order matters + +`observability/go.mod` and `k8ssecret/go.mod` require +`github.com/infobloxopen/hotload/v3` by version. The `replace` directives in +those files only affect building inside this repository — **consumers +resolve the `require` line**, so it must point at a tag that exists. + +For a release that touches the root and the satellite modules: + +1. Tag the root module first: + ```sh + git tag v3.Y.Z && git push origin v3.Y.Z + ``` +2. Bump the require in each satellite module to the new tag: + ```sh + (cd observability && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z) + (cd k8ssecret && go mod edit -require=github.com/infobloxopen/hotload/v3@v3.Y.Z) + ``` + Commit, then tag each satellite: + ```sh + git tag observability/vA.B.C && git push origin observability/vA.B.C + git tag k8ssecret/vA.B.C && git push origin k8ssecret/vA.B.C + ``` + +3. Sanity-check as a consumer (from any directory outside this repo): + ```sh + cd $(mktemp -d) && go mod init smoke + GOFLAGS=-mod=mod go get github.com/infobloxopen/hotload/v3@v3.Y.Z \ + github.com/infobloxopen/hotload/observability@vA.B.C \ + github.com/infobloxopen/hotload/k8ssecret@vA.B.C + ``` + This catches the failure mode of a require pointing at a nonexistent + version (a `replace` masks it inside the repo). + +## Notes + +- The root module path carries the `/v3` suffix; tags below `v3.0.0` on the + root module are invalid for it. +- Do not tag `observability` with a version of the root module's tag series; + the namespaces are independent and need not be aligned. +- `test/integration` has a permanent `replace` and a nominal require + version; it is intentionally excluded from releases. diff --git a/chanGroup_test.go b/chanGroup_test.go deleted file mode 100644 index bb4d47c..0000000 --- a/chanGroup_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - "fmt" - "log" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus/testutil" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" -) - -type testConn struct { - closed bool -} - -func (tc *testConn) Open(name string) (driver.Conn, error) { - return tc, nil -} - -func (tc *testConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} -func (tc *testConn) Begin() (driver.Tx, error) { - return nil, nil -} - -func (tc *testConn) Close() error { - tc.closed = true - return nil -} - -type mockWatcher struct { - values chan string -} - -func newMockWatcher() *mockWatcher { - return &mockWatcher{ - values: make(chan string), - } -} - -func (mw mockWatcher) getReceiveChan() <-chan string { - return mw.values -} - -func (mw mockWatcher) sendValue(value string) { - log.Printf("mockWatcher: sending value '%s'...", value) - mw.values <- value - log.Printf("mockWatcher: sent value '%s'", value) -} - -var _ = DescribeTableSubtree("Driver", Serial, func(forceKill bool) { - var pctx context.Context - var ctx context.Context - var cancel context.CancelFunc - var cg *chanGroup - var mgdConns []*managedConn - var mockw *mockWatcher - Context("chanGroup", func() { - BeforeEach(func(ginkgoCtx context.Context) { - // Do NOT use ginkgoCtx as it will be canceled when BeforeEach finishes - pctx = context.Background() - ctx, cancel = context.WithCancel(pctx) - mockw = newMockWatcher() - cg = &chanGroup{ - name: "fsnotify://postgres/tmp/mydsn.txt", - value: "1st-dsn", - newValChan: mockw.getReceiveChan(), - parentCtx: pctx, - ctx: ctx, - cancel: cancel, - sqlDriver: nil, - mu: sync.RWMutex{}, - forceKill: forceKill, - } - cg.conns = []*managedConn{ - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - newManagedConn(ctx, cg.value, cg.value, &testConn{}, cg.removeMgdConn), - } - mgdConns = cg.conns - - metrics.ResetCollectors() - }, NodeTimeout(5*time.Second)) - - It("Should change value when a value is pushed to the values channel", func(ginkgoCtx context.Context) { - newVal := "2nd-dsn" - go cg.runLoop() - mockw.sendValue(newVal) - - // Yield to cg.runLoop() background thread - time.Sleep(200 * time.Millisecond) - - cg.mu.RLock() - defer cg.mu.RUnlock() - Expect(cg.value).To(Equal(newVal)) - - Expect(len(cg.conns)).To(Equal(0), "number of managed conns should be reset to zero") - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeTrue(), "managed connection should be marked reset") - if cg.forceKill { - Expect(mc.GetKill()).To(BeTrue(), "managed connection should be marked killed") - Expect(mc.conn.(*testConn).closed).To(BeTrue(), "Closed() should have been called on the underlying connection") - } - } - - // HotloadChangeTotal metric should be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader(expectHotloadChangeTotalHelp+ - fmt.Sprintf(expectHotloadChangeTotalMetric, cg.name, 1))) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - - It("Should not reset conns when the same value is pushed to the values channel", func(ginkgoCtx context.Context) { - sameVal := "1st-dsn" - go cg.runLoop() - mockw.sendValue(sameVal) - - // Yield to cg.runLoop() background thread - time.Sleep(200 * time.Millisecond) - - Expect(cg.value).To(Equal(sameVal)) - - Expect(len(cg.conns)).To(Equal(3), "number of managed conns should not be reset to zero") - - for _, c := range cg.conns { - Expect(c.GetReset()).To(BeFalse()) - Expect(c.GetKill()).To(BeFalse()) - Expect(c.conn.(*testConn).closed).To(BeFalse()) - } - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeFalse()) - Expect(mc.GetKill()).To(BeFalse()) - Expect(mc.conn.(*testConn).closed).To(BeFalse()) - } - - // HotloadChangeTotal metric should NOT be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader("")) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - - It("Should change value and reset connections", func(ginkgoCtx context.Context) { - newVal := "2nd-dsn" - cg.processNewValue(newVal) - Expect(cg.value).To(Equal(newVal)) - - Expect(len(cg.conns)).To(Equal(0), "number of managed conns should be reset to zero") - - for _, mc := range mgdConns { - Expect(mc.GetReset()).To(BeTrue(), "managed connection should be marked reset") - if cg.forceKill { - Expect(mc.GetKill()).To(BeTrue(), "managed connection should be marked killed") - Expect(mc.conn.(*testConn).closed).To(BeTrue(), "Closed() should have been called on the underlying connection") - } - } - - // HotloadChangeTotal metric should be incremented - err := testutil.CollectAndCompare(metrics.HotloadChangeTotal, - strings.NewReader(expectHotloadChangeTotalHelp+ - fmt.Sprintf(expectHotloadChangeTotalMetric, cg.name, 1))) - Expect(err).ShouldNot(HaveOccurred()) - - err = internal.CollectAndRegexpCompare(metrics.HotloadLastChangedTimestampSeconds, - strings.NewReader(expectHotloadLastChangedTimestampSecondsMetricRegexp), - metrics.HotloadLastChangedTimestampSecondsName) - Expect(err).ShouldNot(HaveOccurred()) - }, NodeTimeout(5*time.Second)) - }) -}, - Entry("forceKill=false", false), - Entry("forceKill=true", true), -) - -var expectHotloadChangeTotalHelp = ` -# HELP hotload_change_total Hotload change total by url -# TYPE hotload_change_total counter -` - -var expectHotloadChangeTotalMetric = ` -hotload_change_total{url="%s"} %d -` - -var expectHotloadLastChangedTimestampSecondsMetricRegexp = ` -# HELP hotload_last_changed_timestamp_seconds Hotload last changed \(unix timestamp\), by url -# TYPE hotload_last_changed_timestamp_seconds gauge -hotload_last_changed_timestamp_seconds{url="fsnotify://postgres/tmp/mydsn.txt"} \d\.\d+e\+\d+ -` diff --git a/concurrency_test.go b/concurrency_test.go new file mode 100644 index 0000000..55949c3 --- /dev/null +++ b/concurrency_test.go @@ -0,0 +1,153 @@ +package hotload_test + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// TestChangeStorm hammers the pool from many goroutines while config +// changes fire rapidly, in both graceful and forceKill modes. It asserts no +// deadlocks (the test finishes), no unexpected errors, convergence to the +// final DSN, and no goroutine leaks. +func TestChangeStorm(t *testing.T) { + for _, forceKill := range []bool{false, true} { + t.Run(fmt.Sprintf("forceKill=%v", forceKill), func(t *testing.T) { + testutil.NoLeaks(t) + + params := "killWindow=10ms" + if forceKill { + params = "forceKill=true&killWindow=10ms" + } + fx := newFixture(t, fxCfg{params: params}) + + const workers = 8 + const changes = 25 + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var ( + wg sync.WaitGroup + successes atomic.Int64 + unexpected sync.Map + stop = make(chan struct{}) + ) + + allowedErr := func(err error) bool { + // Under forceKill, in-flight work may be retired; the pool + // also surfaces bad-conn errors when retries are exhausted + // mid-storm. + return errors.Is(err, hotload.ErrHotSwap) || + errors.Is(err, context.Canceled) || + err.Error() == "driver: bad connection" + } + + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + var err error + if i%2 == 0 { + _, err = fx.db.ExecContext(ctx, "UPDATE x") + } else { + var dsn string + err = fx.db.QueryRowContext(ctx, "SELECT dsn").Scan(&dsn) + } + if err == nil { + successes.Add(1) + } else if !allowedErr(err) { + unexpected.Store(err.Error(), true) + } + } + }(i) + } + + final := "" + for j := 0; j < changes; j++ { + final = fmt.Sprintf("dsn-%d", j%3+2) + fx.pushAndWait(final) + } + + close(stop) + wg.Wait() + + unexpected.Range(func(k, v any) bool { + t.Errorf("unexpected error during storm: %s", k) + return true + }) + if successes.Load() == 0 { + t.Error("no operation succeeded during the storm") + } + testutil.WaitFor(t, 2*time.Second, "convergence to final DSN", func() bool { + return fx.queryDSN() == final + }) + }) + } +} + +// TestConcurrentCloseAndKill races pool-driven closes against +// generation kills; the underlying conn must be closed exactly once. +func TestConcurrentCloseAndKill(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{params: "forceKill=true&killWindow=5ms"}) + + for round := 0; round < 20; round++ { + fx.queryDSN() + // Race: db.Close-like pool eviction (via change) and explicit swap. + fx.pushAndWait(fmt.Sprintf("dsn-%d", round+2)) + } + + testutil.WaitFor(t, 2*time.Second, "all old conns closed", func() bool { + open := fx.drv.OpenConns() + return len(open) <= 1 + }) + for _, c := range fx.drv.Conns() { + if n := c.CloseCount(); n > 1 { + t.Errorf("conn %d closed %d times, want at most 1", c.ID, n) + } + } +} + +// TestQueryRowsSurviveGracefulChange: rows being iterated when a graceful +// change lands must remain readable — the reason QueryContext passes the +// caller context through unmerged. +func TestQueryRowsSurviveGracefulChange(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + rows, err := fx.db.Query("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + fx.pushAndWait("dsn-2") + + var dsn string + if !rows.Next() { + t.Fatalf("rows.Next() = false after graceful change: %v", rows.Err()) + } + if err := rows.Scan(&dsn); err != nil { + t.Fatal(err) + } + if dsn != "dsn-1" { + t.Errorf("row value = %q, want dsn-1 (rows belong to the old generation)", dsn) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/conn.go b/conn.go index 6aeb13c..e52438b 100644 --- a/conn.go +++ b/conn.go @@ -1,306 +1,286 @@ package hotload +//go:generate go run ./internal/gen + import ( "context" "database/sql" "database/sql/driver" "errors" - "fmt" - "sync" "sync/atomic" - - "github.com/infobloxopen/hotload/logger" - "github.com/teivah/onecontext" ) -// managedConn wraps a sql/driver.Conn so that it can be closed by -// a supervising context. -type managedConn struct { - ctx context.Context - dsn string - redactDsn string - conn driver.Conn - reset bool - killed bool - mu sync.RWMutex +// baseConn wraps an underlying driver.Conn so that it can be retired by the +// generation that owns it. baseConn implements driver.Conn plus the optional +// interfaces whose behavior hotload must control or whose stdlib fallback it +// can replicate exactly: ConnPrepareContext, ConnBeginTx, SessionResetter +// and Validator. The remaining optional interfaces (ExecerContext, +// QueryerContext, Pinger, NamedValueChecker) are exposed only when the +// underlying conn supports them, via the generated combination wrappers +// returned by wrapConn. +type baseConn struct { + inner driver.Conn + gen *generation + group *group + dsn string + redactDsn string + closed atomic.Bool + execStmts atomic.Int64 // exec statements since the last completed transaction + queryStmts atomic.Int64 // query statements since the last completed transaction +} - // callback function to be called after the connection is closed - afterClose func(*managedConn) +// opCtx returns a context canceled when either the caller's ctx or the +// owning generation's ctx is canceled. The returned release function must be +// called (usually deferred) when the operation completes; it unregisters the +// cancellation relay so no resources outlive the call. opCtx returns a nil +// context if the generation is already retired; the caller should return +// driver.ErrBadConn so database/sql retries on a fresh connection. +func (c *baseConn) opCtx(ctx context.Context) (context.Context, func()) { + gen := c.gen + gen.ops.Add(1) + if gen.ctx.Err() != nil { + gen.ops.Add(-1) + return nil, nil + } + mctx, cancel := context.WithCancelCause(ctx) + stop := context.AfterFunc(gen.ctx, func() { + cancel(context.Cause(gen.ctx)) + }) + release := func() { + stop() + cancel(nil) + gen.ops.Add(-1) + } + return mctx, release +} - execStmtsCounter atomic.Int64 // count the number of exec calls in a transaction - queryStmtsCounter atomic.Int64 // count the number of query calls in a transaction +// retired reports whether the conn should no longer be used: its generation +// was drained or killed, or the conn itself was closed. +func (c *baseConn) retired() bool { + return c.closed.Load() || c.gen.retired() } -// BeginTx calls the underlying BeginTx method unless the supervising context -// is closed. -// Returns an error if the underlying driver doesn't implement -// driver.ConnBeginTx interface and TxOptions are non default. If TxOptions are -// of default values it will call the underlying Begin method as like sql -// package. -// If the context is canceled by the user this method will call Tx.Rollback. -func (c *managedConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - select { - case <-c.ctx.Done(): - c.close() +func (c *baseConn) Prepare(query string) (driver.Stmt, error) { + if c.gen.ctx.Err() != nil { return nil, driver.ErrBadConn - default: } - - if conn, ok := c.conn.(driver.ConnBeginTx); ok { - tx, err := conn.BeginTx(ctx, opts) - if err != nil { - return nil, err - } - - return &managedTx{tx: tx, conn: c, ctx: ctx}, nil + stmt, err := c.inner.Prepare(query) + if err != nil { + return nil, err } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil +} - // same as is defined in go sql package to call Begin method if the TxOptions are default - if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { - return nil, errors.New("hotload: underlying driver does not support non-default isolation level") +// PrepareContext prepares a statement, honoring both the caller's context +// and the generation's context. When the underlying conn does not implement +// driver.ConnPrepareContext this replicates database/sql's fallback exactly: +// prepare without a context, then if the context is done, close the +// statement and return the context's error. +func (c *baseConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + octx, release := c.opCtx(ctx) + if octx == nil { + return nil, driver.ErrBadConn } + defer release() - if opts.ReadOnly { - return nil, errors.New("hotload: underlying driver does not support read-only transactions") + if cp, ok := c.inner.(driver.ConnPrepareContext); ok { + stmt, err := cp.PrepareContext(octx, query) + if err != nil { + return nil, err + } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil } - tx, err := c.conn.Begin() + stmt, err := c.inner.Prepare(query) if err == nil { select { + case <-octx.Done(): + stmt.Close() + return nil, octx.Err() default: - case <-ctx.Done(): - tx.Rollback() - return nil, ctx.Err() } } - - return tx, err + if err != nil { + return nil, err + } + return wrapStmt(&baseStmt{inner: stmt, conn: c}), nil } -func newManagedConn(ctx context.Context, dsn, redactDsn string, conn driver.Conn, afterClose func(*managedConn)) *managedConn { - return &managedConn{ - ctx: ctx, - dsn: dsn, - redactDsn: redactDsn, - conn: conn, - afterClose: afterClose, +func (c *baseConn) Begin() (driver.Tx, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn } + tx, err := c.inner.Begin() + if err != nil { + return nil, err + } + return &managedTx{tx: tx, conn: c, ctx: context.Background()}, nil } -func (c *managedConn) Exec(query string, args []driver.Value) (driver.Result, error) { - c.logf("managedConn.Exec", "Exec") +// BeginTx starts a transaction. The caller's context is deliberately passed +// through unmerged: some drivers bind the context to the transaction's +// lifetime, and canceling a merged context when this call returns would +// roll back live transactions. The generation context is checked at entry +// instead; forceKill reaches in-flight transactions by closing the conn. +// When the underlying conn does not implement driver.ConnBeginTx this +// replicates database/sql's fallback exactly. +func (c *baseConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn + } - connCtx, ok := c.conn.(driver.ExecerContext) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] + if cb, ok := c.inner.(driver.ConnBeginTx); ok { + tx, err := cb.BeginTx(ctx, opts) + if err != nil { + return nil, err } - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.Exec", "calling underlying conn.ExecContext()") - return connCtx.ExecContext(c.ctx, query, namedArgs) + return &managedTx{tx: tx, conn: c, ctx: ctx}, nil } - connExr, ok := c.conn.(driver.Execer) - if ok { - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.Exec", "calling underlying conn.Exec()") - return connExr.Exec(query, args) + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + return nil, errors.New("hotload: underlying driver does not support non-default isolation level") + } + if opts.ReadOnly { + return nil, errors.New("hotload: underlying driver does not support read-only transactions") } - return nil, driver.ErrSkip + tx, err := c.inner.Begin() + if err != nil { + return nil, err + } + select { + case <-ctx.Done(): + tx.Rollback() + return nil, ctx.Err() + default: + } + return &managedTx{tx: tx, conn: c, ctx: ctx}, nil } -func (c *managedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - c.logf("managedConn.ExecContext", "ExecContext") - conn, ok := c.conn.(driver.ExecerContext) - if !ok { - return nil, driver.ErrSkip - } - c.incExecStmtsCounter() //increment the exec counter to keep track of the number of exec calls - c.logf("managedConn.ExecContext", "calling underlying conn.ExecContext()") - mergedCtx, cancel := onecontext.Merge(c.ctx, ctx) - defer cancel() - return conn.ExecContext(mergedCtx, query, args) +func (c *baseConn) Close() error { + return c.closeConn(false) } -func (c *managedConn) CheckNamedValue(namedValue *driver.NamedValue) error { - conn, ok := c.conn.(driver.NamedValueChecker) - if !ok { - return driver.ErrSkip +// closeConn closes the underlying conn exactly once, no matter how many +// paths race to close it (the pool, a generation kill, or both). killed +// records in the emitted hook event whether the close was caused by a +// config change rather than the pool retiring the conn. +func (c *baseConn) closeConn(killed bool) error { + if !c.closed.CompareAndSwap(false, true) { + return nil } - return conn.CheckNamedValue(namedValue) + err := c.inner.Close() + c.gen.remove(c) + emitConnClose(ConnEvent{GroupName: c.group.name, RedactedDSN: c.redactDsn, Killed: killed}) + return err } -func (c *managedConn) Query(query string, args []driver.Value) (driver.Rows, error) { - c.logf("managedConn.Query", "Query") - - connCtx, ok := c.conn.(driver.QueryerContext) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] - } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.Query", "calling underlying conn.QueryContext()") - return connCtx.QueryContext(c.ctx, query, namedArgs) +// ResetSession is called by database/sql before reusing a pooled conn. A +// drained or killed generation answers driver.ErrBadConn so the pool +// discards the conn and dials a fresh one on the current DSN. +func (c *baseConn) ResetSession(ctx context.Context) error { + if c.retired() { + return driver.ErrBadConn } - - connQyr, ok := c.conn.(driver.Queryer) - if ok { - namedArgs := make([]driver.NamedValue, len(args), len(args)) - for i := 0; i < len(args); i++ { - namedArgs[i].Name = "" - namedArgs[i].Ordinal = i - namedArgs[i].Value = args[i] - } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.Query", "calling underlying conn.Query()") - return connQyr.Query(query, args) + if sr, ok := c.inner.(driver.SessionResetter); ok { + return sr.ResetSession(ctx) } - - return nil, driver.ErrSkip + return nil } -func (c *managedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - c.logf("managedConn.QueryContext", "QueryContext") - conn, ok := c.conn.(driver.QueryerContext) - if !ok { - return nil, driver.ErrSkip +// IsValid is called by database/sql when returning a conn to the pool. +func (c *baseConn) IsValid() bool { + if c.retired() { + return false } - c.incQueryStmtsCounter() //increment the query counter to keep track of the number of query calls - c.logf("managedConn.QueryContext", "calling underlying conn.QueryContext()") - - // TODO - // We would like to merge the hotload-context with the query-context here, - // and then cancel the merged-context to prevent goroutine-leaks - // (similar to ExecContext() above). - // However the Rows object returned seems to contain the merged-context. - // Canceling the merged-context here invalidates the returned Rows object, - // and causes any cursor iteration of the returned Rows objects to fail - // with context-canceled error. - - return conn.QueryContext(ctx, query, args) + if v, ok := c.inner.(driver.Validator); ok { + return v.IsValid() + } + return true } -func (c *managedConn) Prepare(query string) (driver.Stmt, error) { - select { - case <-c.ctx.Done(): - c.logf("managedConn.Prepare", "ctx done, calling close()") - c.close() +// execContext backs the generated ExecerContext wrappers. It is only +// reachable when the underlying conn implements ExecerContext or the legacy +// Execer; for the legacy case it replicates database/sql's fallback exactly +// (convert named args, poll the context, call Exec). +func (c *baseConn) execContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + octx, release := c.opCtx(ctx) + if octx == nil { return nil, driver.ErrBadConn - default: } - c.logf("managedConn.Prepare", "calling underlying Prepare()") - return c.conn.Prepare(query) -} + defer release() + c.execStmts.Add(1) -// Begin calls the underlying Begin method unless the supervising -// context is closed. -func (c *managedConn) Begin() (driver.Tx, error) { - select { - case <-c.ctx.Done(): - c.close() - return nil, driver.ErrBadConn - default: + if ec, ok := c.inner.(driver.ExecerContext); ok { + return ec.ExecContext(octx, query, args) } - return c.conn.Begin() -} -func (c *managedConn) IsValid() bool { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } select { - case <-c.ctx.Done(): - c.logf("managedConn.IsValid", "ctx done, calling close()") - c.close() - return false + case <-octx.Done(): + return nil, octx.Err() default: } - s, ok := c.conn.(driver.Validator) - if !ok { - return true - } - c.logf("managedConn.IsValid", "calling underlying IsValid()") - return s.IsValid() + return c.inner.(driver.Execer).Exec(query, dargs) } -func (c *managedConn) ResetSession(ctx context.Context) error { - if c.GetReset() { - c.logf("managedConn.ResetSession", "already reset") - return driver.ErrBadConn +// queryContext backs the generated QueryerContext wrappers. The caller's +// context is deliberately passed through unmerged: the returned driver.Rows +// captures the context, and canceling a merged context when this call +// returns would make cursor iteration fail with a context-canceled error. +// The generation context is checked at entry instead; forceKill reaches +// in-flight queries by closing the conn. +func (c *baseConn) queryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if c.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn } + c.queryStmts.Add(1) - s, ok := c.conn.(driver.SessionResetter) - if !ok { - return nil + if qc, ok := c.inner.(driver.QueryerContext); ok { + return qc.QueryContext(ctx, query, args) } - c.logf("managedConn.ResetSession", "calling underlying ResetSession()") - return s.ResetSession(ctx) -} - -func (c *managedConn) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - err := c.close() - - if err == nil { - c.killed = true + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err } - c.logf("managedConn.Close", "closed") - - return err -} - -func (c *managedConn) close() error { - if c.afterClose != nil { - defer c.afterClose(c) + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: } - c.logf("managedConn.close", "calling underlying Close()") - return c.conn.Close() -} - -func (c *managedConn) GetReset() bool { - c.mu.RLock() - defer c.mu.RUnlock() - c.logf("managedConn.GetReset", "reset=%v", c.reset) - return c.reset -} - -func (c *managedConn) Reset(v bool) { - c.mu.Lock() - defer c.mu.Unlock() - c.reset = v - c.logf("managedConn.Reset", "reset=%v", v) -} - -func (c *managedConn) GetKill() bool { - c.mu.RLock() - defer c.mu.RUnlock() - c.logf("managedConn.GetKill", "killed=%v", c.killed) - return c.killed + return c.inner.(driver.Queryer).Query(query, dargs) } -func (c *managedConn) incExecStmtsCounter() { - c.execStmtsCounter.Add(1) -} - -func (c *managedConn) resetExecStmtsCounter() { - c.execStmtsCounter.Store(0) -} - -func (c *managedConn) incQueryStmtsCounter() { - c.queryStmtsCounter.Add(1) +// ping backs the generated Pinger wrappers; only reachable when the +// underlying conn implements driver.Pinger. +func (c *baseConn) ping(ctx context.Context) error { + octx, release := c.opCtx(ctx) + if octx == nil { + return driver.ErrBadConn + } + defer release() + return c.inner.(driver.Pinger).Ping(octx) } -func (c *managedConn) resetQueryStmtsCounter() { - c.queryStmtsCounter.Store(0) +// checkNamedValue backs the generated NamedValueChecker wrappers; only +// reachable when the underlying conn implements driver.NamedValueChecker. +func (c *baseConn) checkNamedValue(nv *driver.NamedValue) error { + return c.inner.(driver.NamedValueChecker).CheckNamedValue(nv) } -func (c *managedConn) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, c.redactDsn) - logger.Logf(logPrefix, format, args...) +// namedValueToValue converts named args to positional args, mirroring the +// unexported helper of the same name in database/sql. +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + return nil, errors.New("sql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil } diff --git a/conn_combos_gen.go b/conn_combos_gen.go new file mode 100644 index 0000000..d240b1d --- /dev/null +++ b/conn_combos_gen.go @@ -0,0 +1,253 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package hotload + +import "database/sql/driver" + +// wrapConn wraps b so that the returned driver.Conn exposes an optional +// interface if and only if the underlying conn supports the capability. +// The legacy Execer/Queryer interfaces are collapsed into their context +// flavors: the wrapper only ever exposes ExecerContext/QueryerContext and +// replicates database/sql's legacy fallback internally (see +// baseConn.execContext and baseConn.queryContext). +func connFlags(c driver.Conn) uint8 { + var f uint8 + if _, ok := c.(driver.ExecerContext); ok { + f |= 1 << 0 + } else if _, ok := c.(driver.Execer); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 0 + } + if _, ok := c.(driver.QueryerContext); ok { + f |= 1 << 1 + } else if _, ok := c.(driver.Queryer); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 1 + } + if _, ok := c.(driver.Pinger); ok { + f |= 1 << 2 + } + if _, ok := c.(driver.NamedValueChecker); ok { + f |= 1 << 3 + } + return f +} + +type conn_E struct { + *baseConn + cExecer +} + +var ( + _ driver.Conn = conn_E{} + _ driver.ExecerContext = conn_E{} +) + +type conn_Q struct { + *baseConn + cQueryer +} + +var ( + _ driver.Conn = conn_Q{} + _ driver.QueryerContext = conn_Q{} +) + +type conn_EQ struct { + *baseConn + cExecer + cQueryer +} + +var ( + _ driver.Conn = conn_EQ{} + _ driver.ExecerContext = conn_EQ{} + _ driver.QueryerContext = conn_EQ{} +) + +type conn_P struct { + *baseConn + cPinger +} + +var ( + _ driver.Conn = conn_P{} + _ driver.Pinger = conn_P{} +) + +type conn_EP struct { + *baseConn + cExecer + cPinger +} + +var ( + _ driver.Conn = conn_EP{} + _ driver.ExecerContext = conn_EP{} + _ driver.Pinger = conn_EP{} +) + +type conn_QP struct { + *baseConn + cQueryer + cPinger +} + +var ( + _ driver.Conn = conn_QP{} + _ driver.QueryerContext = conn_QP{} + _ driver.Pinger = conn_QP{} +) + +type conn_EQP struct { + *baseConn + cExecer + cQueryer + cPinger +} + +var ( + _ driver.Conn = conn_EQP{} + _ driver.ExecerContext = conn_EQP{} + _ driver.QueryerContext = conn_EQP{} + _ driver.Pinger = conn_EQP{} +) + +type conn_N struct { + *baseConn + cNVChecker +} + +var ( + _ driver.Conn = conn_N{} + _ driver.NamedValueChecker = conn_N{} +) + +type conn_EN struct { + *baseConn + cExecer + cNVChecker +} + +var ( + _ driver.Conn = conn_EN{} + _ driver.ExecerContext = conn_EN{} + _ driver.NamedValueChecker = conn_EN{} +) + +type conn_QN struct { + *baseConn + cQueryer + cNVChecker +} + +var ( + _ driver.Conn = conn_QN{} + _ driver.QueryerContext = conn_QN{} + _ driver.NamedValueChecker = conn_QN{} +) + +type conn_EQN struct { + *baseConn + cExecer + cQueryer + cNVChecker +} + +var ( + _ driver.Conn = conn_EQN{} + _ driver.ExecerContext = conn_EQN{} + _ driver.QueryerContext = conn_EQN{} + _ driver.NamedValueChecker = conn_EQN{} +) + +type conn_PN struct { + *baseConn + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_PN{} + _ driver.Pinger = conn_PN{} + _ driver.NamedValueChecker = conn_PN{} +) + +type conn_EPN struct { + *baseConn + cExecer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_EPN{} + _ driver.ExecerContext = conn_EPN{} + _ driver.Pinger = conn_EPN{} + _ driver.NamedValueChecker = conn_EPN{} +) + +type conn_QPN struct { + *baseConn + cQueryer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_QPN{} + _ driver.QueryerContext = conn_QPN{} + _ driver.Pinger = conn_QPN{} + _ driver.NamedValueChecker = conn_QPN{} +) + +type conn_EQPN struct { + *baseConn + cExecer + cQueryer + cPinger + cNVChecker +} + +var ( + _ driver.Conn = conn_EQPN{} + _ driver.ExecerContext = conn_EQPN{} + _ driver.QueryerContext = conn_EQPN{} + _ driver.Pinger = conn_EQPN{} + _ driver.NamedValueChecker = conn_EQPN{} +) + +func wrapConn(b *baseConn) driver.Conn { + switch connFlags(b.inner) { + case 1: + return conn_E{b, cExecer{b}} + case 2: + return conn_Q{b, cQueryer{b}} + case 3: + return conn_EQ{b, cExecer{b}, cQueryer{b}} + case 4: + return conn_P{b, cPinger{b}} + case 5: + return conn_EP{b, cExecer{b}, cPinger{b}} + case 6: + return conn_QP{b, cQueryer{b}, cPinger{b}} + case 7: + return conn_EQP{b, cExecer{b}, cQueryer{b}, cPinger{b}} + case 8: + return conn_N{b, cNVChecker{b}} + case 9: + return conn_EN{b, cExecer{b}, cNVChecker{b}} + case 10: + return conn_QN{b, cQueryer{b}, cNVChecker{b}} + case 11: + return conn_EQN{b, cExecer{b}, cQueryer{b}, cNVChecker{b}} + case 12: + return conn_PN{b, cPinger{b}, cNVChecker{b}} + case 13: + return conn_EPN{b, cExecer{b}, cPinger{b}, cNVChecker{b}} + case 14: + return conn_QPN{b, cQueryer{b}, cPinger{b}, cNVChecker{b}} + case 15: + return conn_EQPN{b, cExecer{b}, cQueryer{b}, cPinger{b}, cNVChecker{b}} + default: + return b + } +} diff --git a/conn_pieces.go b/conn_pieces.go new file mode 100644 index 0000000..11fdfff --- /dev/null +++ b/conn_pieces.go @@ -0,0 +1,40 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// The piece types below each carry exactly one optional driver.Conn method. +// The generated combination wrappers in conn_combos_gen.go embed a subset of +// them, so a wrapped conn's method set — and therefore the type assertions +// database/sql performs — reflects exactly what the underlying conn +// supports. + +// cExecer carries ExecContext (driver.ExecerContext). +type cExecer struct{ b *baseConn } + +func (p cExecer) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return p.b.execContext(ctx, query, args) +} + +// cQueryer carries QueryContext (driver.QueryerContext). +type cQueryer struct{ b *baseConn } + +func (p cQueryer) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return p.b.queryContext(ctx, query, args) +} + +// cPinger carries Ping (driver.Pinger). +type cPinger struct{ b *baseConn } + +func (p cPinger) Ping(ctx context.Context) error { + return p.b.ping(ctx) +} + +// cNVChecker carries CheckNamedValue (driver.NamedValueChecker). +type cNVChecker struct{ b *baseConn } + +func (p cNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return p.b.checkNamedValue(nv) +} diff --git a/conn_test.go b/conn_test.go deleted file mode 100644 index 33d3c31..0000000 --- a/conn_test.go +++ /dev/null @@ -1,472 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - "io" - "runtime" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus/testutil" - - "github.com/infobloxopen/hotload/metrics" -) - -var _ = Describe("managedConn", func() { - It("Should set .reset in a threadsafe way", func() { - mc := managedConn{ - ctx: nil, - conn: nil, - reset: false, - mu: sync.RWMutex{}, - } - // Lock the mutex - mc.mu.Lock() - writeLockAcquired := false - readLockAcquired := false - - // Verify that neither Reset or GetReset can return while the managedConn's write lock is held - go func() { - mc.Reset(true) - writeLockAcquired = true - }() - - go func() { - mc.GetReset() - readLockAcquired = true - }() - - Consistently(writeLockAcquired).Should(BeFalse()) - Consistently(readLockAcquired).Should(BeFalse()) - }) - - It("Should not leak goroutines when using ExecContext and QueryContext", func() { - // Force garbage collection and get baseline - runtime.GC() - runtime.GC() - time.Sleep(50 * time.Millisecond) - initialGoroutines := runtime.NumGoroutine() - - // Create a long-lived parent context that won't be cancelled - parentCtx := context.Background() // This simulates a long-lived connection context - - // Create managed connection - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create many operations with different child contexts - // This will trigger onecontext.Merge calls, and without defer mCancel(), - // goroutines will leak because parentCtx is never cancelled - numOperations := 50 - - for i := 0; i < numOperations; i++ { - // Create short-lived contexts for each operation - childCtx := context.Background() // Also long-lived for this test - - // Each call to ExecContext will create a merged context internally - // Without defer mCancel(), the goroutine will wait indefinitely - mc.ExecContext(childCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.QueryContext(childCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - } - - // Close the connection - err := mc.Close() - Expect(err).ShouldNot(HaveOccurred()) - - // Give time for any leaked goroutines to be detected - time.Sleep(200 * time.Millisecond) - runtime.GC() - runtime.GC() - - // Check that we don't have significantly more goroutines than we started with - finalGoroutines := runtime.NumGoroutine() - - // With the leak, we expect to see many leaked goroutines from onecontext.Merge - // Each ExecContext and QueryContext call that doesn't call defer mCancel() will leak a goroutine - expectedLeaks := numOperations * 2 // One for each ExecContext and QueryContext call - - Expect(finalGoroutines).To(BeNumerically("<=", initialGoroutines+10), - "Expected no significant goroutine leaks. Initial: %d, Final: %d, Expected leaks if broken: ~%d", - initialGoroutines, finalGoroutines, expectedLeaks) - }) - - It("Should handle context cancellation properly in ExecContext", func() { - parentCtx, parentCancel := context.WithCancel(context.Background()) - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create a context that's already cancelled - cancelledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - // ExecContext should still work even with cancelled child context - // because the parent context is still active - _, err := mc.ExecContext(cancelledCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - Expect(err).ShouldNot(HaveOccurred()) - - // Now cancel the parent context - parentCancel() - - // Give some time for context cancellation to propagate - time.Sleep(10 * time.Millisecond) - - // This should fail because parent context is cancelled - mc.ExecContext(context.Background(), "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - // The mock doesn't return context.Canceled, but the connection should work - // The important thing is that no goroutines are leaked - }) - - It("Should not leak goroutines with slow operations and context cancellation", func() { - // Force garbage collection and get very precise baseline - runtime.GC() - runtime.GC() - time.Sleep(100 * time.Millisecond) - runtime.GC() - initialGoroutines := runtime.NumGoroutine() - - // Create contexts that will be cancelled during operations - parentCtx, parentCancel := context.WithCancel(context.Background()) - defer parentCancel() - - // Create managed connection with the slow mock to simulate real database behavior - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockSlowDriverConn{}, nil) - - // Create many concurrent operations that will force onecontext.Merge goroutine creation - var wg sync.WaitGroup - numOperations := 100 // Much higher number to force obvious leaks - - for i := 0; i < numOperations; i++ { - wg.Add(2) - - go func(iteration int) { - defer wg.Done() - // Create a unique context for each operation to force new onecontext.Merge calls - timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer timeoutCancel() - - // This should timeout and without defer mCancel(), leak goroutines - mc.ExecContext(timeoutCtx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - }(i) - - go func(iteration int) { - defer wg.Done() - // Create a unique context for each operation to force new onecontext.Merge calls - timeoutCtx, timeoutCancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer timeoutCancel() - - // This should timeout and without defer mCancel(), leak goroutines - mc.QueryContext(timeoutCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - }(i) - } - - wg.Wait() - - // Close the connection - err := mc.Close() - Expect(err).ShouldNot(HaveOccurred()) - - // Give substantial time for leaked goroutines to accumulate - time.Sleep(500 * time.Millisecond) - runtime.GC() - runtime.GC() - - // Check that we don't have significantly more goroutines than we started with - finalGoroutines := runtime.NumGoroutine() - - // This should definitely fail if there are goroutine leaks from onecontext.Merge - // Being very strict about detecting leaks - Expect(finalGoroutines).To(BeNumerically("<=", initialGoroutines+10), - "Expected no significant goroutine leaks from onecontext.Merge. Initial: %d, Final: %d, Diff: %d", - initialGoroutines, finalGoroutines, finalGoroutines-initialGoroutines) - }) - - It("Should handle context cancellation properly in QueryContext", func() { - parentCtx, parentCancel := context.WithCancel(context.Background()) - mc := newManagedConn(parentCtx, "dsn", "redactDsn", mockDriverConn{}, nil) - - // Create a context that's already cancelled - cancelledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - // QueryContext should still work even with cancelled child context - // because the parent context is still active - _, err := mc.QueryContext(cancelledCtx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - Expect(err).ShouldNot(HaveOccurred()) - - // Now cancel the parent context - parentCancel() - - // Give some time for context cancellation to propagate - time.Sleep(10 * time.Millisecond) - - // This should work but no goroutines should be leaked - mc.QueryContext(context.Background(), "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - // The mock doesn't return context.Canceled, but the connection should work - // The important thing is that no goroutines are leaked - }) -}) - -/**** Mocks for Prometheus Metrics ****/ - -type mockDriverConn struct{} - -type mockTx struct{} - -// mockSlowDriverConn simulates a driver that might have slow operations -// This helps test for goroutine leaks in scenarios where context cancellation matters -type mockSlowDriverConn struct{} - -func (mockTx) Commit() error { - return nil -} - -func (mockTx) Rollback() error { - return nil -} - -func (mockDriverConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} - -func (mockDriverConn) Begin() (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockDriverConn) Close() error { - return nil -} - -func (mockDriverConn) IsValid() bool { - return true -} - -func (mockDriverConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockDriverConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return nil, nil -} - -func (mockDriverConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return nil, nil -} - -func (mockDriverConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - return nil, nil -} - -func (mockDriverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - return nil, nil -} - -// mockSlowDriverConn methods - simulates operations that might be slow -func (mockSlowDriverConn) Prepare(query string) (driver.Stmt, error) { - return nil, nil -} - -func (mockSlowDriverConn) Begin() (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockSlowDriverConn) Close() error { - return nil -} - -func (mockSlowDriverConn) IsValid() bool { - return true -} - -func (mockSlowDriverConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - return mockTx{}, nil -} - -func (mockSlowDriverConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return nil, nil -} - -func (mockSlowDriverConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return nil, nil -} - -func (mockSlowDriverConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - // Check if context is cancelled before proceeding - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - // Simulate a slow operation that keeps goroutines alive longer - timer := time.NewTimer(200 * time.Millisecond) - defer timer.Stop() - select { - case <-timer.C: - return nil, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (mockSlowDriverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - // Check if context is cancelled before proceeding - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - // Simulate a slow operation that keeps goroutines alive longer - timer := time.NewTimer(200 * time.Millisecond) - defer timer.Stop() - select { - case <-timer.C: - return nil, nil - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -/**** End Mocks for Prometheus Metrics ****/ - -var _ = Describe("PrometheusMetrics", func() { - const help = ` - # HELP transaction_sql_stmts The number of sql stmts called in a transaction by statement type per grpc service and method - # TYPE transaction_sql_stmts summary - ` - - var service1Metrics = ` - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 3 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="query"} 3 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="query"} 1 - ` - - var service2Metrics = ` - transaction_sql_stmts_sum{grpc_method="method_2",grpc_service="service_2",stmt="exec"} 4 - transaction_sql_stmts_count{grpc_method="method_2",grpc_service="service_2",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="method_2",grpc_service="service_2",stmt="query"} 4 - transaction_sql_stmts_count{grpc_method="method_2",grpc_service="service_2",stmt="query"} 1 - ` - - var service1RerunMetrics = ` - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 4 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="exec"} 2 - transaction_sql_stmts_sum{grpc_method="method_1",grpc_service="service_1",stmt="query"} 4 - transaction_sql_stmts_count{grpc_method="method_1",grpc_service="service_1",stmt="query"} 2 - ` - - var noMethodMetrics = ` - transaction_sql_stmts_sum{grpc_method="",grpc_service="",stmt="exec"} 1 - transaction_sql_stmts_count{grpc_method="",grpc_service="",stmt="exec"} 1 - transaction_sql_stmts_sum{grpc_method="",grpc_service="",stmt="query"} 1 - transaction_sql_stmts_count{grpc_method="",grpc_service="",stmt="query"} 1 - ` - - It("Should emit the correct metrics", func() { - mc := newManagedConn(context.Background(), "dsn", "redactDsn", mockDriverConn{}, nil) - - ctx := ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_1", "grpc_service": "service_1"}) - - // begin a transaction - tx, err := mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - - // query a statement - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // reset the metrics - // new context - ctx = ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_2", "grpc_service": "service_2"}) - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - mc.Exec("INSERT INTO table (column) VALUES (?)", []driver.Value{"value"}) - - // query a statement - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - mc.Query("SELECT * FROM table WHERE column = ?", []driver.Value{"value"}) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1Metrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // rerun with initial metrics - ctx = ContextWithExecLabels(context.Background(), map[string]string{"grpc_method": "method_1", "grpc_service": "service_1"}) - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec a statement with context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // rollback the transaction - err = tx.Rollback() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+service1RerunMetrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - - // non labeled context - ctx = context.Background() - // begin a transaction - tx, err = mc.BeginTx(ctx, driver.TxOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - // exec query context - mc.ExecContext(ctx, "INSERT INTO table (column) VALUES (?)", []driver.NamedValue{{Value: "value"}}) - - // query a statement with context - mc.QueryContext(ctx, "SELECT * FROM table WHERE column = ?", []driver.NamedValue{{Value: "value"}}) - - // commit the transaction - err = tx.Commit() - Expect(err).ShouldNot(HaveOccurred()) - - // collect and compare metrics - err = testutil.CollectAndCompare(metrics.SqlStmtsSummary, strings.NewReader(help+noMethodMetrics+service1RerunMetrics+service2Metrics)) - Expect(err).ShouldNot(HaveOccurred()) - }) -}) - -func CollectAndCompareMetrics(r io.Reader) error { - return testutil.CollectAndCompare(metrics.SqlStmtsSummary, r) -} diff --git a/connector.go b/connector.go new file mode 100644 index 0000000..4254724 --- /dev/null +++ b/connector.go @@ -0,0 +1,43 @@ +package hotload + +import ( + "context" + "database/sql/driver" + "fmt" + "sync/atomic" +) + +// connector ties a sql.DB to a group. database/sql obtains one per sql.Open +// call (through driver.DriverContext) and closes it when the sql.DB is +// closed, which releases the group reference and — once the last reference +// is gone — stops the strategy watch and the group's run loop. +type connector struct { + h *hdriver + g *group + name string + closed atomic.Bool +} + +var ( + _ driver.Connector = (*connector)(nil) +) + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + if c.closed.Load() { + return nil, fmt.Errorf("hotload: connector for %q is closed", c.name) + } + return c.g.conn(ctx) +} + +func (c *connector) Driver() driver.Driver { + return c.h +} + +// Close implements io.Closer; database/sql calls it from DB.Close. +func (c *connector) Close() error { + if !c.closed.CompareAndSwap(false, true) { + return nil + } + c.h.releaseGroup(c.name) + return nil +} diff --git a/context.go b/context.go new file mode 100644 index 0000000..93f070c --- /dev/null +++ b/context.go @@ -0,0 +1,27 @@ +package hotload + +import "context" + +type execLabelKeyType struct{} + +var execLabelKey = execLabelKeyType{} + +// ContextWithExecLabels returns a context carrying labels that describe the +// caller (for example a gRPC service and method). Observability adapters can +// retrieve them from TxEvent.Ctx with GetExecLabelsFromContext. +func ContextWithExecLabels(ctx context.Context, labels map[string]string) context.Context { + if labels == nil { + return ctx + } + return context.WithValue(ctx, execLabelKey, labels) +} + +// GetExecLabelsFromContext returns the labels stored by +// ContextWithExecLabels, or nil if there are none. +func GetExecLabelsFromContext(ctx context.Context) map[string]string { + if ctx == nil { + return nil + } + labels, _ := ctx.Value(execLabelKey).(map[string]string) + return labels +} diff --git a/driver.go b/driver.go index 7c08578..6db6a32 100644 --- a/driver.go +++ b/driver.go @@ -1,6 +1,7 @@ -// Package hotload is a database/sql driver that dynamically loads connection strings for other -// database drivers. To use it, import it like any other database driver and register -// the real database driver you want to use with hotload. +// Package hotload is a database/sql driver that dynamically loads connection +// strings for other database drivers. To use it, import it like any other +// database driver and register the real database driver you want to use with +// hotload. // // import ( // // import the std lib sql package @@ -9,10 +10,10 @@ // log "github.com/sirupsen/logrus" // // // this import registers hotload with the sql package -// "github.com/infobloxopen/hotload" +// "github.com/infobloxopen/hotload/v3" // // // this import registers the fsnotify hotload strategy -// _ "github.com/infobloxopen/hotload/fsnotify" +// _ "github.com/infobloxopen/hotload/v3/fsnotify" // // // this import registers the postgres driver with the sql package // "github.com/lib/pq" @@ -37,11 +38,13 @@ // * registers the lib/pq postgres driver with database/sql // * registers the lib/pq postgres driver with hotload // -// Then in the main() function the sql.Open call uses the hotload driver. The URL for the -// connection string specifies fsnotify in the scheme. This is the hotload strategy. The -// hostname in the URL specifies the real database driver. Finally the path and query parameters -// are left for the hotload strategy plugin to configure themselves. Below is an example -// of a lib/pq postgres connection string that would have been stored at /tmp/myconfig.txt +// Then in the main() function the sql.Open call uses the hotload driver. The +// URL for the connection string specifies fsnotify in the scheme. This is +// the hotload strategy. The hostname in the URL specifies the real database +// driver. Finally the path and query parameters are left for the hotload +// strategy plugin to configure themselves. Below is an example of a lib/pq +// postgres connection string that would have been stored at +// /tmp/myconfig.txt // // user=pqgotest dbname=pqgotest sslmode=verify-full package hotload @@ -56,33 +59,40 @@ import ( "sync" "time" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" + "github.com/infobloxopen/hotload/v3/logger" ) -// Strategy is the plugin interface for hotload. +// Strategy is the plugin interface for hotload: given a resource, watch it +// and stream its values. type Strategy interface { - // Watch returns back the contents of the resource as well as a channel - // for subsequent updates (if the value has changed). If there is an error - // getting the initial value, an error is returned. - Watch(ctx context.Context, pth string, pathQry string) (value string, newValChan <-chan string, err error) - - // CloseWatch closes the specified watch. - CloseWatch(pth string, pathQry string) error + // Watch begins watching the resource identified by pth (pathQry carries + // the hotload DSN's encoded query parameters). It returns the resource's + // current value and a Watchable streaming subsequent values. Each call + // establishes an independent watch, even for a path and query already + // being watched. The watch lives until its Watchable is closed or ctx + // is canceled. If the current value cannot be obtained, an error is + // returned and nothing is watched. + Watch(ctx context.Context, pth string, pathQry string) (value string, watch Watchable, err error) +} - // Close resets/closes strategy, in particular closes all the update channels. - Close() +// Watchable is one active watch established by Strategy.Watch. Values may +// carry secrets — they must not be logged unredacted. +type Watchable interface { + // Values returns the channel on which changed values of the watched + // resource are delivered. The strategy closes the channel when the + // watch ends. + Values() <-chan string + + // Close releases the watch: the strategy frees the resources backing it + // and closes the Values channel. Close is idempotent. It must not call + // back into hotload (the core calls it while holding internal locks). + Close() error } -const forceKill = "forceKill" -const driverOptions = "driverOptions" +const forceKillParam = "forceKill" +const killWindowParam = "killWindow" var ( - ErrUnsupportedStrategy = fmt.Errorf("unsupported hotload strategy") - ErrMalformedConnectionString = fmt.Errorf("malformed hotload connection string") - ErrUnknownDriver = fmt.Errorf("target driver is not registered with hotload") - mu sync.RWMutex sqlDrivers = make(map[string]*driverInstance) strategies = make(map[string]Strategy) @@ -95,9 +105,10 @@ type driverInstance struct { type driverOption func(*driverInstance) -// WithDriverOptions allows you to specify query parameters to the underlying driver. -// The underlying driver must support URL style connection strings. The given options -// are appended to the connection string when a connection is opened. +// WithDriverOptions allows you to specify query parameters to the underlying +// driver. The underlying driver must support URL style connection strings. +// The given options are appended to the connection string when a connection +// is opened. func WithDriverOptions(options map[string]string) driverOption { return func(d *driverInstance) { if d.options == nil { @@ -110,8 +121,8 @@ func WithDriverOptions(options map[string]string) driverOption { } // RegisterSQLDriver makes a database driver available by the provided name. -// If RegisterSQLDriver is called twice with the same name or if driver is nil, -// it panics. +// If RegisterSQLDriver is called twice with the same name or if driver is +// nil, it panics. func RegisterSQLDriver(name string, driver driver.Driver, options ...driverOption) { mu.Lock() defer mu.Unlock() @@ -125,18 +136,9 @@ func RegisterSQLDriver(name string, driver driver.Driver, options ...driverOptio for _, opt := range options { opt(di) } - sqlDrivers[name] = di } -func unregisterAll() { - mu.Lock() - defer mu.Unlock() - // For tests. - sqlDrivers = make(map[string]*driverInstance) - strategies = make(map[string]Strategy) -} - // SQLDrivers returns a sorted list of the names of the registered drivers. func SQLDrivers() []string { mu.RLock() @@ -149,9 +151,9 @@ func SQLDrivers() []string { return list } -// RegisterStrategy makes a database driver available by the provided name. -// If RegisterStrategy is called twice with the same name or if strategy is nil, -// it panics. +// RegisterStrategy makes a hotload strategy available by the provided name. +// If RegisterStrategy is called twice with the same name or if strategy is +// nil, it panics. func RegisterStrategy(name string, strategy Strategy) { mu.Lock() defer mu.Unlock() @@ -165,21 +167,17 @@ func RegisterStrategy(name string, strategy Strategy) { } // UnregisterStrategy unregisters the named driver strategy. -// Does nothing if strategy does not exist. -// Intended for internal unit-testing. +// Does nothing if strategy does not exist. Watches already established +// through the strategy are unaffected; they end when their groups close +// them. Intended for internal unit-testing. func UnregisterStrategy(name string) { mu.Lock() defer mu.Unlock() - strategy, ok := strategies[name] - if ok { - if strategy != nil { - strategy.Close() - } - delete(strategies, name) - } + delete(strategies, name) } -// Strategies returns a sorted list of the names of the registered drivers. +// Strategies returns a sorted list of the names of the registered +// strategies. func Strategies() []string { mu.RLock() defer mu.RUnlock() @@ -192,305 +190,189 @@ func Strategies() []string { } func init() { - ctx := context.Background() - sql.Register("hotload", &hdriver{ + sql.Register("hotload", newHdriver(context.Background())) +} + +func newHdriver(ctx context.Context) *hdriver { + return &hdriver{ ctx: ctx, - cgroup: make(map[string]*chanGroup), - }) + groups: make(map[string]*group), + } } // hdriver is the hotload driver. type hdriver struct { ctx context.Context - cgroup map[string]*chanGroup mu sync.Mutex -} + groups map[string]*group -// chanGroup represents a hotload location that is being monitored -type chanGroup struct { - name string - value string - redactVal string - newValChan <-chan string - parentCtx context.Context - ctx context.Context - cancel context.CancelFunc - sqlDriver *driverInstance - mu sync.RWMutex - forceKill bool - conns []*managedConn - prevCancel context.CancelFunc - prevRedactVal string - prevConns []*managedConn + // hooksNote emits the one-time no-hooks notice (see warnIfNoHooks). + hooksNote sync.Once } -// monitor the location for changes -func (cg *chanGroup) runLoop() { - for { - cg.logf("chanGroup.runLoop", "select waiting...") - select { - case <-cg.parentCtx.Done(): - cg.cancel() - cg.logf("chanGroup.runLoop", "parent context done, canceled chanGroup context, terminating") - return - - case newValue, ok := <-cg.newValChan: - if !ok { - cg.logf("chanGroup.runLoop", "newValChan closed, terminating") - return - } - cg.processNewValue(newValue) - } - } -} +var ( + _ driver.Driver = (*hdriver)(nil) + _ driver.DriverContext = (*hdriver)(nil) +) -func (cg *chanGroup) processNewValue(newValue string) { - type oldInfo struct { - changedFlag bool - prevPrevCancel context.CancelFunc - prevPrevRedactVal string - prevPrevConns []*managedConn - prevCancel context.CancelFunc - prevRedactVal string - prevConns []*managedConn +// Open implements the legacy driver.Driver dial path. Groups created here +// are pinned: they have no teardown signal, so their strategy watch and run +// loop live for the life of the process (as in hotload v1). Prefer the +// connector path (database/sql uses it automatically), which tears the +// group down when the last sql.DB using it is closed. +func (h *hdriver) Open(name string) (driver.Conn, error) { + g, err := h.getGroup(name, true) + if err != nil { + return nil, err } + return g.conn(context.Background()) +} - criticalSection := func() oldInfo { - cg.mu.Lock() - defer cg.mu.Unlock() - - prevValue := cg.value - prevRedactVal := cg.redactVal - - newRedactVal := internal.RedactUrl(newValue) - cg.logf("chanGroup.processNewValue", "old conn dsn: '%s'", prevRedactVal) - cg.logf("chanGroup.processNewValue", "new conn dsn: '%s'", newRedactVal) - - if newValue == prevValue { - // next update is the same, just ignore it - cg.logf("chanGroup.processNewValue", "conn dsn not changed") - return oldInfo{} - } - cg.logf("chanGroup.processNewValue", "conn dsn changed") - - result := oldInfo{ - changedFlag: true, - prevPrevConns: cg.prevConns, - prevPrevCancel: cg.prevCancel, - prevPrevRedactVal: cg.prevRedactVal, - } - - // Prepare shallow copy of existing connections, - // and reset new connections to zero - cg.prevConns = cg.conns - cg.conns = make([]*managedConn, 0) - - // Prepare copy of existing cancel ctx fn, - // and reset to new cancelable ctx - cg.prevCancel = cg.cancel - cg.ctx, cg.cancel = context.WithCancel(cg.parentCtx) - - // Prepare copy of existing value, - // and reset to new value - cg.prevRedactVal = cg.redactVal - cg.value = newValue - cg.redactVal = newRedactVal - - result.prevConns = cg.prevConns - result.prevCancel = cg.prevCancel - result.prevRedactVal = cg.prevRedactVal - - return result +// OpenConnector implements driver.DriverContext. The returned connector +// holds a reference on the group; database/sql closes the connector when +// the sql.DB is closed, and the group shuts down when its last reference is +// released. +func (h *hdriver) OpenConnector(name string) (driver.Connector, error) { + g, err := h.getGroup(name, false) + if err != nil { + return nil, err } + return &connector{h: h, g: g, name: name}, nil +} - prev := criticalSection() - if !prev.changedFlag { - return +// getGroup returns the group watching name, creating it (and its strategy +// watch) on first use. Groups are shared across sql.DB handles opened with +// the same DSN. +func (h *hdriver) getGroup(name string, pin bool) (*group, error) { + uri, err := url.Parse(name) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrMalformedConnectionString, err) } - // Mutex MUST be unlocked at this point before continuing - - // Update metrics - metrics.IncHotloadChangeTotal(cg.name) - metrics.SetHotloadLastChangedTimestampSeconds(cg.name, float64(time.Now().Unix())) + h.mu.Lock() + defer h.mu.Unlock() - // Canceling previous ctx can potentially cause other threads - // to call managedConn.Close(), which calls managedConn.afterClose(), - // which calls chanGroup.removeMgdConn(), which tries to lock mutex. - if cg.forceKill { - // Immediately cancel the previous dsn - if prev.prevCancel != nil { - prev.prevCancel() - cg.logf("chanGroup.processNewValue", "canceled context for previous dsn: '%s'", prev.prevRedactVal) + g, ok := h.groups[name] + if !ok { + mu.RLock() + strategy, okStrategy := strategies[uri.Scheme] + sqlDriver, okDriver := sqlDrivers[uri.Host] + mu.RUnlock() + if !okStrategy { + return nil, ErrUnsupportedStrategy } - } else { - // Immediately cancel the previous-previous dsn. - // We let the previous dsn to gracefully continue until the next dsn-change. - if prev.prevPrevCancel != nil { - prev.prevPrevCancel() - cg.logf("chanGroup.processNewValue", "canceled context for previous-previous dsn: '%s'", prev.prevPrevRedactVal) + if !okDriver { + return nil, ErrUnknownDriver } - } - // Yield to let other threads process cancel signal. - // Otherwise, there's a race and what happens (esp if forceKill=true) - // is that sometimes a db.Exec completes successfully (before cancel is processed), - // but db.Exec is later killed (closed) below because dsn changed, resulting in - // db.Exec returning error. This is inconsistent. - time.Sleep(1 * time.Millisecond) - - // Reset previous connections - // Mutex MUST NOT be held by this point, because in the same thread, - // we will call managedConn.Close() if forceKill is true, - // which calls managedConn.afterClose(), which calls chanGroup.removeMgdConn(), - // which tries to lock mutex. - if cg.forceKill { - // Immediately reset/close previous conns - cg.logf("chanGroup.processNewValue", "reset/close conns for previous dsn: '%s'", prev.prevRedactVal) - for _, c := range prev.prevConns { - c.Reset(true) - // ignore errors from close - c.Close() + queryParams := uri.Query() + forceKill, killWindow, err := parseGroupParams(queryParams) + if err != nil { + return nil, err } - } else { - // Immediately close previous-previous conns. - // We let the previous conns to gracefully continue until the next dsn-change. - cg.logf("chanGroup.processNewValue", "close conns for previous-previous dsn: '%s'", prev.prevPrevRedactVal) - for _, c := range prev.prevPrevConns { - // ignore errors from close - c.Close() + + // The watch is scoped to the group's parent context: canceling it + // (group teardown) ends the watch even if Close were never called. + parentCtx, parentCancel := context.WithCancel(h.ctx) + value, watch, err := strategy.Watch(parentCtx, uri.Path, queryParams.Encode()) + if err != nil { + parentCancel() + return nil, err } - // Immediately reset (but do not close) previous conns. - // We let the previous conns to gracefully continue until the next dsn-change. - cg.logf("chanGroup.processNewValue", "reset conns for previous dsn: '%s'", prev.prevPrevRedactVal) - for _, c := range prev.prevConns { - c.Reset(true) + g = &group{ + name: name, + strategyName: uri.Scheme, + path: uri.Path, + sqlDriver: sqlDriver, + forceKill: forceKill, + killWindow: killWindow, + parentCtx: parentCtx, + parentCancel: parentCancel, + watch: watch, } + g.cur = newGeneration(parentCtx, value) + h.groups[name] = g + h.hooksNote.Do(warnIfNoHooks) + h.logf("hotload", "new group: '%s'", name) + EmitWatchEvent(WatchEvent{GroupName: name, Strategy: uri.Scheme, Path: uri.Path}) + go g.runLoop() } -} -func mergeConnStringOptions(dsn string, options map[string]string) (string, error) { - if len(options) == 0 { - return dsn, nil - } - u, err := url.ParseRequestURI(dsn) - if err != nil { - return "", fmt.Errorf("unable to parse connection string when specifying extra driver options: %v", err) - } - values, err := url.ParseQuery(u.RawQuery) - if err != nil { - return "", fmt.Errorf("unable to parse query options in connection string when specifying extra driver options: %v", err) - } - for k, v := range options { - values.Set(k, v) + if pin { + g.pinned = true + } else { + g.refs++ } - u.RawQuery = values.Encode() - return u.String(), nil + return g, nil } -func (cg *chanGroup) Open() (driver.Conn, error) { - cg.mu.Lock() - defer cg.mu.Unlock() - dsn, err := mergeConnStringOptions(cg.value, cg.sqlDriver.options) - if err != nil { - return nil, err - } - redactDsn := internal.RedactUrl(dsn) - conn, err := cg.sqlDriver.driver.Open(dsn) - if err != nil { - return conn, err +// releaseGroup drops one connector reference and shuts the group down when +// no references remain (unless a legacy Open pinned it). The strategy watch +// is closed while h.mu is still held: getGroup establishes watches under +// the same lock, so a dying group's watch has fully released its strategy +// resources before a new group for the same DSN can establish its own. +// This is why Watchable.Close must never call back into hotload. +func (h *hdriver) releaseGroup(name string) { + h.mu.Lock() + g, ok := h.groups[name] + if ok { + g.refs-- + if g.refs > 0 || g.pinned { + g = nil + } else { + delete(h.groups, name) + g.closeWatch() + } } + h.mu.Unlock() - manConn := newManagedConn(cg.ctx, dsn, redactDsn, conn, cg.removeMgdConn) - cg.conns = append(cg.conns, manConn) - cg.logf("chanGroup.Open", "opened managed conn: '%s'", manConn.redactDsn) - - return manConn, nil -} - -func (cg *chanGroup) removeMgdConn(conn *managedConn) { - cg.mu.Lock() - defer cg.mu.Unlock() - for i, c := range cg.conns { - if c == conn { - cg.conns = append(cg.conns[:i], cg.conns[i+1:]...) - cg.logf("chanGroup.removeMgdConn", "%d: removed: '%s'", i, conn.redactDsn) - return - } + if g != nil { + g.finishShutdown() } } -func (cg *chanGroup) parseUrlValues(vs url.Values) { - cg.logf("chanGroup.parseUrlValues", "values: %s", vs) - v, ok := vs[forceKill] - if ok && len(v) > 0 { - firstValue := v[0] - cg.forceKill = firstValue == "true" - cg.logf("chanGroup.parseUrlValues", "forceKill set to true") +// warnIfNoHooks logs a one-time notice when the first watch starts with no +// hooks registered. Hotload v1 exported prometheus metrics as an import +// side effect; a v3 consumer that ports the import paths and nothing else +// would lose those metrics with no other signal. The notice goes to the +// error logger because, unlike the info logger, it is visible by default; +// silence it by registering hooks (e.g. the observability module) or via +// logger.WithErrLogger. +func warnIfNoHooks() { + if hooksRegistered() { + return } + logger.ErrLogf("hotload:", "no hooks registered; unlike hotload v1, v3 does not export prometheus metrics unless enabled — "+ + "import github.com/infobloxopen/hotload/observability and call observability.MustEnablePrometheus(nil) before opening connections (see MIGRATION.md)") } -func (h *hdriver) Open(name string) (driver.Conn, error) { - uri, err := url.Parse(name) - if err != nil { - return nil, err +func parseGroupParams(vs url.Values) (forceKill bool, killWindow time.Duration, err error) { + killWindow = DefaultKillWindow + if v, ok := vs[forceKillParam]; ok && len(v) > 0 { + forceKill = v[0] == "true" } - mu.Lock() - defer mu.Unlock() - - // look up in the chan group - cgroup, ok := h.cgroup[name] - if !ok { - strategy, ok := strategies[uri.Scheme] - if !ok { - return nil, ErrUnsupportedStrategy - } - sqlDriver, ok := sqlDrivers[uri.Host] - if !ok { - return nil, ErrUnknownDriver - } - queryParams := uri.Query() - value, newValChan, err := strategy.Watch(h.ctx, uri.Path, queryParams.Encode()) + if v, ok := vs[killWindowParam]; ok && len(v) > 0 { + killWindow, err = time.ParseDuration(v[0]) if err != nil { - return nil, err - } - ctx, cancel := context.WithCancel(h.ctx) - cgroup = &chanGroup{ - name: name, - value: value, - redactVal: internal.RedactUrl(value), - newValChan: newValChan, - parentCtx: h.ctx, - ctx: ctx, - cancel: cancel, - sqlDriver: sqlDriver, - conns: make([]*managedConn, 0), + return false, 0, fmt.Errorf("%w: invalid %s: %v", ErrMalformedConnectionString, killWindowParam, err) } - cgroup.parseUrlValues(queryParams) - h.cgroup[name] = cgroup - h.logf("hotload", "new chanGroup: '%s'", name) - go cgroup.runLoop() } - return cgroup.Open() + return forceKill, killWindow, nil } func (h *hdriver) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s:", prefix) - logger.Logf(logPrefix, format, args...) -} - -func (cg *chanGroup) logf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, cg.name) - logger.Logf(logPrefix, format, args...) + logger.Logf(fmt.Sprintf("%s:", prefix), format, args...) } -// Deprecated: Use logger.WithLogger() instead, retained for backwards-compatibility only +// Deprecated: Use logger.WithLogger() instead, retained for +// backwards-compatibility only. func WithLogger(l logger.Logger) { logger.WithLogger(l) } -// Deprecated: Use logger.GetLogger() instead, retained for backwards-compatibility only +// Deprecated: Use logger.GetLogger() instead, retained for +// backwards-compatibility only. func GetLogger() logger.Logger { return logger.GetLogger() } diff --git a/driver_internal_test.go b/driver_internal_test.go index 10864d2..22f830d 100644 --- a/driver_internal_test.go +++ b/driver_internal_test.go @@ -1,131 +1,352 @@ package hotload import ( + "context" "database/sql/driver" "fmt" - "reflect" + "net/url" + "strings" + "sync" "testing" + "time" + + "github.com/infobloxopen/hotload/v3/internal/dbfake" + "github.com/infobloxopen/hotload/v3/logger" ) -type testDriver struct { - options map[string]string +// testStrategy is a minimal in-package fake Strategy. The richer fake in +// the external test package cannot be used here (it imports hotload), and +// dbfake cannot host one (the Strategy interface names hotload.Watchable). +type testStrategy struct { + mu sync.Mutex + initial map[string]string + watches int } -func (d *testDriver) Open(name string) (driver.Conn, error) { - return nil, fmt.Errorf("not implemented") +func newTestStrategy(initial map[string]string) *testStrategy { + return &testStrategy{initial: initial} } -func withConnectionStringOptions(options map[string]string) driverOption { - return func(di *driverInstance) { - di.options = options +func (s *testStrategy) Watch(ctx context.Context, pth string, pathQry string) (string, Watchable, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.initial[pth] + if !ok { + return "", nil, fmt.Errorf("testStrategy: no initial value for path %q", pth) } + s.watches++ + return value, &testWatch{strat: s, ch: make(chan string)}, nil } -func TestRegisterSQLDriverWithOptions(t *testing.T) { - type args struct { - name string - driver driver.Driver - options []driverOption - } - tests := []struct { - name string - args args - }{ - { - name: "driver with an option", - args: args{ - name: "test with options", - driver: &testDriver{}, - options: []driverOption{ - withConnectionStringOptions(map[string]string{"a": "b"}), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - RegisterSQLDriver(tt.args.name, tt.args.driver, tt.args.options...) - mu.Lock() - defer mu.Unlock() +func (s *testStrategy) Watches() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.watches +} - d, ok := sqlDrivers[tt.args.name] - if !ok { - t.Errorf("RegisterSQLDriver() did not register the driver") - } - gotOptions := d.driver.(*testDriver).options - if reflect.DeepEqual(gotOptions, tt.args.options) { - t.Errorf("RegisterSQLDriver() did not set the options") - } - }) +type testWatch struct { + strat *testStrategy + ch chan string + closed bool // guarded by strat.mu +} + +func (w *testWatch) Values() <-chan string { return w.ch } + +func (w *testWatch) Close() error { + w.strat.mu.Lock() + defer w.strat.mu.Unlock() + if !w.closed { + w.closed = true + close(w.ch) + w.strat.watches-- } + return nil } func Test_mergeConnStringOptions(t *testing.T) { - type args struct { - dsn string - options map[string]string - } tests := []struct { name string - args args + dsn string + options map[string]string want string wantErr bool }{ { name: "empty", - args: args{ - dsn: "", - options: nil, - }, - want: "", - wantErr: false, + want: "", }, { name: "bad dsn with no options", - args: args{ - dsn: "bad dsn", - options: nil, - }, - want: "bad dsn", - wantErr: false, + dsn: "bad dsn", + want: "bad dsn", }, { - name: "bad dsn with options", - args: args{ - dsn: "bad dsn", - options: map[string]string{"a": "b"}, - }, - want: "", + name: "bad dsn with options", + dsn: "bad dsn", + options: map[string]string{"a": "b"}, wantErr: true, }, { name: "good dsn with no options", - args: args{ - dsn: "postgres://localhost:5432/postgres?sslmode=disable", - }, - want: "postgres://localhost:5432/postgres?sslmode=disable", - wantErr: false, + dsn: "postgres://localhost:5432/postgres?sslmode=disable", + want: "postgres://localhost:5432/postgres?sslmode=disable", }, { - name: "good dsn with options", - args: args{ - dsn: "postgres://localhost:5432/postgres?sslmode=disable", - options: map[string]string{"disable_cache": "true"}, - }, + name: "good dsn with options", + dsn: "postgres://localhost:5432/postgres?sslmode=disable", + options: map[string]string{"disable_cache": "true"}, want: "postgres://localhost:5432/postgres?disable_cache=true&sslmode=disable", - wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := mergeConnStringOptions(tt.args.dsn, tt.args.options) + got, err := mergeConnStringOptions(tt.dsn, tt.options) if (err != nil) != tt.wantErr { - t.Errorf("mergeConnStringOptions() error = %v, wantErr %v", err, tt.wantErr) - return + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) } if got != tt.want { - t.Errorf("mergeConnStringOptions() = %v, want %v", got, tt.want) + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +func Test_parseGroupParams(t *testing.T) { + tests := []struct { + name string + query string + wantForceKill bool + wantKillWindow time.Duration + wantErr bool + }{ + {name: "defaults", query: "", wantKillWindow: DefaultKillWindow}, + {name: "forceKill true", query: "forceKill=true", wantForceKill: true, wantKillWindow: DefaultKillWindow}, + {name: "forceKill false", query: "forceKill=false", wantKillWindow: DefaultKillWindow}, + {name: "forceKill garbage", query: "forceKill=yes", wantKillWindow: DefaultKillWindow}, + {name: "killWindow", query: "killWindow=250ms", wantKillWindow: 250 * time.Millisecond}, + {name: "killWindow invalid", query: "killWindow=bogus", wantErr: true}, + {name: "both", query: "forceKill=true&killWindow=1s", wantForceKill: true, wantKillWindow: time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vs, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatal(err) + } + forceKill, killWindow, err := parseGroupParams(vs) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if forceKill != tt.wantForceKill { + t.Errorf("forceKill = %v, want %v", forceKill, tt.wantForceKill) + } + if killWindow != tt.wantKillWindow { + t.Errorf("killWindow = %v, want %v", killWindow, tt.wantKillWindow) } }) } } + +func TestRegisterSQLDriverWithOptions(t *testing.T) { + name := "internal-test-driver-options" + RegisterSQLDriver(name, &dbfake.Driver{}, WithDriverOptions(map[string]string{"a": "b"})) + + mu.RLock() + defer mu.RUnlock() + di, ok := sqlDrivers[name] + if !ok { + t.Fatal("RegisterSQLDriver did not register the driver") + } + if di.options["a"] != "b" { + t.Errorf("options = %v, want a=b", di.options) + } +} + +func TestRegisterPanics(t *testing.T) { + mustPanic := func(name string, fn func()) { + t.Run(name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("expected panic") + } + }() + fn() + }) + } + + mustPanic("nil driver", func() { RegisterSQLDriver("internal-test-nil", nil) }) + mustPanic("dup driver", func() { + RegisterSQLDriver("internal-test-dup", &dbfake.Driver{}) + RegisterSQLDriver("internal-test-dup", &dbfake.Driver{}) + }) + mustPanic("nil strategy", func() { RegisterStrategy("internal-test-nilstrat", nil) }) + mustPanic("dup strategy", func() { + s := newTestStrategy(nil) + RegisterStrategy("internal-test-dupstrat", s) + defer UnregisterStrategy("internal-test-dupstrat") + RegisterStrategy("internal-test-dupstrat", s) + }) +} + +func TestRegistryLists(t *testing.T) { + RegisterSQLDriver("internal-test-list-b", &dbfake.Driver{}) + RegisterSQLDriver("internal-test-list-a", &dbfake.Driver{}) + RegisterStrategy("internal-test-list-strat", newTestStrategy(nil)) + defer UnregisterStrategy("internal-test-list-strat") + + drivers := SQLDrivers() + prev := "" + seen := 0 + for _, d := range drivers { + if d < prev { + t.Errorf("SQLDrivers not sorted: %v", drivers) + } + prev = d + if d == "internal-test-list-a" || d == "internal-test-list-b" { + seen++ + } + } + if seen != 2 { + t.Errorf("registered drivers missing from SQLDrivers(): %v", drivers) + } + + found := false + for _, s := range Strategies() { + if s == "internal-test-list-strat" { + found = true + } + } + if !found { + t.Errorf("registered strategy missing from Strategies(): %v", Strategies()) + } +} + +// TestLegacyOpenPath drives the non-connector driver.Open path directly +// against a private hdriver instance (the path database/sql no longer uses, +// but third-party pools might). The group is pinned, so teardown happens via +// the parent context. +func TestLegacyOpenPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h := newHdriver(ctx) + drv := &dbfake.Driver{Caps: dbfake.CapsModern} + strat := newTestStrategy(map[string]string{"/cfg": "dsn-1"}) + RegisterSQLDriver("internal-test-legacy-drv", drv) + RegisterStrategy("internal-test-legacy-strat", strat) + defer UnregisterStrategy("internal-test-legacy-strat") + + name := "internal-test-legacy-strat://internal-test-legacy-drv/cfg" + conn, err := h.Open(name) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if _, ok := conn.(driver.QueryerContext); !ok { + t.Error("legacy-opened conn should expose QueryerContext for a modern underlying conn") + } + + // A second open shares the group (one watch). + conn2, err := h.Open(name) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + if n := strat.Watches(); n != 1 { + t.Errorf("watches = %d, want 1", n) + } +} + +// TestNoHooksNotice: creating the first group on a driver with no hooks +// registered logs a one-time notice pointing at the observability module +// (v1 exported prometheus metrics as an import side effect; without the +// notice a ported service loses them silently). With hooks registered, +// nothing is logged. +func TestNoHooksNotice(t *testing.T) { + resetHooks() + defer resetHooks() + + var mu sync.Mutex + notices := 0 + logger.WithErrLogger(func(args ...any) { + mu.Lock() + defer mu.Unlock() + if strings.Contains(fmt.Sprint(args...), "no hooks registered") { + notices++ + } + }) + defer logger.WithErrLogger(nil) + count := func() int { + mu.Lock() + defer mu.Unlock() + return notices + } + + RegisterSQLDriver("internal-test-notice-drv", &dbfake.Driver{Caps: dbfake.CapsModern}) + RegisterStrategy("internal-test-notice-strat", newTestStrategy(map[string]string{"/cfg": "dsn-1"})) + defer UnregisterStrategy("internal-test-notice-strat") + base := "internal-test-notice-strat://internal-test-notice-drv/cfg" + + // Cancelable driver contexts: legacy Open pins groups, so their run + // loops must be terminated via the parent context or they trip the + // goroutine-leak checks of later tests. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h := newHdriver(ctx) + conn, err := h.Open(base) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + conn2, err := h.Open(base + "?forceKill=true") // second group, same driver + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + + if got := count(); got != 1 { + t.Errorf("notices = %d, want exactly 1 per driver", got) + } + + // With hooks registered, a fresh driver stays quiet. + RegisterHooks(Hooks{OnConfigChange: func(ConfigChangeEvent) {}}) + h2 := newHdriver(ctx) + conn3, err := h2.Open(base + "?killWindow=200ms") + if err != nil { + t.Fatal(err) + } + defer conn3.Close() + + if got := count(); got != 1 { + t.Errorf("notices after hooks registered = %d, want still 1", got) + } +} + +// TestEmitHelpersNilSafe ensures hooks with nil fields are skipped. +func TestEmitHelpersNilSafe(t *testing.T) { + resetHooks() + defer resetHooks() + RegisterHooks(Hooks{}) // all nil + + emitConfigChange(ConfigChangeEvent{}) + emitConnOpen(ConnEvent{}) + emitConnClose(ConnEvent{}) + emitTxComplete(TxEvent{}) + EmitWatchEvent(WatchEvent{}) + EmitModTimeEvent(ModTimeEvent{}) +} + +func ExampleContextWithExecLabels() { + ctx := ContextWithExecLabels(context.Background(), map[string]string{ + "grpc_service": "ContactsService", + "grpc_method": "ListContacts", + }) + labels := GetExecLabelsFromContext(ctx) + fmt.Println(labels["grpc_service"], labels["grpc_method"]) + // Output: ContactsService ListContacts +} diff --git a/driver_test.go b/driver_test.go deleted file mode 100644 index 29980e2..0000000 --- a/driver_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package hotload_test - -import ( - "database/sql" - "database/sql/driver" - "os" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/infobloxopen/hotload" - "github.com/infobloxopen/hotload/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func getDriverFromSqlMock() driver.Driver { - littleBuddy, mock, _ := sqlmock.NewWithDSN("user=pqgotest dbname=pqgotest sslmode=verify-full") - mockDriver = mock - return littleBuddy.Driver() -} - -func getRandomDriver() driver.Driver { - db, _, _ := sqlmock.New() - return db.Driver() -} - -var mockDriver sqlmock.Sqlmock -var configFile string -var configFileDir string - -var _ = BeforeSuite(func() { - driver := getDriverFromSqlMock() - - if driver == nil { - Fail("driver is nil, boo!") - } - - hotload.RegisterSQLDriver("sqlmock", driver) - Expect(hotload.SQLDrivers()).To(ContainElement("sqlmock")) - var err error - configFile, err = os.Getwd() - Expect(err).ToNot(HaveOccurred()) - configFileDir = configFile + "/testdata/" - configFile += "/testdata/myconfig.txt" -}) - -var _ = Describe("Driver", func() { - Context("RegisterSQLDriver", func() { - It("Should panic when registering the same driver twice", func() { - driver := getRandomDriver() - Expect(func() { hotload.RegisterSQLDriver("sqlmock", driver) }). - To(PanicWith(MatchRegexp("Register called twice for driver"))) - }) - - It("Should panic on nil driver", func() { - Expect(func() { hotload.RegisterSQLDriver("", nil) }). - To(PanicWith(MatchRegexp("Register driver is nil"))) - }) - }) - - Context("RegisterStrategy", func() { - It("Should panic when registering the same strategy twice", func() { - strat := fsnotify.NewStrategy() - Expect(func() { hotload.RegisterStrategy("fsnotify", strat) }). - To(PanicWith(MatchRegexp("RegisterStrategy called twice for strategy"))) - }) - - It("Should panic on nil driver", func() { - Expect(func() { hotload.RegisterStrategy("", nil) }). - To(PanicWith(MatchRegexp("strategy is nil"))) - }) - }) - - Context("Open", func() { - It("Should throw an error with unknown driver", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmaybe?"+configFile) - Expect(err).ToNot(HaveOccurred()) - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(hotload.ErrUnknownDriver)) - - }) - - It("Should not throw an error with a registered driver and strategy", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmock"+configFile) - Expect(err).ToNot(HaveOccurred()) - - Expect(db.Ping()).ToNot(HaveOccurred()) - }) - - It("Should throw an unsupported strategy error", func() { - db, err := sql.Open("hotload", "fstransmogrify://sqlmock/"+configFile) - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(hotload.ErrUnsupportedStrategy)) - }) - - It("Should throw an error if it can't find the config file", func() { - db, err := sql.Open("hotload", "fsnotify://sqlmock/temple/run/2021-edition") - err = db.Ping() - Expect(err).To(HaveOccurred()) - }) - - It("Should throw an error the url is unparseable", func() { - db, err := sql.Open("hotload", "://") - err = db.Ping() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("missing protocol scheme")) - }) - - //It("Should close my connection when the connection information changes", func() { - // db, err := sql.Open("hotload", "fsnotify://sqlmock"+configFileDir+"urconfig.txt") - // Expect(err).ToNot(HaveOccurred()) - // - // Expect(db.Ping()).ToNot(HaveOccurred()) - // // Open dat - // // Do a thing - // // change connection file - // mockDriver.ExpectBegin() - // mockDriver.ExpectExec("SELECT 1") - // mockDriver.ExpectCommit() - // tx, err := db.Begin() - // Expect(err).ToNot(HaveOccurred()) - // tx.Exec("SELECT 1") - // err = ioutil.WriteFile(configFileDir+"urconfig.txt", []byte("user=pqgotest dbname=pqgotestorooni sslmode=verify-full"), 0644) - // Expect(err).ToNot(HaveOccurred()) - // go func () { - // tx.Commit() - // Expect(mockDriver.ExpectationsWereMet()).ToNot(HaveOccurred()) - // }() - //}) - }) -}) diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..45966f9 --- /dev/null +++ b/errors.go @@ -0,0 +1,22 @@ +package hotload + +import "errors" + +var ( + // ErrUnsupportedStrategy is returned when the DSN names a strategy that + // has not been registered with RegisterStrategy. + ErrUnsupportedStrategy = errors.New("unsupported hotload strategy") + + // ErrMalformedConnectionString is returned when the hotload DSN cannot + // be parsed. + ErrMalformedConnectionString = errors.New("malformed hotload connection string") + + // ErrUnknownDriver is returned when the DSN names an underlying driver + // that has not been registered with RegisterSQLDriver. + ErrUnknownDriver = errors.New("target driver is not registered with hotload") + + // ErrHotSwap is the cancellation cause used when hotload retires a + // connection because the connection string changed. Callers can detect + // it with context.Cause and errors.Is. + ErrHotSwap = errors.New("hotload: connection retired by config change") +) diff --git a/export_test.go b/export_test.go new file mode 100644 index 0000000..7f740ac --- /dev/null +++ b/export_test.go @@ -0,0 +1,6 @@ +package hotload + +// ResetHooks removes all registered hooks. Exported to the external test +// package only; tests register per-test hooks and must not see hooks from +// earlier tests. +var ResetHooks = resetHooks diff --git a/fallback_test.go b/fallback_test.go new file mode 100644 index 0000000..956c464 --- /dev/null +++ b/fallback_test.go @@ -0,0 +1,229 @@ +package hotload_test + +import ( + "context" + "database/sql" + "strings" + "testing" + + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// methodsCalled returns the set of methods recorded by the fake driver. +func methodsCalled(drv *dbfake.Driver) map[string]int { + out := map[string]int{} + for _, c := range drv.Log.Calls() { + out[c.Method]++ + } + return out +} + +func assertCalled(t *testing.T, drv *dbfake.Driver, method string) { + t.Helper() + if drv.Log.Count(method) == 0 { + t.Errorf("expected %s to be called; calls: %v", method, methodsCalled(drv)) + } +} + +func assertNotCalled(t *testing.T, drv *dbfake.Driver, method string) { + t.Helper() + if n := drv.Log.Count(method); n > 0 { + t.Errorf("expected %s not to be called, got %d calls; calls: %v", method, n, methodsCalled(drv)) + } +} + +// TestExecDispatch verifies that db.Exec reaches the underlying conn through +// the path matching its capabilities — and, crucially, that a conn with no +// exec support makes database/sql fall back to a prepared statement instead +// of bouncing off driver.ErrSkip as hotload v1 did. +func TestExecDispatch(t *testing.T) { + t.Run("ExecerContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapExecerContext, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "ExecContext") + assertNotCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("legacy Execer", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapExecer, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "ExecContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("no exec support falls back to prepared stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if _, err := fx.db.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Prepare") + assertCalled(t, fx.drv, "StmtExec") + assertNotCalled(t, fx.drv, "Exec") + assertNotCalled(t, fx.drv, "ExecContext") + }) +} + +// TestQueryDispatch is the query-side analog of TestExecDispatch. +func TestQueryDispatch(t *testing.T) { + t.Run("QueryerContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapQueryerContext, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "QueryContext") + assertNotCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("legacy Queryer", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapQueryer, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "QueryContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("no query support falls back to prepared stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + assertCalled(t, fx.drv, "Prepare") + assertCalled(t, fx.drv, "StmtQuery") + assertNotCalled(t, fx.drv, "Query") + assertNotCalled(t, fx.drv, "QueryContext") + }) +} + +// TestPrepareDispatch verifies PrepareContext synthesis: with underlying +// support it is delegated; without it hotload falls back to Prepare exactly +// like database/sql would. +func TestPrepareDispatch(t *testing.T) { + t.Run("with ConnPrepareContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + stmt, err := fx.db.Prepare("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + assertCalled(t, fx.drv, "PrepareContext") + assertNotCalled(t, fx.drv, "Prepare") + }) + + t.Run("without ConnPrepareContext", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + stmt, err := fx.db.Prepare("SELECT dsn") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + assertCalled(t, fx.drv, "Prepare") + assertNotCalled(t, fx.drv, "PrepareContext") + }) +} + +// TestStmtDispatch verifies prepared statements run through the stmt +// wrappers' context methods when supported and the legacy methods when not. +func TestStmtDispatch(t *testing.T) { + t.Run("modern stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + stmt, err := fx.db.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "StmtExecContext") + assertNotCalled(t, fx.drv, "StmtExec") + }) + + t.Run("legacy stmt", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + stmt, err := fx.db.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "StmtExec") + assertNotCalled(t, fx.drv, "StmtExecContext") + }) +} + +// TestPingDispatch: with no underlying Pinger, db.Ping must succeed without +// reaching the driver (database/sql treats absence as "no health check"). +func TestPingDispatch(t *testing.T) { + t.Run("with Pinger", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapPinger, rawCaps: true}) + if err := fx.db.Ping(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Ping") + }) + + t.Run("without Pinger", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + if err := fx.db.Ping(); err != nil { + t.Fatal(err) + } + assertNotCalled(t, fx.drv, "Ping") + }) +} + +// TestBeginTxFallback verifies the ConnBeginTx synthesis: delegation when +// supported, the exact database/sql fallback errors when not. +func TestBeginTxFallback(t *testing.T) { + ctx := context.Background() + + t.Run("with ConnBeginTx", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: dbfake.CapsModern}) + tx, err := fx.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) + if err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "BeginTx") + }) + + t.Run("without ConnBeginTx default options", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + assertCalled(t, fx.drv, "Begin") + }) + + t.Run("without ConnBeginTx non-default isolation", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + _, err := fx.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) + if err == nil || !strings.Contains(err.Error(), "non-default isolation level") { + t.Fatalf("BeginTx error = %v, want non-default isolation level error", err) + } + }) + + t.Run("without ConnBeginTx read-only", func(t *testing.T) { + fx := newFixture(t, fxCfg{caps: 0, rawCaps: true}) + _, err := fx.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err == nil || !strings.Contains(err.Error(), "read-only") { + t.Fatalf("BeginTx error = %v, want read-only error", err) + } + }) +} diff --git a/fixture_test.go b/fixture_test.go new file mode 100644 index 0000000..ed23bc8 --- /dev/null +++ b/fixture_test.go @@ -0,0 +1,158 @@ +package hotload_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// fixtureSeq makes registered driver/strategy names unique per fixture; +// hotload registries are global and panic on duplicates. +var fixtureSeq atomic.Int64 + +const fixturePath = "/cfg" + +type fxCfg struct { + caps dbfake.Caps // dbfake.CapsModern if zero and rawCaps unset + rawCaps bool // use caps even if zero (a bare conn) + params string // hotload DSN query params, e.g. "forceKill=true" + initial string // initial underlying DSN; default "dsn-1" + execFn func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) + queryFn func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) + noDB bool // register only; do not sql.Open +} + +type fixture struct { + t *testing.T + db *sql.DB + drv *dbfake.Driver + strat *fakeStrategy + dsn string // the hotload DSN + changes chan hotload.ConfigChangeEvent +} + +// newFixture registers a fake driver and strategy under unique names, opens +// a hotload sql.DB on them, and subscribes to config-change events. Hooks +// are reset so events from earlier tests cannot interfere; tests using +// fixtures must not run in parallel. +func newFixture(t *testing.T, cfg fxCfg) *fixture { + t.Helper() + + caps := cfg.caps + if caps == 0 && !cfg.rawCaps { + caps = dbfake.CapsModern + } + initial := cfg.initial + if initial == "" { + initial = "dsn-1" + } + + n := fixtureSeq.Add(1) + stratName := fmt.Sprintf("fakestrat%d", n) + drvName := fmt.Sprintf("fakedrv%d", n) + + fx := &fixture{ + t: t, + drv: &dbfake.Driver{Caps: caps, ExecFn: cfg.execFn, QueryFn: cfg.queryFn}, + strat: newFakeStrategy(map[string]string{fixturePath: initial}), + changes: make(chan hotload.ConfigChangeEvent, 100), + } + + hotload.ResetHooks() + t.Cleanup(hotload.ResetHooks) + hotload.RegisterHooks(hotload.Hooks{ + OnConfigChange: func(ev hotload.ConfigChangeEvent) { + select { + case fx.changes <- ev: + default: + } + }, + }) + + hotload.RegisterSQLDriver(drvName, fx.drv) + hotload.RegisterStrategy(stratName, fx.strat) + t.Cleanup(func() { hotload.UnregisterStrategy(stratName) }) + + fx.dsn = stratName + "://" + drvName + fixturePath + if cfg.params != "" { + fx.dsn += "?" + cfg.params + } + + if !cfg.noDB { + db, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatalf("sql.Open(%q): %v", fx.dsn, err) + } + t.Cleanup(func() { db.Close() }) + fx.db = db + } + return fx +} + +// queryDSN reports which underlying DSN served the query — the fake's +// default result set is one row holding the conn's DSN. +func (fx *fixture) queryDSN() string { + fx.t.Helper() + var dsn string + if err := fx.db.QueryRow("SELECT dsn").Scan(&dsn); err != nil { + fx.t.Fatalf("queryDSN: %v", err) + } + return dsn +} + +// push delivers a new value to the strategy watcher; it returns once the +// group run loop has received (but not necessarily processed) it. +func (fx *fixture) push(value string) { + fx.t.Helper() + fx.strat.Push(fixturePath, value) +} + +// pushAndWait pushes a changed value and waits for the resulting +// config-change event. Connection retirement happens after the event fires, +// so tests must still poll for conn-state assertions. +func (fx *fixture) pushAndWait(value string) hotload.ConfigChangeEvent { + fx.t.Helper() + fx.push(value) + select { + case ev := <-fx.changes: + return ev + case <-time.After(5 * time.Second): + fx.t.Fatalf("timed out waiting for config change event after pushing %q", value) + return hotload.ConfigChangeEvent{} + } +} + +// noPendingChange asserts no config-change event is buffered. +func (fx *fixture) noPendingChange() { + fx.t.Helper() + select { + case ev := <-fx.changes: + fx.t.Fatalf("unexpected config change event: %+v", ev) + default: + } +} + +// rawConn hands the wrapped driver.Conn of a pooled connection to fn. +func (fx *fixture) rawConn(fn func(dc driver.Conn)) { + fx.t.Helper() + ctx := context.Background() + conn, err := fx.db.Conn(ctx) + if err != nil { + fx.t.Fatalf("db.Conn: %v", err) + } + defer conn.Close() + err = conn.Raw(func(dc any) error { + fn(dc.(driver.Conn)) + return nil + }) + if err != nil { + fx.t.Fatalf("conn.Raw: %v", err) + } +} diff --git a/fsnotify/filewatcher.go b/fsnotify/filewatcher.go index 46cf159..93c16d7 100644 --- a/fsnotify/filewatcher.go +++ b/fsnotify/filewatcher.go @@ -2,6 +2,7 @@ package fsnotify import ( "context" + "errors" "fmt" "os" "path" @@ -10,61 +11,68 @@ import ( "time" rfsnotify "github.com/fsnotify/fsnotify" - "github.com/infobloxopen/hotload" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" - "github.com/pkg/errors" + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/logger" ) func init() { hotload.RegisterStrategy("fsnotify", NewStrategy()) } -var resyncPeriod = time.Second * 2 +const defaultResyncPeriod = time.Second * 2 // NewStrategy implements a hotload strategy that monitors config changes // in a file using fsnotify. func NewStrategy() *Strategy { return &Strategy{ - paths: make(map[string]*pathWatch), + paths: make(map[string]*pathWatch), + resyncPeriod: defaultResyncPeriod, } } -// Strategy implements the hotload Strategy inferface by using +// Strategy implements the hotload Strategy interface by using // fsnotify under the covers. type Strategy struct { - mu sync.RWMutex - paths map[string]*pathWatch - watcher watcher + mu sync.RWMutex + paths map[string]*pathWatch + watcher watcher + resyncPeriod time.Duration } -type pendingOperation struct { - operation string - watchPath string - pathQuery string +// update is one value change queued for delivery to a watcher. +type update struct { dsn string redactDsn string } +// queryWatch is one active watch handed out by Watch (it implements +// hotload.Watchable). It fans one path's updates out to one subscriber. Its +// opLoop goroutine is the only writer of updateChan: it forwards queued +// updates and closes the channel when operChan closes, so channel +// operations never race. done is closed (under the strategy lock) when the +// watch is closed, unblocking an opLoop stuck sending to a subscriber that +// stopped receiving. type queryWatch struct { parentPathW *pathWatch - pathQuery string + pathQuery string // for logging only updateChan chan string - operChan chan pendingOperation + operChan chan update + done chan struct{} + stopAfter func() bool // detaches the ctx-cancel hook installed by Watch } type pathWatch struct { parentStrat *Strategy watchPath string value string - queries map[string]*queryWatch + queries map[*queryWatch]struct{} } func (s *Strategy) readConfigFile(path string) (v []byte, err error) { v, err = os.ReadFile(path) if err != nil { - return nil, errors.Wrapf(err, "could not read %v", path) + return nil, fmt.Errorf("could not read %v: %w", path, err) } v = []byte(strings.TrimSpace(string(v))) return @@ -83,11 +91,11 @@ func (s *Strategy) resync(w watcher, pth string) (string, error) { return string(bs), w.Add(pth) } -func (s *Strategy) runLoop() { +func (s *Strategy) runLoop(w watcher) { failedPaths := make(map[string]struct{}) for { select { - case ev, ok := <-s.watcher.GetEvents(): + case ev, ok := <-w.GetEvents(): if !ok { s.logf("fsnotify.runLoop", "Events chan closed, terminating") return @@ -98,27 +106,28 @@ func (s *Strategy) runLoop() { continue } - val, err := s.resync(s.watcher, ev.Name) - if err != nil { - s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", ev.Name, err) - failedPaths[ev.Name] = struct{}{} - break + for _, pth := range s.affectedPaths(ev.Name) { + val, err := s.resync(w, pth) + if err != nil { + s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", pth, err) + failedPaths[pth] = struct{}{} + continue + } + delete(failedPaths, pth) + s.setVal(pth, val) } - s.setVal(ev.Name, val) - - case err, ok := <-s.watcher.GetErrors(): + case err, ok := <-w.GetErrors(): if !ok { s.logf("fsnotify.runLoop", "Errors chan closed, terminating") return } s.logf("fsnotify.runLoop", "got error: %s", err.Error()) - case <-time.After(resyncPeriod): - s.logf("fsnotify.runLoop", "resyncPeriod %s timedout", resyncPeriod.String()) + case <-time.After(s.resyncPeriod): var fixedPaths []string for pth := range failedPaths { - val, err := s.resync(s.watcher, pth) + val, err := s.resync(w, pth) if err != nil { s.errlogf("fsnotify.runLoop", "resync(%s) err: %v", pth, err) } else { @@ -133,31 +142,75 @@ func (s *Strategy) runLoop() { } } +// affectedPaths maps a notification name to the watched paths that need a +// resync. The name usually is a watched path, but when the watched path is +// a symlink (the Kubernetes ConfigMap pattern) some platforms report events +// under the resolved target — kqueue even adds a /private prefix on macOS — +// so an unknown name conservatively resyncs every watched path. Events are +// rare and the watched files are small, so the cost is negligible. +func (s *Strategy) affectedPaths(name string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + if _, ok := s.paths[name]; ok { + return []string{name} + } + out := make([]string, 0, len(s.paths)) + for pth := range s.paths { + out = append(out, pth) + } + return out +} + func (s *Strategy) setVal(pth string, val string) { s.mu.Lock() defer s.mu.Unlock() - if _, ok := s.paths[pth]; !ok { + pathW, ok := s.paths[pth] + if !ok { s.logf("fsnotify.setVal", "ignoring path not in map: '%s'", pth) return } - s.paths[pth].value = val + pathW.value = val redactDsn := internal.RedactUrl(val) - for _, qryW := range s.paths[pth].queries { - pendOp := pendingOperation{ - operation: "send", - dsn: val, - redactDsn: redactDsn, + for qryW := range pathW.queries { + qryW.enqueue(update{dsn: val, redactDsn: redactDsn}) + } +} + +// enqueue queues an update for delivery without ever blocking: only the +// latest value matters to a hotload group, so when a subscriber's queue is +// full (it stopped receiving, or is slow), the oldest queued value is +// dropped to make room. A blocking send here would wedge the whole strategy +// behind one slow subscriber, since enqueue runs under the strategy lock. +func (qw *queryWatch) enqueue(up update) { + for { + select { + case qw.operChan <- up: + return + case <-qw.done: + return + default: + } + // Queue full: drop the oldest queued update and retry. opLoop may + // have consumed one concurrently, so the drain is non-blocking too. + select { + case <-qw.operChan: + qw.logf("fsnotify.enqueue", "subscriber slow; dropped oldest queued update") + default: } - qryW.operChan <- pendOp } } -// Watch implements the hotload.Strategy interface. -func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value string, values <-chan string, err error) { +// Watch implements the hotload.Strategy interface. Every call returns an +// independent watch; watches on the same path share one underlying file +// watch. +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { pth = path.Clean(pth) pathQry = strings.TrimSpace(pathQry) s.mu.Lock() defer s.mu.Unlock() + if s.paths == nil { + s.paths = make(map[string]*pathWatch) + } // if this is the first time this strategy is called, initialize ourselves if s.watcher == nil { watcher, err := notifyConstructor() @@ -165,7 +218,7 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value return "", nil, err } s.watcher = watcher - go s.runLoop() + go s.runLoop(watcher) } pathW, found := s.paths[pth] if found { @@ -173,163 +226,112 @@ func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (value } else { s.logf("fsnotify.Watch", "new path to be watched: '%s'", pth) if err := s.watcher.Add(pth); err != nil { + s.closeWatcherIfIdle() return "", nil, err } - if err := metrics.AddToDefaultPathChksum(pth); err != nil { - if err != metrics.ErrDuplicatePath { - s.errlogf("fsnotify.Watch", "AddToDefaultPathChksum(%s) failed, err=%v", pth, err) - return "", nil, err - } - } bs, err := s.readConfigFile(pth) if err != nil { s.watcher.Remove(pth) + s.closeWatcherIfIdle() return "", nil, err } pathW = &pathWatch{ parentStrat: s, watchPath: pth, value: string(bs), - queries: make(map[string]*queryWatch), + queries: make(map[*queryWatch]struct{}), } s.paths[pth] = pathW } - qryW, found := pathW.queries[pathQry] - if found { - qryW.logf("fsnotify.Watch", "query already being watched") - } else { - pathW.logf("fsnotify.Watch", "new query to be watched: '%s'", pathQry) - qryW = &queryWatch{ - parentPathW: pathW, - pathQuery: pathQry, - updateChan: make(chan string), - operChan: make(chan pendingOperation, 30), - } - pathW.queries[pathQry] = qryW - go qryW.opLoop() + qryW := &queryWatch{ + parentPathW: pathW, + pathQuery: pathQry, + updateChan: make(chan string), + operChan: make(chan update, 30), + done: make(chan struct{}), } + pathW.queries[qryW] = struct{}{} + go qryW.opLoop() + qryW.stopAfter = context.AfterFunc(ctx, func() { qryW.Close() }) + qryW.logf("fsnotify.Watch", "new watch") - return pathW.value, qryW.updateChan, nil + return pathW.value, qryW, nil } -// CloseWatch implements the hotload.Strategy interface. -// Closes the specified watch by removing the path -// from the watcher and closing the path's update channel. -func (s *Strategy) CloseWatch(pth string, pathQry string) error { - pth = path.Clean(pth) - pathQry = strings.TrimSpace(pathQry) +// Values implements hotload.Watchable. +func (qw *queryWatch) Values() <-chan string { + return qw.updateChan +} + +// Close implements hotload.Watchable. The update channel is closed +// (asynchronously, by the watch's delivery goroutine); closing the last +// watch on a path removes the path from the file watcher, and closing the +// last watch on the strategy closes the underlying watcher. +func (qw *queryWatch) Close() error { + pathW := qw.parentPathW + s := pathW.parentStrat s.mu.Lock() defer s.mu.Unlock() - pathW, found := s.paths[pth] - if found { - qryW, ok := pathW.queries[pathQry] - if ok { - pendOp := pendingOperation{ - operation: "close", - watchPath: pth, - pathQuery: pathQry, - } - qryW.operChan <- pendOp - qryW.logf("fsnotify.CloseWatch", "sent pending close operation") - } + select { + case <-qw.done: + return nil // already closed + default: } - return nil -} + qw.stopAfter() + qw.shutdown() + delete(pathW.queries, qw) + qw.logf("fsnotify.Close", "closed watch") -func (s *Strategy) processWatchClosure(pendOp pendingOperation) error { + if len(pathW.queries) > 0 { + return nil + } + delete(s.paths, pathW.watchPath) var err error - s.mu.Lock() - defer s.mu.Unlock() - pathW, found := s.paths[pendOp.watchPath] - if found { - qryW, ok := pathW.queries[pendOp.pathQuery] - if ok { - delete(pathW.queries, pendOp.pathQuery) - qryW.closeUpdateChan() - qryW.logf("fsnotify.processWatchClosure", "closed update channel") - } - if len(pathW.queries) <= 0 { - err = s.watcher.Remove(pendOp.watchPath) - if err == nil { - s.logf("fsnotify.processWatchClosure", "removed path from being watched '%s'", pendOp.watchPath) - } else { - s.errlogf("fsnotify.processWatchClosure", "failed to remove '%s' from watcher, err=%v", pendOp.watchPath, err) - } - delete(s.paths, pendOp.watchPath) - s.logf("fsnotify.processWatchClosure", "strategy removed path '%s'", pendOp.watchPath) - } + if err = s.watcher.Remove(pathW.watchPath); err != nil { + s.errlogf("fsnotify.Close", "failed to remove '%s' from watcher, err=%v", pathW.watchPath, err) + } else { + s.logf("fsnotify.Close", "removed path from being watched '%s'", pathW.watchPath) } + s.closeWatcherIfIdle() return err } -// Close implements the hotload.Strategy interface. -// Closes this strategy by closing the internal watcher -// and closing all the update channels. -func (s *Strategy) Close() { - s.mu.Lock() - defer s.mu.Unlock() - if s.watcher != nil { +// closeWatcherIfIdle closes the internal watcher when no paths remain +// watched, so an idle strategy holds no OS resources; the next Watch +// re-initializes it. Callers must hold the strategy lock. +func (s *Strategy) closeWatcherIfIdle() { + if len(s.paths) == 0 && s.watcher != nil { s.watcher.Close() - s.logf("fsnotify.Close", "closed internal watcher") s.watcher = nil + s.logf("fsnotify", "no paths watched; closed internal watcher") } - for _, pathW := range s.paths { - for _, qryW := range pathW.queries { - qryW.closeUpdateChan() - qryW.logf("fsnotify.Close", "closed update channel") - } - pathW.queries = nil - } - s.paths = nil -} - -func (qw *queryWatch) sendUpdate(val, redactDsn string) { - if qw.updateChan == nil { - return - } - - defer func() { - // Recover/ignore from "panic: send on closed channel" - r := recover() - if r != nil { - qw.logf("fsnotify.sendUpdate", "panic recovery '%s'", r) - } - }() - - qw.logf("fsnotify.sendUpdate", "block-sending redactDsn='%s'", redactDsn) - qw.updateChan <- val - qw.logf("fsnotify.sendUpdate", "successfully sent redactDsn='%s'", redactDsn) } -func (qw *queryWatch) closeUpdateChan() { - close(qw.updateChan) - qw.updateChan = nil +// shutdown stops the watch's delivery goroutine. Callers must hold the +// strategy lock (it is the lock that serializes shutdown with setVal's +// sends, making the channel close safe). +func (qw *queryWatch) shutdown() { + close(qw.done) + close(qw.operChan) } +// opLoop forwards queued updates to the subscriber. It is the only +// goroutine that sends on or closes updateChan. The done channel unblocks +// the forwarding send if the subscriber stopped receiving (e.g. the hotload +// group was torn down before the watch was closed). func (qw *queryWatch) opLoop() { - for { + for op := range qw.operChan { + qw.logf("fsnotify.opLoop", "sending redactDsn='%s'", op.redactDsn) select { - case pendOp, ok := <-qw.operChan: - if !ok { - qw.logf("fsnotify.opLoop", "operChan closed, terminating") - return - } - switch pendOp.operation { - case "close": - qw.logf("fsnotify.opLoop", "pendingOperation '%s', pendingPath=%s, pendingQuery='%s'", - pendOp.operation, pendOp.watchPath, pendOp.pathQuery) - qw.parentPathW.parentStrat.processWatchClosure(pendOp) - case "send": - qw.logf("fsnotify.opLoop", "pendingOperation '%s', redactDsn='%s'", - pendOp.operation, pendOp.redactDsn) - qw.sendUpdate(pendOp.dsn, pendOp.redactDsn) - default: - qw.logf("fsnotify.opLoop", "ignore invalid pendingOperation '%s'", - pendOp.operation) - } + case qw.updateChan <- op.dsn: + qw.logf("fsnotify.opLoop", "successfully sent redactDsn='%s'", op.redactDsn) + case <-qw.done: + qw.logf("fsnotify.opLoop", "watch closed while sending redactDsn='%s'", op.redactDsn) } } + close(qw.updateChan) + qw.logf("fsnotify.opLoop", "operChan closed, terminating") } func (s *Strategy) logf(prefix, format string, args ...any) { @@ -351,13 +353,3 @@ func (s *Strategy) errlogf(prefix, format string, args ...any) { logPrefix := fmt.Sprintf("%s:", prefix) logger.ErrLogf(logPrefix, format, args...) } - -func (pw *pathWatch) errlogf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s]:", prefix, pw.watchPath) - logger.ErrLogf(logPrefix, format, args...) -} - -func (qw *queryWatch) errlogf(prefix, format string, args ...any) { - logPrefix := fmt.Sprintf("%s[%s?%s]:", prefix, qw.parentPathW.watchPath, qw.pathQuery) - logger.ErrLogf(logPrefix, format, args...) -} diff --git a/fsnotify/filewatcher_test.go b/fsnotify/filewatcher_test.go index 2f9e2ca..6a06957 100644 --- a/fsnotify/filewatcher_test.go +++ b/fsnotify/filewatcher_test.go @@ -2,292 +2,360 @@ package fsnotify import ( "context" - "fmt" - "net/url" "os" + "path/filepath" + "testing" "time" - - rfsnotify "github.com/fsnotify/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) -func assertStringFromChannel(name string, want string, from <-chan string) { -again: - select { - case got := <-from: - fmt.Printf("assertStringFromChannel: expecting '%s', got update: '%s'\n", want, got) - if got == want { - return - } else { - // dedup fsnotify WRITE events - goto again +// newTestStrategy returns a strategy with a short resync period (the +// recovery path after failed re-watches). Watches are tied to per-test +// contexts (see testCtx), so cleanup happens by cancellation. +func newTestStrategy(t *testing.T) *Strategy { + t.Helper() + s := NewStrategy() + s.resyncPeriod = 50 * time.Millisecond + return s +} + +// testCtx returns a context canceled when the test ends, closing every +// watch established with it. +func testCtx(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// awaitValue reads updates until want arrives, tolerating duplicate +// notifications (fsnotify can deliver several events per change). +func awaitValue(t *testing.T, ch <-chan string, want string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case got, ok := <-ch: + if !ok { + t.Fatalf("update channel closed while waiting for %q", want) + } + t.Logf("update: %q", got) + if got == want { + return + } + case <-deadline: + t.Fatalf("timed out waiting for value %q", want) } - case <-time.After(resyncPeriod * 2): - Fail(fmt.Sprintf("%s: timeout: expecting '%s'", name, want)) } } -type args struct { - pth string - options url.Values +// awaitClosed waits for the update channel to be closed. +func awaitClosed(t *testing.T, ch <-chan string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("timed out waiting for update channel to close") + } + } } -type test struct { - name string - setup func(*args) - args args - wantErr bool - post func(args *args, value string, values <-chan string) error - tearDown func(*args) +func TestWatchMissingFile(t *testing.T) { + s := newTestStrategy(t) + _, _, err := s.Watch(testCtx(t), filepath.Join(t.TempDir(), "missing"), "") + if err == nil { + t.Fatal("expected an error watching a missing file") + } } -type testWatcher struct { - eventChannel chan rfsnotify.Event - paths map[string]bool - closed bool - errors chan error +func TestWatchInitialValueTrimmed(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, " dsn-1 \n") + + value, _, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + if value != "dsn-1" { + t.Errorf("initial value = %q, want %q (whitespace must be trimmed)", value, "dsn-1") + } } -func newTestWatcher() *testWatcher { - return &testWatcher{ - eventChannel: make(chan rfsnotify.Event), - paths: make(map[string]bool), - errors: make(chan error), - closed: false, +func TestWatchSeesWrites(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + _, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) } + + writeFile(t, p, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") + + writeFile(t, p, "dsn-3") + awaitValue(t, w.Values(), "dsn-3") } -func (tw *testWatcher) Add(s string) error { - tw.paths[s] = true - return nil +// TestWatchSeesAtomicRename covers the write-then-rename pattern used by +// most config writers (and by viper, kustomize, etc.). +func TestWatchSeesAtomicRename(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + p := filepath.Join(dir, "config") + writeFile(t, p, "dsn-1") + + _, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + + tmp := filepath.Join(dir, "config.tmp") + writeFile(t, tmp, "dsn-2") + if err := os.Rename(tmp, p); err != nil { + t.Fatal(err) + } + awaitValue(t, w.Values(), "dsn-2") } -func (tw *testWatcher) Remove(s string) error { - tw.paths[s] = false - return nil +// TestWatchRecoversFromRemoveAndRecreate: deleting the file fails the +// resync; the periodic retry must pick the value up once the file returns. +func TestWatchRecoversFromRemoveAndRecreate(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + p := filepath.Join(dir, "config") + writeFile(t, p, "dsn-1") + + _, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + writeFile(t, p, "dsn-2") + awaitValue(t, w.Values(), "dsn-2") } -func (tw *testWatcher) Close() error { - tw.closed = true - return nil +// TestWatchSeesKubernetesConfigMapSwap reproduces the kubelet's ConfigMap +// update dance: the watched path is a symlink chain through a `..data` +// symlink that is swapped atomically to a new timestamped directory. +func TestWatchSeesKubernetesConfigMapSwap(t *testing.T) { + s := newTestStrategy(t) + dir := t.TempDir() + + tsDir1 := filepath.Join(dir, "..2026_06_09_01") + if err := os.Mkdir(tsDir1, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(tsDir1, "config"), "dsn-1") + + dataLink := filepath.Join(dir, "..data") + if err := os.Symlink(tsDir1, dataLink); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "config") + if err := os.Symlink(filepath.Join(dataLink, "config"), p); err != nil { + t.Fatal(err) + } + + value, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + if value != "dsn-1" { + t.Fatalf("initial value = %q, want dsn-1", value) + } + + // The swap: new timestamped dir, retarget ..data atomically via rename, + // remove the old dir. + tsDir2 := filepath.Join(dir, "..2026_06_09_02") + if err := os.Mkdir(tsDir2, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(tsDir2, "config"), "dsn-2") + + tmpLink := filepath.Join(dir, "..data_tmp") + if err := os.Symlink(tsDir2, tmpLink); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmpLink, dataLink); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(tsDir1); err != nil { + t.Fatal(err) + } + + awaitValue(t, w.Values(), "dsn-2") } -func (tw *testWatcher) GetEvents() <-chan rfsnotify.Event { - return tw.eventChannel +// TestMultipleWatchersOnePath: watches established by separate Watch calls +// get independent channels fed from one underlying file watch — even for an +// identical path and query (two hotload DSNs can differ only in the driver +// component). +func TestMultipleWatchersOnePath(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, p, "forceKill=true") + if err != nil { + t.Fatal(err) + } + _, w2, err := s.Watch(ctx, p, "forceKill=true") + if err != nil { + t.Fatal(err) + } + + writeFile(t, p, "dsn-2") + awaitValue(t, w1.Values(), "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") } -func (tw *testWatcher) GetErrors() <-chan error { - return tw.errors +func TestCloseClosesChannel(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + _, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + awaitClosed(t, w.Values()) + + // Close is idempotent. + if err := w.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } } -var _ = Describe("FileWatcher", func() { - const ( - paramsURL = "postgres://login:password@host:1234/database?sslmode=disable" - paramsParsed = "host=a login=b password=c" - ) +// TestCloseKeepsOtherWatches: closing one watch leaves the other subscriber +// of the same path working. +func TestCloseKeepsOtherWatches(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") - s := NewStrategy() - DescribeTable("Watch", - func(tt test) { - if tt.setup != nil { - tt.setup(&tt.args) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - gotValue, gotValues, err := s.Watch(ctx, tt.args.pth, tt.args.options.Encode()) - if (err != nil) != tt.wantErr { - Expect(err).To(HaveOccurred()) - return - } - if tt.post != nil { - if err := tt.post(&tt.args, gotValue, gotValues); err != nil { - Expect(err).ToNot(HaveOccurred()) - } - } - if tt.tearDown != nil { - tt.tearDown(&tt.args) - } + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, p, "q=1") + if err != nil { + t.Fatal(err) + } + _, w2, err := s.Watch(ctx, p, "q=2") + if err != nil { + t.Fatal(err) + } + + if err := w1.Close(); err != nil { + t.Fatal(err) + } + awaitClosed(t, w1.Values()) + + writeFile(t, p, "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") +} + +// TestCtxCancelClosesWatch: canceling the Watch context releases the watch, +// exactly like Close. +func TestCtxCancelClosesWatch(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") - _ = s.CloseWatch(tt.args.pth, tt.args.options.Encode()) - time.Sleep(10 * time.Millisecond) - }, - Entry("file not found", test{ - args: args{ - pth: "somefile does not exist", - }, - wantErr: true, - }), - Entry("URL surrounded with whitespaces --> URL trimmed", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("\r\n \t " + paramsURL + " \t \r\n")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != paramsURL { - return fmt.Errorf("expected '"+paramsURL+"' got %v", value) - } - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("params surrounded with whitespaces --> params trimmed", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("\r\n \t " + paramsParsed + " \t \r\n")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != paramsParsed { - return fmt.Errorf("expected '"+paramsParsed+"' got %v", value) - } - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("a, update b", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - os.WriteFile(args.pth, []byte("b"), 0660) - assertStringFromChannel("waiting for update b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("extra slash in path", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = "/" + f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - os.WriteFile(args.pth, []byte("b"), 0660) - assertStringFromChannel("waiting for update b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - Entry("a, rm a, create b", test{ - setup: func(args *args) { - f, _ := os.CreateTemp("", "unittest_") - f.Write([]byte("a")) - args.pth = f.Name() - f.Close() - }, - wantErr: false, - post: func(args *args, value string, values <-chan string) error { - if value != "a" { - return fmt.Errorf("expected 'a' got %v", value) - } - err := os.Remove(args.pth) - Expect(err).ToNot(HaveOccurred(), "removing config file") - - select { - case v := <-values: - return fmt.Errorf("expected no change, got %v", v) - case <-time.After(time.Second): - } - err = os.WriteFile(args.pth, []byte("b"), 0660) - - Expect(err).ToNot(HaveOccurred(), "creating new file") - - assertStringFromChannel("waiting for create b", "b", values) - return nil - }, - tearDown: func(args *args) { - os.Remove(args.pth) - }, - }), - ) - - Context("run", func() { - var strat *Strategy - var watcher *testWatcher - BeforeEach(func() { - strat = NewStrategy() - watcher = newTestWatcher() - strat.watcher = watcher - }) - It("Should not respond to chmod events", func() { - // add only a bad path to the testWatcher - // This path should not end up removed from the map, ie, marked 'false' - // we'll pass a CHMOD event and verify the 'bad path' is still in the paths map - bp := "badpath" - strat.watcher.Add(bp) - go s.runLoop() - go func() { - watcher.eventChannel <- rfsnotify.Event{ - Name: "chaff", - Op: rfsnotify.Chmod, - } - }() - time.Sleep(1 * time.Millisecond) - _, v := watcher.paths[bp] - // run didn't pass through resync - Expect(v).To(BeTrue()) - }) - }) - - Context("Two watches with same path but diff query-params", func() { - It("Should update channels for both watches", func() { - f, _ := os.CreateTemp("", "hotload_fsnotify_filewatcher_two_watches_unittest_") - f.Write([]byte("_")) - watchPath := f.Name() - f.Close() - defer os.Remove(watchPath) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - gotValue1, updateChan1, err := s.Watch(ctx, watchPath, "") - Expect(err).ToNot(HaveOccurred()) - Expect(updateChan1).ToNot(BeNil()) - Expect(gotValue1).To(Equal("_")) - - urlValues := url.Values{"forceKill": []string{"true"}} - gotValue2, updateChan2, err := s.Watch(ctx, watchPath, urlValues.Encode()) - Expect(err).ToNot(HaveOccurred()) - Expect(updateChan2).ToNot(BeNil()) - Expect(gotValue2).To(Equal("_")) - - os.WriteFile(watchPath, []byte("a"), 0666) - assertStringFromChannel("updateChan1 waiting for update a", "a", updateChan1) - assertStringFromChannel("updateChan2 waiting for update a", "a", updateChan2) - - os.WriteFile(watchPath, []byte("b"), 0666) - assertStringFromChannel("updateChan1 waiting for update b", "b", updateChan1) - assertStringFromChannel("updateChan2 waiting for update b", "b", updateChan2) - - _ = s.CloseWatch(watchPath, "") - _ = s.CloseWatch(watchPath, urlValues.Encode()) - time.Sleep(10 * time.Millisecond) - }) - }) -}) + ctx, cancel := context.WithCancel(context.Background()) + _, w, err := s.Watch(ctx, p, "") + if err != nil { + t.Fatal(err) + } + cancel() + awaitClosed(t, w.Values()) +} + +// TestWatchAfterAllClosed: closing the last watch releases the internal +// watcher; a later Watch must re-initialize it (the registered global +// instance lives for the process). +func TestWatchAfterAllClosed(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + _, w, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + + value, w2, err := s.Watch(testCtx(t), p, "") + if err != nil { + t.Fatalf("Watch after last Close: %v", err) + } + if value != "dsn-1" { + t.Errorf("value = %q, want dsn-1", value) + } + writeFile(t, p, "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") +} + +// TestAbandonedSubscriberDoesNotWedgeStrategy: if a subscriber stops +// receiving (its hotload group is gone), pending updates must not block the +// strategy's delivery or the watch's Close path. +func TestAbandonedSubscriberDoesNotWedgeStrategy(t *testing.T) { + s := newTestStrategy(t) + p := filepath.Join(t.TempDir(), "config") + writeFile(t, p, "dsn-1") + + ctx := testCtx(t) + _, abandoned, err := s.Watch(ctx, p, "q=abandoned") + if err != nil { + t.Fatal(err) + } + // abandoned is never read from. + + _, live, err := s.Watch(ctx, p, "q=live") + if err != nil { + t.Fatal(err) + } + + // Generate more updates than the abandoned watch's queue can hold. + for i := 2; i < 40; i++ { + writeFile(t, p, "dsn-x") + writeFile(t, p, "dsn-2") + } + awaitValue(t, live.Values(), "dsn-2") + + // Closing the abandoned watch must not deadlock. + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + if err := abandoned.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }() + select { + case <-doneCh: + case <-time.After(5 * time.Second): + t.Fatal("Close deadlocked on an abandoned subscriber") + } + awaitClosed(t, abandoned.Values()) +} diff --git a/fsnotify/fsnotify_suite_test.go b/fsnotify/fsnotify_suite_test.go deleted file mode 100644 index 010fed9..0000000 --- a/fsnotify/fsnotify_suite_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package fsnotify - -import ( - "log" - "testing" - - "github.com/infobloxopen/hotload/logger" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestFsnotify(t *testing.T) { - //log.SetFlags(log.Flags() | log.Lmicroseconds) - log.SetFlags(log.Ltime | log.Lmicroseconds) - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - RegisterFailHandler(Fail) - RunSpecs(t, "Fsnotify Suite") -} diff --git a/go.mod b/go.mod index 524ba35..e8799b3 100644 --- a/go.mod +++ b/go.mod @@ -1,41 +1,7 @@ -module github.com/infobloxopen/hotload +module github.com/infobloxopen/hotload/v3 go 1.23.0 -toolchain go1.24.6 +require github.com/fsnotify/fsnotify v1.6.0 -require ( - github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/colega/gaugefuncvec v0.1.0 - github.com/fsnotify/fsnotify v1.6.0 - github.com/google/uuid v1.6.0 - github.com/lib/pq v1.10.8 - github.com/onsi/ginkgo/v2 v2.25.3 - github.com/onsi/gomega v1.38.2 - github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.0 - github.com/prometheus/common v0.55.0 - github.com/teivah/onecontext v1.3.0 -) - -require ( - github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.36.0 // indirect - google.golang.org/protobuf v1.36.7 // indirect -) +require golang.org/x/sys v0.35.0 // indirect diff --git a/go.sum b/go.sum index 9500385..d67f982 100644 --- a/go.sum +++ b/go.sum @@ -1,167 +1,5 @@ -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/colega/gaugefuncvec v0.1.0 h1:ocD7PAmGxioCVM6mlYvbSpwB+rYCcvgdFbEuLfNm35Y= -github.com/colega/gaugefuncvec v0.1.0/go.mod h1:uXDw0W/fhrHeWsaRNEJgtJL7hYa+vFZ5oQH5MTN6+nE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= -github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= -github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= -github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= -github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= -github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/teivah/onecontext v1.3.0 h1:tbikMhAlo6VhAuEGCvhc8HlTnpX4xTNPTOseWuhO1J0= -github.com/teivah/onecontext v1.3.0/go.mod h1:hoW1nmdPVK/0jrvGtcx8sCKYs2PiS4z0zzfdeuEVyb0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work new file mode 100644 index 0000000..512edb2 --- /dev/null +++ b/go.work @@ -0,0 +1,8 @@ +go 1.23.0 + +use ( + . + ./k8ssecret + ./observability + ./test/integration +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..8fca17f --- /dev/null +++ b/go.work.sum @@ -0,0 +1,33 @@ +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= diff --git a/group.go b/group.go new file mode 100644 index 0000000..d42daef --- /dev/null +++ b/group.go @@ -0,0 +1,318 @@ +package hotload + +import ( + "context" + "database/sql/driver" + "fmt" + "net/url" + "sync" + "sync/atomic" + "time" + + "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/logger" +) + +// DefaultKillWindow is how long a killed generation waits for in-flight +// operations to observe cancellation before their connections are +// force-closed. Override per DSN with the killWindow query parameter. +const DefaultKillWindow = 100 * time.Millisecond + +// generation owns everything whose lifetime matches one value of the +// connection string: the cancellation context, the set of live connections, +// and the drained flag consulted by ResetSession/IsValid. A config change +// swaps generation pointers on the group; nothing else has to be mutated. +type generation struct { + dsn string + redactDsn string + + // ctx is derived from the group's parent context and canceled with + // cause ErrHotSwap when the generation is killed. + ctx context.Context + cancel context.CancelCauseFunc + + // drained marks the generation retired without canceling in-flight + // work; conns answer driver.ErrBadConn on the next pool reuse. + drained atomic.Bool + + // ops counts operations running under opCtx, so kill can wait + // (bounded) for them to observe cancellation before force-closing. + // A plain atomic (rather than a WaitGroup) because operations keep + // starting while kill waits, which WaitGroup forbids. + ops atomic.Int64 + + // mu is a leaf lock guarding conns only. No driver call, hook, or + // other lock acquisition ever happens while holding it. + mu sync.Mutex + conns map[*baseConn]struct{} + + // connector lazily caches the underlying driver's connector when it + // implements driver.DriverContext. + connOnce sync.Once + connector driver.Connector + connErr error +} + +func newGeneration(parent context.Context, dsn string) *generation { + ctx, cancel := context.WithCancelCause(parent) + return &generation{ + dsn: dsn, + redactDsn: internal.RedactUrl(dsn), + ctx: ctx, + cancel: cancel, + conns: make(map[*baseConn]struct{}), + } +} + +// retired reports whether conns of this generation should be discarded by +// the pool. +func (gen *generation) retired() bool { + return gen.drained.Load() || gen.ctx.Err() != nil +} + +func (gen *generation) add(c *baseConn) { + gen.mu.Lock() + defer gen.mu.Unlock() + gen.conns[c] = struct{}{} +} + +func (gen *generation) remove(c *baseConn) { + gen.mu.Lock() + defer gen.mu.Unlock() + delete(gen.conns, c) +} + +// drain retires the generation gracefully: in-flight work continues, and +// the pool discards each conn the next time it tries to reuse it. +func (gen *generation) drain() { + gen.drained.Store(true) +} + +// kill retires the generation immediately: the generation context is +// canceled (with cause ErrHotSwap), in-flight operations get up to window to +// observe the cancellation, and then every remaining conn is force-closed. +// The bounded wait guarantees the caller (the group run loop) cannot be +// wedged by a driver that ignores context cancellation. +func (gen *generation) kill(window time.Duration) { + gen.drained.Store(true) + gen.cancel(ErrHotSwap) + gen.awaitIdle(window) + + gen.mu.Lock() + snap := make([]*baseConn, 0, len(gen.conns)) + for c := range gen.conns { + snap = append(snap, c) + } + gen.mu.Unlock() + + for _, c := range snap { + c.closeConn(true) + } +} + +// awaitIdle polls until no operations are in flight or the window elapses. +// Well-behaved drivers observe the canceled context within microseconds; +// drivers that ignore cancellation are unblocked by the conn close that +// follows in kill. +func (gen *generation) awaitIdle(window time.Duration) { + deadline := time.Now().Add(window) + for gen.ops.Load() > 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } +} + +// group represents one watched hotload DSN. It receives new connection +// string values from the strategy and swaps generations in response. Its +// mutex guards only the generation pointers and the closed flag; all driver +// calls, hook emissions and strategy calls happen outside it. +type group struct { + name string // the full hotload DSN + strategyName string + path string + sqlDriver *driverInstance + forceKill bool + killWindow time.Duration + parentCtx context.Context + parentCancel context.CancelFunc + watch Watchable + + // refs and pinned are guarded by hdriver.mu. refs counts live + // connectors; pinned marks groups created through the legacy + // driver.Open path, which have no teardown signal and live forever. + refs int + pinned bool + + mu sync.Mutex + cur *generation + prev *generation + closed bool +} + +// runLoop receives new connection string values from the strategy until the +// group is shut down or the strategy closes the channel. It is the only +// goroutine that swaps generations. +func (g *group) runLoop() { + updates := g.watch.Values() + for { + select { + case <-g.parentCtx.Done(): + g.logf("group.runLoop", "parent context done, terminating") + return + case newValue, ok := <-updates: + if !ok { + g.logf("group.runLoop", "strategy channel closed, terminating") + EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) + return + } + g.onNewValue(newValue) + } + } +} + +func (g *group) onNewValue(v string) { + g.mu.Lock() + if g.closed || v == g.cur.dsn { + g.mu.Unlock() + g.logf("group.onNewValue", "conn dsn not changed") + return + } + old, older := g.cur, g.prev + next := newGeneration(g.parentCtx, v) + g.cur = next + if g.forceKill { + g.prev = nil + } else { + g.prev = old + } + g.mu.Unlock() + + g.logf("group.onNewValue", "conn dsn changed: '%s' -> '%s'", old.redactDsn, next.redactDsn) + emitConfigChange(ConfigChangeEvent{ + GroupName: g.name, + OldRedactedDSN: old.redactDsn, + NewRedactedDSN: next.redactDsn, + ForceKill: g.forceKill, + At: time.Now(), + }) + + if g.forceKill { + // Cancel in-flight work on the old DSN and close its conns. + old.kill(g.killWindow) + return + } + // Graceful: the old generation keeps serving in-flight work and drains + // through the pool; the generation before it has had its grace period + // and is killed now. + old.drain() + if older != nil { + older.kill(g.killWindow) + } +} + +// conn dials a new connection on the current generation. The underlying +// dial happens outside all locks; if a config change lands mid-dial the new +// conn belongs to an already-retired generation and self-evicts on first +// pool reuse, which the pool handles by dialing again. +func (g *group) conn(ctx context.Context) (driver.Conn, error) { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return nil, fmt.Errorf("hotload: group %q is closed", g.name) + } + gen := g.cur + g.mu.Unlock() + + dsn, err := mergeConnStringOptions(gen.dsn, g.sqlDriver.options) + if err != nil { + return nil, err + } + redactDsn := gen.redactDsn + if dsn != gen.dsn { + // Driver options changed the DSN; redact the actual dial string. + redactDsn = internal.RedactUrl(dsn) + } + inner, err := g.dial(ctx, gen, dsn) + if err != nil { + return nil, err + } + + c := &baseConn{ + inner: inner, + gen: gen, + group: g, + dsn: dsn, + redactDsn: redactDsn, + } + gen.add(c) + emitConnOpen(ConnEvent{GroupName: g.name, RedactedDSN: c.redactDsn}) + g.logf("group.conn", "opened managed conn: '%s'", c.redactDsn) + return wrapConn(c), nil +} + +// dial opens an underlying connection, using the underlying driver's +// connector when it implements driver.DriverContext (so the dial honors +// ctx), and falling back to Open otherwise — the same split database/sql +// applies. +func (g *group) dial(ctx context.Context, gen *generation, dsn string) (driver.Conn, error) { + dc, ok := g.sqlDriver.driver.(driver.DriverContext) + if !ok { + return g.sqlDriver.driver.Open(dsn) + } + gen.connOnce.Do(func() { + gen.connector, gen.connErr = dc.OpenConnector(dsn) + }) + if gen.connErr != nil { + return nil, gen.connErr + } + return gen.connector.Connect(ctx) +} + +// closeWatch marks the group closed and closes its strategy watch. It is +// called by hdriver.releaseGroup with hdriver.mu held, which serializes it +// against getGroup's strategy.Watch calls (see releaseGroup). Watchable +// implementations must therefore never call back into hotload from Close. +func (g *group) closeWatch() { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return + } + g.closed = true + g.mu.Unlock() + + if err := g.watch.Close(); err != nil { + g.logf("group.closeWatch", "watch close error: %v", err) + } +} + +// finishShutdown completes the teardown started by closeWatch: the parent +// context cancel terminates runLoop and cancels every generation context. +// Runs outside all locks so hook callbacks cannot deadlock. +func (g *group) finishShutdown() { + g.parentCancel() + EmitWatchEvent(WatchEvent{GroupName: g.name, Strategy: g.strategyName, Path: g.path, Closed: true}) + g.logf("group.finishShutdown", "group closed") +} + +func mergeConnStringOptions(dsn string, options map[string]string) (string, error) { + if len(options) == 0 { + return dsn, nil + } + u, err := url.ParseRequestURI(dsn) + if err != nil { + return "", fmt.Errorf("unable to parse connection string when specifying extra driver options: %v", err) + } + values, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", fmt.Errorf("unable to parse query options in connection string when specifying extra driver options: %v", err) + } + for k, v := range options { + values.Set(k, v) + } + u.RawQuery = values.Encode() + return u.String(), nil +} + +func (g *group) logf(prefix, format string, args ...any) { + logger.Logf(fmt.Sprintf("%s[%s]:", prefix, g.name), format, args...) +} diff --git a/hotload_suite_test.go b/hotload_suite_test.go deleted file mode 100644 index 06cd01f..0000000 --- a/hotload_suite_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package hotload_test - -import ( - "log" - "testing" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestHotload(t *testing.T) { - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - nrr := internal.NewNonRandomReader(1) - uuid.SetRand(nrr) - - RegisterFailHandler(Fail) - RunSpecs(t, "Hotload Suite") -} diff --git a/integrationtests/context_test.go b/integrationtests/context_test.go deleted file mode 100644 index 05e58e2..0000000 --- a/integrationtests/context_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("postgres-direct (non-hotload) context test (verify underlying postgres driver is correct)", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - userDb = openDbPostgres(hldatabaseSuperDsn) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - userDb.Close() - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=false)", func(ctx context.Context) { - callerCancelContextTestFn(false, superDb, userDb) - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=true)", func(ctx context.Context) { - callerCancelContextTestFn(true, superDb, userDb) - }) -}) - -var _ = Describe("hotload context test", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - userDb = openDbHotload(false) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - userDb.Close() - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=false)", func(ctx context.Context) { - callerCancelContextTestFn(false, superDb, userDb) - }) - - It("Cancel ExecContext, rowcount and error return should be consistent (forceKill=true)", func(ctx context.Context) { - callerCancelContextTestFn(true, superDb, userDb) - }) -}) - -func callerCancelContextTestFn(forceKill bool, superDb, userDb *sql.DB) { - callerCtx, cancelFn := context.WithCancel(context.Background()) - csource := "caller-cancel-exec-context" - cnum := 141421 - insertStmt := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(1.0))", cnum, csource) - - By("Insert 1st row") - _, err := userDb.ExecContext(callerCtx, insertStmt) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("error inserting into table test")) - - // Long-running fn to insert 2nd row - errChan := make(chan error) - longExecContextFn := func(errChan chan error) { - log.Printf("start db.ExecContext INSERT PG_SLEEP") - result, err := userDb.ExecContext(callerCtx, insertStmt) - if err == nil { - log.Printf("db.ExecContext INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.ExecContext INSERT PG_SLEEP error=%+v", err) - } - errChan <- err - } - - By("Spawn long-running db background thread") - go longExecContextFn(errChan) - - By("Momentarily sleep/yield to long-running db background thread") - time.Sleep(100 * time.Millisecond) - - By("Cancel context of long-running db background thread") - cancelFn() - time.Sleep(1 * time.Millisecond) - - if forceKill { - By("Close connection of long-running db background thread") - userDb.Close() - } - - By("Wait for return from long-running db background thread") - err = <-errChan - log.Printf("returned from long-running db background thread, err=%v", err) - By("Got return from long-running db background thread") - Expect(err).To(HaveOccurred(), fmt.Sprintf("expect error inserting into table test")) - - expectRowCountInDb(superDb, csource, false, 1, int64(cnum)) -} diff --git a/integrationtests/docker/docker-compose.yaml b/integrationtests/docker/docker-compose.yaml deleted file mode 100644 index 3c4d594..0000000 --- a/integrationtests/docker/docker-compose.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' - -# postgres configuration mimics helm postgres configured in ../helm/hotload-integration-tests/values.yaml -services: - db: - image: postgres:10.3 - environment: - - POSTGRES_USER=admin - - POSTGRES_PASSWORD=test - - POSTGRES_DB=hldatabase - - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true - ports: - - '5432:5432' - volumes: - # https://github.com/felipewom/docker-compose-postgres - # https://geshan.com.np/blog/2021/12/docker-postgres/ - # https://github.com/docker-library/docs/tree/master/postgres#initialization-scripts - - ./intgtest_init.sql:/docker-entrypoint-initdb.d/intgtest_init.sql diff --git a/integrationtests/helm/hotload-integration-tests/.helmignore b/integrationtests/helm/hotload-integration-tests/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/integrationtests/helm/hotload-integration-tests/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/integrationtests/helm/hotload-integration-tests/Chart.yaml b/integrationtests/helm/hotload-integration-tests/Chart.yaml deleted file mode 100644 index 78d5b1f..0000000 --- a/integrationtests/helm/hotload-integration-tests/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: hotload-integration-tests -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -# No external dependencies - using official PostgreSQL container directly \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl b/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl deleted file mode 100644 index 95b37ce..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "hotload-integration-tests.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "hotload-integration-tests.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "hotload-integration-tests.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "hotload-integration-tests.labels" -}} -helm.sh/chart: {{ include "hotload-integration-tests.chart" . }} -{{ include "hotload-integration-tests.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hotload-integration-tests.selectorLabels" -}} -app.kubernetes.io/name: {{ include "hotload-integration-tests.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "hotload-integration-tests.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "hotload-integration-tests.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml deleted file mode 100644 index 7b5368c..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-configmap.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.name }}-postgres-initdb - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -data: - init.sql: {{ .Values.postgres.initScript | quote }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml deleted file mode 100644 index 4150306..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-deployment.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}-postgresql - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 6 }} - template: - metadata: - labels: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 8 }} - spec: - containers: - - name: postgres - image: {{ .Values.postgres.image }} - ports: - - containerPort: 5432 - name: postgres - envFrom: - - secretRef: - name: {{ .Values.name }}-postgres-secret - volumeMounts: - - name: initdb - mountPath: /docker-entrypoint-initdb.d - readOnly: true - {{- if not .Values.postgres.persistence.enabled }} - - name: postgres-data - mountPath: /var/lib/postgresql/data - {{- end }} - livenessProbe: - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U {{ .Values.postgres.user }} -d {{ .Values.postgres.database }} -h 127.0.0.1 -p 5432 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U {{ .Values.postgres.user }} -d {{ .Values.postgres.database }} -h 127.0.0.1 -p 5432 - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - volumes: - - name: initdb - configMap: - name: {{ .Values.name }}-postgres-initdb - {{- if not .Values.postgres.persistence.enabled }} - - name: postgres-data - emptyDir: {} - {{- end }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml deleted file mode 100644 index 5b1c5db..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.name }}-postgres-secret - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -type: Opaque -data: - POSTGRES_USER: {{ .Values.postgres.user | b64enc | quote }} - POSTGRES_PASSWORD: {{ .Values.postgres.password | b64enc | quote }} - POSTGRES_DB: {{ .Values.postgres.database | b64enc | quote }} - postgres-password: {{ .Values.postgres.postgresPassword | b64enc | quote }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml b/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml deleted file mode 100644 index 14474b4..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/postgres-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.name }}-postgresql - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: postgres - protocol: TCP - name: postgres - selector: - app: postgresql -{{ include "hotload-integration-tests.selectorLabels" . | indent 4 }} \ No newline at end of file diff --git a/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml b/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml deleted file mode 100644 index afe85d6..0000000 --- a/integrationtests/helm/hotload-integration-tests/templates/tests/test-hotload.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ .Values.name }}-job" - labels: -{{ include "hotload-integration-tests.labels" . | indent 4 }} - annotations: - "helm.sh/hook": test-success -spec: - containers: - - name: "hotload-integration-tests" - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: Never - command: ["./integrationtests.test"] - env: - - name: HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST - value: hotload-integration-tests-postgresql.default.svc.cluster.local - - name: HOTLOAD_PATH_CHKSUM_METRICS_ENABLE - value: "true" - restartPolicy: Never diff --git a/integrationtests/helm/hotload-integration-tests/values.yaml b/integrationtests/helm/hotload-integration-tests/values.yaml deleted file mode 100644 index 5ecd906..0000000 --- a/integrationtests/helm/hotload-integration-tests/values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Default values for hotload-integration-tests. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -name: hotload-integration-tests - -image: - repository: hotload-integration-tests - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -# postgres configuration using official postgres:10 image -postgres: - image: postgres:10 - user: admin - password: test - database: hldatabase - postgresPassword: postgres # password for 'postgres' superuser - persistence: - enabled: false - initScript: | - CREATE DATABASE hotload_test; - CREATE DATABASE hotload_test1; - GRANT ALL PRIVILEGES ON DATABASE hotload_test TO admin; - GRANT ALL PRIVILEGES ON DATABASE hotload_test1 TO admin; diff --git a/integrationtests/hotload_test.go b/integrationtests/hotload_test.go deleted file mode 100644 index 475223f..0000000 --- a/integrationtests/hotload_test.go +++ /dev/null @@ -1,314 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "os" - "strings" - "time" - - "github.com/infobloxopen/hotload" - _ "github.com/infobloxopen/hotload/fsnotify" - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" - "github.com/infobloxopen/hotload/modtime" - "github.com/lib/pq" - _ "github.com/lib/pq" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -const ( - fsnotifyStrategy = "fsnotify" - configPath = "/tmp/hotload_integration_test_dsn_config.txt" - userSqlSetup = "CREATE USER uuser WITH PASSWORD 'ppass1'" - userSqlTeardown = "DROP USER IF EXISTS uuser" - testSqlSetup = "CREATE TABLE IF NOT EXISTS test (cnum INT, csource TEXT, csleep TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)" - testSqlTeardown = "DROP TABLE IF EXISTS test" - truncSqlSetup = "TRUNCATE TABLE test" - initSqlSetup = "INSERT INTO test (cnum, csource) VALUES (161803, 'initial')" - grantSqlSetup = "GRANT ALL ON test TO PUBLIC" - - setDSNSleepDur = 100 * time.Millisecond -) - -var ( - mtmCtx context.Context - mtmCancelCtxFn context.CancelFunc - mtm *modtime.ModTimeMonitor -) - -func init() { - // this function call registers the lib/pq postgres driver with hotload - hotload.RegisterSQLDriver("postgres", &pq.Driver{}) -} - -func formHotloadDsn(forceKill bool) string { - dsnUrl := "fsnotify://postgres" + configPath - if forceKill { - dsnUrl = dsnUrl + "?forceKill=true" - } - return dsnUrl -} - -func setDSN(dsn string, path string) { - err := os.WriteFile(path, []byte(dsn), 0777) - if err != nil { - Fail(fmt.Sprintf("setDSN: error writing dsn file: %v", err)) - } - log.Printf("setDSN: success writing '%s' to '%s'", dsn, path) - - // Yield thread to let switch over take place - time.Sleep(setDSNSleepDur) - log.Printf("setDSN: slept/yielded %s", setDSNSleepDur) -} - -// Open a db using postgres driver, or die -func openDbPostgres(dsn string) *sql.DB { - db, err := sql.Open("postgres", dsn) - if err != nil { - Fail(fmt.Sprintf("openDbPostgres: error opening db dsn '%s': %v", dsn, err)) - } - log.Printf("openDbPostgres: opened db dsn '%s'", dsn) - - return db -} - -// Open a db using hotload driver, or die -func openDbHotload(forceKill bool) *sql.DB { - dsnUrl := formHotloadDsn(forceKill) - - db, err := sql.Open("hotload", dsnUrl) - if err != nil { - Fail(fmt.Sprintf("openDbHotload: err opening db dsn '%s': %v", dsnUrl, err)) - } - log.Printf("openDbHotload: opened hotload dsn '%s'", dsnUrl) - - err = db.Ping() - if err != nil { - Fail(fmt.Sprintf("openDbHotload: err pinging db dsn '%s': %v", dsnUrl, err)) - } - log.Printf("openDbHotload: pinged hotload dsn '%s'", dsnUrl) - - return db -} - -func expectValueInDb(db *sql.DB, source string, expErr bool, expRowCount int, expVal int64) { - GinkgoHelper() - log.Printf("expectValueInDb: expErr=%v, expRowCount=%d, expVal=%d", expErr, expRowCount, expVal) - r, err := db.Query(fmt.Sprintf("SELECT cnum FROM test WHERE csource = '%s'", source)) - var cnum int64 - if expErr { - Expect(err).To(HaveOccurred(), fmt.Sprintf("expectValueInDb: expect error reading from table test")) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: error reading from table test: %v", err)) - } - gotRowCount := 0 - if !r.Next() { - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: cursor iteration err: %v", r.Err())) - } else { - gotRowCount = 1 - err = r.Scan(&cnum) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectValueInDb: error calling r.Scan(): %v", err)) - Expect(cnum).To(Equal(expVal)) - } - Expect(gotRowCount).To(Equal(expRowCount)) -} - -func expectRowCountInDb(db *sql.DB, source string, expErr bool, expRowCount int, expVal int64) { - GinkgoHelper() - log.Printf("expectRowCountInDb: expErr=%v, expRowCount=%d, expVal=%d", expErr, expRowCount, expVal) - r, err := db.Query(fmt.Sprintf("SELECT COUNT(*) FROM test WHERE csource = '%s' AND cnum = %d", source, expVal)) - if expErr { - Expect(err).To(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: expect error reading from table test")) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: error reading from table test: %v", err)) - } - if !r.Next() { - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: cursor iteration err: %v", r.Err())) - } else { - var gotRowCount int - err = r.Scan(&gotRowCount) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectRowCountInDb: error calling r.Scan(): %v", err)) - Expect(gotRowCount).To(Equal(expRowCount)) - } -} - -// TODO: expectConnCountInDb is not reliable, pg_stat_activity connections fluctuates -func expectConnCountInDb(db *sql.DB, expConnCount int) { - GinkgoHelper() - log.Printf("expectConnCountInDb: expConnCount=%d", expConnCount) - r, err := db.Query(fmt.Sprintf("SELECT datname, usename, application_name, client_addr, state, backend_type, query FROM pg_stat_activity WHERE client_addr IS NOT NULL")) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: error reading from table pg_stat_activity: %v", err)) - gotConnCount := 0 - for r.Next() { - gotConnCount++ - var datname string - var usename string - var application_name string - var client_addr string - var state string - var backend_type string - var query string - err = r.Scan(&datname, &usename, &application_name, &client_addr, &state, &backend_type, &query) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: error calling r.Scan(): %v", err)) - log.Printf("pg_stat_activity: datname='%s', usename='%s', app_name='%s', client_addr='%s', state='%s', backend_type='%s', query='%s'", - datname, usename, application_name, client_addr, state, backend_type, query) - } - Expect(r.Err()).ToNot(HaveOccurred(), fmt.Sprintf("expectConnCountInDb: cursor iteration err: %v", r.Err())) - Expect(gotConnCount).To(Equal(expConnCount)) -} - -func expectModTime(modPath string, prevModTime time.Time) time.Time { - GinkgoHelper() - nextModTime := prevModTime - - // Get modPath modtime - sts, err := mtm.GetPathStatus(fsnotifyStrategy, modPath) - if err != nil { - Fail(fmt.Sprintf("expectModTime: GetPathStatus(%s) err: %v", modPath, err)) - } - log.Printf("expectModTime: GetPathStatus(%s): %v", modPath, sts) - - // Verify modPath modtime was updated after modPath was updated with new DSN - nextModTime = sts.ModTime - if !nextModTime.After(prevModTime) { - Fail(fmt.Sprintf("expectModTime: %s: new sts.ModTime(%s) <= prevModTime(%s)", modPath, nextModTime, prevModTime)) - } - - return nextModTime -} - -var _ = BeforeSuite(func(ctx context.Context) { - // create tables and chairs - hlt, err := sql.Open("postgres", hotloadTestDsn) - hlt1, err := sql.Open("postgres", hotloadTest1Dsn) - defer hlt.Close() - defer hlt1.Close() - - for { - time.Sleep(5 * time.Second) - _, err = hlt.Exec(testSqlSetup) - if err != nil { - log.Printf("BeforeSuite: error creating test table in hlt: %v", err) - continue - //Fail(fmt.Sprintf()) - } - - _, err = hlt1.Exec(testSqlSetup) - if err != nil { - log.Printf("BeforeSuite: error creating test table in hlt1: %v", err) - continue - } - - break - } - - // enable ModTimeMonitor to monitor configPath - // (do NOT use the ginkgo supplied ctx parm, - // as it will be canceled when BeforeSuite finishes) - mtmCtx, mtmCancelCtxFn = context.WithCancel(context.Background()) - mtm = modtime.NewModTimeMonitor(mtmCtx, - // note the check-interval must be shorter than the sleep interval in setDSN() - modtime.WithCheckInterval(setDSNSleepDur/3), - modtime.WithLogger(testLogger), - modtime.WithErrLogger(testLogger), - ) - mtm.AddMonitoredPath(fsnotifyStrategy, configPath) - time.Sleep(200 * time.Millisecond) -}, NodeTimeout(240*time.Second)) - -var _ = AfterSuite(func(ctx context.Context) { - log.Printf("AfterSuite canceling ModTimeMonitor context") - mtmCancelCtxFn() - time.Sleep(200 * time.Millisecond) - - hlt, err := sql.Open("postgres", hotloadTestDsn) - hlt1, err := sql.Open("postgres", hotloadTest1Dsn) - defer hlt.Close() - defer hlt1.Close() - - _, err = hlt.Exec(testSqlTeardown) - if err != nil { - log.Printf("AfterSuite: error dropping test table in hlt: %v", err) - } - - _, err = hlt1.Exec(testSqlTeardown) - if err != nil { - log.Printf("AfterSuite: error dropping test table in hlt1: %v", err) - } - - //expectConnCountInDb(hlt, 3) - - err = internal.CollectAndRegexpCompare(metrics.HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(metrics.ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(metrics.ExpectHotloadPathChksumTimestampSecondsRegexp, - "/tmp/hotload_integration_test_dsn_config.txt")), - metrics.HotloadPathChksumTimestampSecondsName) - Expect(err).ShouldNot(HaveOccurred()) -}, NodeTimeout(240*time.Second)) - -var _ = Describe("hotload integration tests - sanity", Serial, func() { - var ( - db *sql.DB - hltDb *sql.DB - hlt1Db *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - setDSN(hotloadTestDsn, configPath) - hltDb = openDbPostgres(hotloadTestDsn) - hlt1Db = openDbPostgres(hotloadTest1Dsn) - - _, err := hltDb.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error truncating test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error truncating test table in hlt1Db: %v", err)) - } - - _, err = hltDb.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error initing test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("BeforeEach: error initing test table in hlt1Db: %v", err)) - } - - db = openDbHotload(false) - }) - - AfterEach(func(ctx context.Context) { - hltDb.Close() - hlt1Db.Close() - db.Close() - }) - - It("should connect to new db when file changes", func(ctx context.Context) { - var prevModTime time.Time - for i := 0; i < 2; i++ { - // Verify configPath modtime was updated after configPath was updated with new DSN - prevModTime = expectModTime(configPath, prevModTime) - - r, err := db.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, 'sanity')", i)) - if err != nil { - Fail(fmt.Sprintf("error inserting cnum=%d row: %v", i, err)) - } else { - log.Printf("inserted cnum=%d row", i) - } - log.Print(r) - - // Set new DSN, note that this sleeps for 250 millisecs - setDSN(hotloadTest1Dsn, configPath) - } - expectValueInDb(hltDb, "sanity", false, 1, 0) - expectValueInDb(hlt1Db, "sanity", false, 1, 1) - }) -}) diff --git a/integrationtests/integrationtests_suite_test.go b/integrationtests/integrationtests_suite_test.go deleted file mode 100644 index dfbab22..0000000 --- a/integrationtests/integrationtests_suite_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package integrationtests - -import ( - "fmt" - "log" - "os" - "strings" - "testing" - - "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/logger" - - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var ( - postgresHost = "localhost" - postgresPort = "5432" - - hotloadTestDsn string - hotloadTest1Dsn string - - hldatabaseSuperDsn string - hldatabaseAdminDsn string - - superUser = "admin" - superPass = "test" - adminUser = "admin" - adminPass = "test" - testDbUser = "uuser" -) - -func testDbPass(which int) string { - return fmt.Sprintf("ppass%d", which) -} - -func hldatabasePassDsn(which int) string { - return fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - testDbUser, testDbPass(which), postgresHost, postgresPort) -} - -func testLogger(args ...any) { - log.Println(args...) -} - -func TestIntegrationtests(t *testing.T) { - //log.SetFlags(log.Flags() | log.Lmicroseconds) - log.SetFlags(log.Ltime | log.Lmicroseconds) - log.SetOutput(GinkgoWriter) - logger.WithLogger(testLogger) - logger.WithErrLogger(testLogger) - - nrr := internal.NewNonRandomReader(1) - uuid.SetRand(nrr) - - pgHost, ok := os.LookupEnv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST") - pgHost = strings.TrimSpace(pgHost) - if ok && len(pgHost) > 0 { - postgresHost = pgHost - } - - pgPort, ok := os.LookupEnv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT") - pgPort = strings.TrimSpace(pgPort) - if ok && len(pgPort) > 0 { - postgresPort = pgPort - } - - hotloadTestDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hotload_test?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - hotloadTest1Dsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hotload_test1?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - - hldatabaseSuperDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - superUser, superPass, postgresHost, postgresPort) - hldatabaseAdminDsn = fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", - adminUser, adminPass, postgresHost, postgresPort) - - RegisterFailHandler(Fail) - RunSpecs(t, "Integration Tests") -} diff --git a/integrationtests/longdbtxn_test.go b/integrationtests/longdbtxn_test.go deleted file mode 100644 index cceefe6..0000000 --- a/integrationtests/longdbtxn_test.go +++ /dev/null @@ -1,503 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - _ "github.com/infobloxopen/hotload" - _ "github.com/infobloxopen/hotload/fsnotify" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -type longTestType int - -const ( - LongDatabaseChange longTestType = iota - LongPasswordChange -) - -var longTestNames = map[longTestType]string{ - LongDatabaseChange: "LongDatabaseChange", - LongPasswordChange: "LongPasswordChange", -} - -func (l longTestType) String() string { - return longTestNames[l] -} - -type longDbMode int - -const ( - LongExec longDbMode = iota - LongExecContext - LongQuery - LongQueryContext - LongBegin - LongBeginTx -) - -var longModeNames = map[longDbMode]string{ - LongExec: "LongExec", - LongExecContext: "LongExecContext", - LongQueryContext: "LongQueryContext", - LongQuery: "LongQuery", - LongBegin: "LongBegin", - LongBeginTx: "LongBeginTx", -} - -func (l longDbMode) String() string { - return longModeNames[l] -} - -type longDbTestCase struct { - forceKill bool - testType longTestType - testMode longDbMode - source string - expErr bool - expRowCount int - expCnum int64 - gotErr error - gotRowCount int - longChan chan longDbTestCase - userDb *sql.DB - superDb *sql.DB -} - -// longDbExecFn DB.Exec's a long-running INSERT stmt -func longDbExecFn(tc longDbTestCase) { - tc.gotRowCount = 0 - log.Printf("start db.Exec INSERT PG_SLEEP") - insertStmt := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(1.0))", tc.expCnum, tc.source) - var result sql.Result - var err error - if tc.testMode == LongExec { - result, err = tc.userDb.Exec(insertStmt) - } else { - result, err = tc.userDb.ExecContext(context.Background(), insertStmt) - } - if err == nil { - log.Printf("db.Exec INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.Exec INSERT PG_SLEEP error=%+v", err) - } - tc.gotErr = err - tc.longChan <- tc -} - -// longDbQueryFn DB.Query's a long-running SELECT stmt -func longDbQueryFn(tc longDbTestCase) { - tc.gotRowCount = 0 - log.Printf("start db.Query SELECT PG_SLEEP") - rows, err := tc.userDb.Query(fmt.Sprintf("SELECT COUNT(*), PG_SLEEP(1.0) FROM test WHERE csource = '%s' AND cnum = %d", tc.source, tc.expCnum)) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP rows=%+v", rows) - if !rows.Next() { - err = rows.Err() - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor iteration error=%+v", err) - } - log.Printf("db.Query SELECT PG_SLEEP cursor returned no rows") - } else { - var csleep string - err = rows.Scan(&tc.gotRowCount, &csleep) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor scan error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP cursor scanned rowcount=%d", tc.gotRowCount) - } - } - } - tc.gotErr = err - tc.longChan <- tc -} - -// longDbBeginFn DB.Begin's (or DB.BeginTx's) and DB.Commit/DB.Rollback's -// around a long-running db txn that does a INSERT, sleep(1s), SELECT. -func longDbBeginFn(tc longDbTestCase) { - tc.gotRowCount = 0 - txnFn := func() error { - var txnDb *sql.Tx - var err error - if tc.testMode == LongBeginTx { - log.Printf("start db.BeginTx") - txnDb, err = tc.userDb.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelReadCommitted, - }) - if err == nil { - log.Printf("db.BeginTx success") - } else { - log.Printf("db.BeginTx failure err=%+v", err) - return err - } - } else { - log.Printf("start db.Begin") - txnDb, err = tc.userDb.Begin() - if err == nil { - log.Printf("db.Begin success") - } else { - log.Printf("db.Begin failure err=%+v", err) - return err - } - } - defer func() { - if err == nil { - commitErr := txnDb.Commit() - log.Printf("txnDb.Commit err=%v", commitErr) - } else { - rollbackErr := txnDb.Rollback() - log.Printf("txnDb.Rollback err=%v", rollbackErr) - } - }() - - log.Printf("start db.Exec INSERT PG_SLEEP") - result, err := txnDb.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(0.000001))", tc.expCnum, tc.source)) - if err == nil { - log.Printf("db.Exec INSERT PG_SLEEP result=%+v", result) - } else { - log.Printf("db.Exec INSERT PG_SLEEP error=%+v", err) - } - - time.Sleep(1 * time.Second) - - log.Printf("start db.Query SELECT PG_SLEEP") - rows, err := txnDb.Query(fmt.Sprintf("SELECT COUNT(*), PG_SLEEP(0.000001) FROM test WHERE csource = '%s' AND cnum = %d", tc.source, tc.expCnum)) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP rows=%+v", rows) - if !rows.Next() { - err = rows.Err() - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor iteration error=%+v", err) - } - log.Printf("db.Query SELECT PG_SLEEP cursor returned no rows") - } else { - var csleep string - err = rows.Scan(&tc.gotRowCount, &csleep) - if err != nil { - log.Printf("db.Query SELECT PG_SLEEP cursor scan error=%+v", err) - } else { - log.Printf("db.Query SELECT PG_SLEEP cursor scanned rowcount=%d", tc.gotRowCount) - } - } - } - return err - } - err := txnFn() - tc.gotErr = err - tc.longChan <- tc -} - -func longDbTestFn(ginkgoCtx context.Context, tc longDbTestCase) { - //GinkgoHelper() - - dsn1 := hotloadTestDsn - dsn2 := hotloadTest1Dsn - if tc.testType == LongPasswordChange { - dsn1 = hldatabasePassDsn(1) - dsn2 = hldatabasePassDsn(2) - } - - db := openDbHotload(tc.forceKill) - defer func() { - log.Println("longDbTestFn: closing hotload db conn") - db.Close() - }() - - var prevModTime time.Time - prevModTime = expectModTime(configPath, prevModTime) - - longChan := make(chan longDbTestCase) - longDbFn := longDbExecFn - switch tc.testMode { - case LongExec, LongExecContext: - longDbFn = longDbExecFn - case LongQuery, LongQueryContext: - longDbFn = longDbQueryFn - case LongBegin, LongBeginTx: - longDbFn = longDbBeginFn - default: - longDbFn = nil - Fail(fmt.Sprintf("invalid testMode=%d", int(tc.testMode))) - } - tc.longChan = longChan - tc.userDb = db - - // 1st background long-running db thread, set new DSN in middle of long-running db txn - - By("1st: Spawn long-running db background thread") - go longDbFn(tc) - - By("1st: Momentarily sleep/yield to long-running db background thread") - time.Sleep(10 * time.Millisecond) - - By("1st: Set new DSN, this sleeps/yields for 250 millisecs") - if tc.testType == LongPasswordChange { - dbExecAlterUserPass(tc.superDb, testDbUser, testDbPass(2)) - } - setDSN(dsn2, configPath) - - By("1st: Verify configPath modtime was updated after configPath was updated with new DSN") - prevModTime = expectModTime(configPath, prevModTime) - - By("1st: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("1st: tc returned from long-running db background thread = %#v", tc) - By("1st: Got return from long-running db background thread") - if tc.expErr { - Expect(tc.gotErr).To(HaveOccurred(), fmt.Sprintf("expect error inserting into table test")) - } else { - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("error inserting into table test: %v", tc.gotErr)) - } - - Expect(tc.gotRowCount).To(Equal(tc.expRowCount)) - - // 2nd background long-running db thread, without setting new DSN - - By("2nd: Spawn long-running db background thread") - go longDbFn(tc) - - By("2nd: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("2nd: tc returned from long-running db background thread = %#v", tc) - By("2nd: Got return from long-running db background thread") - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("2nd: error executing long db test: %v", tc.gotErr)) - - // 3rd background long-running db thread, after setting new DSN - - By("3rd: Revert to old DSN, this sleeps/yields for 250 millisecs") - if tc.testType == LongPasswordChange { - dbExecAlterUserPass(tc.superDb, testDbUser, testDbPass(1)) - } - setDSN(dsn1, configPath) - - By("3rd: Verify configPath modtime was updated after configPath was updated with new DSN") - prevModTime = expectModTime(configPath, prevModTime) - - By("3rd: Spawn long-running db background thread") - go longDbFn(tc) - - By("3rd: Wait for return from long-running db background thread") - tc = <-longChan - log.Printf("3rd: tc returned from long-running db background thread = %#v", tc) - By("3rd: Got return from long-running db background thread") - Expect(tc.gotErr).ToNot(HaveOccurred(), fmt.Sprintf("3rd: error executing long db test: %v", tc.gotErr)) -} - -var _ = Describe("hotload integration tests - long running db transaction", Serial, func() { - var ( - hltDb *sql.DB - hlt1Db *sql.DB - ) - - BeforeEach(func(ginkgoCtx context.Context) { - //hotload.UnregisterStrategy("fsnotify") - //hotload.RegisterStrategy("fsnotify", fsnotify.NewStrategy()) - //time.Sleep(100 * time.Millisecond) // Sleep/yield to fsnotify background threads - - By("Resetting and truncating test db") - setDSN(hotloadTestDsn, configPath) - hltDb = openDbPostgres(hotloadTestDsn) - hlt1Db = openDbPostgres(hotloadTest1Dsn) - - _, err := hltDb.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error truncating test table in hltDb: %v", err)) - } - - _, err = hlt1Db.Exec(truncSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error truncating test table in hlt1Db: %v", err)) - } - - // insert 1 initial row into hltDb - _, err = hltDb.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 1 into test table in hltDb: %v", err)) - } - - // insert 2 initial rows into hlt1Db - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 1 into test table in hlt1Db: %v", err)) - } - time.Sleep(100 * time.Millisecond) - _, err = hlt1Db.Exec(initSqlSetup) - if err != nil { - Fail(fmt.Sprintf("error inserting initial row 2 into test table in hlt1Db: %v", err)) - } - }, NodeTimeout(60*time.Second)) - - AfterEach(func(ginkgoCtx context.Context) { - //expectConnCountInDb(hltDb, 3) - hltDb.Close() - hlt1Db.Close() - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongExec, - source: LongExec.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongExec, - source: LongExec.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecContextFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 314159, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: false, - expRowCount: 1, - expCnum: 161803, // see initSqlSetup variable - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: true, - expRowCount: 0, - expCnum: 161803, // see initSqlSetup variable - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongBegin, - source: LongBegin.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongBegin, - source: LongBegin.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: false, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 2, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - forceKill: true, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - expectRowCountInDb(hlt1Db, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) -}) diff --git a/integrationtests/multipleconn_test.go b/integrationtests/multipleconn_test.go deleted file mode 100644 index dfd5be4..0000000 --- a/integrationtests/multipleconn_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("hotload context test", Serial, func() { - var ( - superDb *sql.DB - userDb [10]*sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(0)) - setDSN(hldatabasePassDsn(0), configPath) - - for i := 0; i < len(userDb); i++ { - userDb[i] = openDbHotload(false) - } - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - - for i := 0; i < len(userDb); i++ { - userDb[i].Close() - } - }) - - It("forceKill=false multiple long db.Exec in succession", func(ctx context.Context) { - multipleLongTestFn(LongExec, superDb, userDb) - }) -}) - -func multipleLongTestFn(lMode longDbMode, superDb *sql.DB, userDb [10]*sql.DB) { - callerCtx := context.Background() - csource := "multiple-long-in-succession" - cnum := 141421 - sleepSecs := 2 - insertStmtSleep := fmt.Sprintf("INSERT INTO test (cnum, csource, csleep) VALUES (%d, '%s', PG_SLEEP(%d))", cnum, csource, sleepSecs) - insertStmtNoSleep := fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, '%s')", cnum, csource) - - // userDb[0] inserts 1st row - _, err := userDb[0].ExecContext(callerCtx, insertStmtNoSleep) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("0: error inserting into table test")) - - // userDb[1] inserts 2nd row - _, err = userDb[1].ExecContext(callerCtx, insertStmtNoSleep) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("1: error inserting into table test")) - - // Long-running fn to insert more rows - errChan := make(chan error) - longDbFn := func(which int, lMode longDbMode, userDb *sql.DB, errChan chan error) { - log.Printf("%d: start db.ExecContext INSERT PG_SLEEP", which) - result, err := userDb.ExecContext(callerCtx, insertStmtSleep) - if err == nil { - log.Printf("%d: db.ExecContext INSERT PG_SLEEP result=%+v", which, result) - } else { - log.Printf("%d: db.ExecContext INSERT PG_SLEEP error=%+v", which, err) - } - errChan <- err - } - - go longDbFn(2, lMode, userDb[2], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[2]), - // but since forceKill=false, will gracefully let userDb[2] txn continue. - // Any previous conn/txn will be canceled/closed (but there - // isn't any previous txn at this point in the test). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - - go longDbFn(3, lMode, userDb[3], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[3]), - // but since forceKill=false, will gracefully let userDb[3] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[2]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(2)) - setDSN(hldatabasePassDsn(2), configPath) - - go longDbFn(4, lMode, userDb[4], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[4]), - // but since forceKill=false, will gracefully let userDb[4] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[3]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(3)) - setDSN(hldatabasePassDsn(3), configPath) - - go longDbFn(5, lMode, userDb[5], errChan) - time.Sleep(10 * time.Millisecond) - - // This dsn-change will reset existing conn/txn (userDb[5]), - // but since forceKill=false, will gracefully let userDb[5] txn continue. - // Any previous conn/txn will be canceled/closed (userDb[4]). - dbExecAlterUserPass(superDb, testDbUser, testDbPass(4)) - setDSN(hldatabasePassDsn(4), configPath) - - // At this point in the test, only userDb[5] txn is allowed to gracefully continue - // and insert 3rd row; the other txn (userDb[2]/3/4 have been canceled). - - for i := 2; i <= 5; i++ { - err = <-errChan - log.Printf("%d: returned from long-running db background thread, err=%v", i, err) - if i < 5 { - Expect(err).To(HaveOccurred(), fmt.Sprintf("%d: expect error inserting into table test", i)) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("%d: unexpected error inserting into table test: %v", i, err)) - } - } - expectRowCountInDb(superDb, csource, false, 3, int64(cnum)) - - dbExecSqlStmt(userDb[0], insertStmtNoSleep) - dbExecSqlStmt(userDb[1], insertStmtNoSleep) - dbExecSqlStmt(userDb[2], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(5)) - setDSN(hldatabasePassDsn(5), configPath) - - dbExecSqlStmt(userDb[3], insertStmtNoSleep) - dbExecSqlStmt(userDb[4], insertStmtNoSleep) - dbExecSqlStmt(userDb[5], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(6)) - setDSN(hldatabasePassDsn(6), configPath) - - dbExecSqlStmt(userDb[6], insertStmtNoSleep) - dbExecSqlStmt(userDb[7], insertStmtNoSleep) - dbExecSqlStmt(userDb[8], insertStmtNoSleep) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(7)) - setDSN(hldatabasePassDsn(7), configPath) - - dbExecSqlStmt(userDb[9], insertStmtNoSleep) - expectRowCountInDb(superDb, csource, false, 13, int64(cnum)) -} diff --git a/integrationtests/passwdchange_test.go b/integrationtests/passwdchange_test.go deleted file mode 100644 index 6929a97..0000000 --- a/integrationtests/passwdchange_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package integrationtests - -import ( - "context" - "database/sql" - "fmt" - "log" - "time" - - . "github.com/onsi/ginkgo/v2" -) - -func dbExecSqlStmt(db *sql.DB, sqlStmt string) { - GinkgoHelper() - _, err := db.Exec(sqlStmt) - if err != nil { - Fail(fmt.Sprintf("db.Exec(%s) error: %v", sqlStmt, err)) - } - time.Sleep(1 * time.Millisecond) -} - -func dbExecAlterUserPass(db *sql.DB, user, pass string) { - GinkgoHelper() - sqlStmt := fmt.Sprintf("ALTER USER %s WITH PASSWORD '%s'", user, pass) - dbExecSqlStmt(db, sqlStmt) -} - -var _ = Describe("hotload integration tests - db passwd change", Serial, func() { - var ( - superDb *sql.DB - userDb *sql.DB - ) - - BeforeEach(func(ctx context.Context) { - superDb = openDbPostgres(hldatabaseSuperDsn) - dbExecSqlStmt(superDb, userSqlTeardown) - dbExecSqlStmt(superDb, userSqlSetup) - dbExecSqlStmt(superDb, testSqlSetup) - dbExecSqlStmt(superDb, truncSqlSetup) - dbExecSqlStmt(superDb, initSqlSetup) - dbExecSqlStmt(superDb, grantSqlSetup) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(1)) - setDSN(hldatabasePassDsn(1), configPath) - }) - - AfterEach(func(ctx context.Context) { - dbExecSqlStmt(superDb, testSqlTeardown) - dbExecSqlStmt(superDb, userSqlTeardown) - superDb.Close() - }) - - It("should reconnect to db when password changes", func(ctx context.Context) { - userDb = openDbHotload(false) - defer userDb.Close() - - csource := "db-passwd-change" - cnum := 184775 - var prevModTime time.Time - for i := 0; i < 2; i++ { - // Verify configPath modtime was updated after configPath was updated with new DSN - prevModTime = expectModTime(configPath, prevModTime) - - r, err := userDb.Exec(fmt.Sprintf("INSERT INTO test (cnum, csource) VALUES (%d, '%s')", - cnum, csource)) - if err != nil { - Fail(fmt.Sprintf("%d: error inserting cnum=%d row: %v", i, cnum, err)) - } else { - log.Printf("%d: inserted cnum=%d row", i, cnum) - } - log.Print(r) - - dbExecAlterUserPass(superDb, testDbUser, testDbPass(2)) - - // Set new DSN, note that this sleeps for 250 millisecs - setDSN(hldatabasePassDsn(2), configPath) - } - expectRowCountInDb(superDb, csource, false, 2, int64(cnum)) - }) - - It("long-running db.Exec, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongExec, - source: LongExec.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Exec, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongExec, - source: LongExec.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: false, - expRowCount: 0, // expRowCount=0 b/c longDbExecContextFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.ExecContext, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongExecContext, - source: LongExecContext.String(), - expErr: true, - expRowCount: 0, // expRowCount=0 b/c longDbExecFn doesn't query SELECT - expCnum: 184775, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - // TODO: tc.expErr=true, so why expRowCount=2? why not expRowCount=1? - // INSERT fails with this (tc.expErr=true above): - // error=read tcp 127.0.0.1:58390->127.0.0.1:5432: use of closed network connection - // yet the INSERT seems to've succeeded?!? - //expectRowCountInDb(hltDb, tc.source, false, 1, tc.expCnum) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: false, - expRowCount: 1, - expCnum: 161803, // see initSqlSetup variable - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Query, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongQuery, - source: "initial", // see initSqlSetup variable - expErr: true, - expRowCount: 0, - expCnum: 161803, // see initSqlSetup variable - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 1, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongBegin, - source: LongBegin.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.Begin, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongBegin, - source: LongBegin.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=false", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: false, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: false, - expRowCount: 1, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 3, tc.expCnum) - }, NodeTimeout(60*time.Second)) - - It("long-running db.BeginTx, forceKill=true", func(ginkgoCtx context.Context) { - tc := longDbTestCase{ - testType: LongPasswordChange, - forceKill: true, - testMode: LongBeginTx, - source: LongBeginTx.String(), - expErr: true, - expRowCount: 0, - expCnum: 271828, - superDb: superDb, - } - longDbTestFn(ginkgoCtx, tc) - - expectRowCountInDb(superDb, tc.source, false, 2, tc.expCnum) - }, NodeTimeout(60*time.Second)) -}) diff --git a/interfaces_test.go b/interfaces_test.go new file mode 100644 index 0000000..590cfc9 --- /dev/null +++ b/interfaces_test.go @@ -0,0 +1,132 @@ +package hotload_test + +import ( + "database/sql/driver" + "fmt" + "testing" + + "github.com/infobloxopen/hotload/v3/internal/dbfake" +) + +// TestConnInterfaceTruthfulness verifies the heart of the wrapper design: +// the conn hotload hands to database/sql implements an optional interface if +// and only if the underlying conn supports the capability. It walks every +// combination of the mirrored capabilities plus the legacy-interface +// variants and asserts presence and absence of each interface. +func TestConnInterfaceTruthfulness(t *testing.T) { + // The four mirrored axes. + axes := []dbfake.Caps{ + dbfake.CapExecerContext, + dbfake.CapQueryerContext, + dbfake.CapPinger, + dbfake.CapNamedValueChecker, + } + + var combos []dbfake.Caps + for mask := 0; mask < 1< 0 { + time.Sleep(d.OpenDelay) + } + if d.OpenErr != nil { + if err := d.OpenErr(dsn); err != nil { + return nil, err + } + } + d.mu.Lock() + d.nextID++ + c := &Conn{drv: d, ID: d.nextID, DSN: dsn, caps: d.Caps} + d.conns = append(d.conns, c) + d.mu.Unlock() + d.Log.add(c.ID, "Open", dsn) + return viewConn(c), nil +} + +// Conns returns a snapshot of every conn the driver has opened, in order. +func (d *Driver) Conns() []*Conn { + d.mu.Lock() + defer d.mu.Unlock() + out := make([]*Conn, len(d.conns)) + copy(out, d.conns) + return out +} + +// OpenConns returns the conns that have not been closed yet. +func (d *Driver) OpenConns() []*Conn { + d.mu.Lock() + defer d.mu.Unlock() + var out []*Conn + for _, c := range d.conns { + if !c.Closed() { + out = append(out, c) + } + } + return out +} + +// Conn is the fake connection. It is always handed to hotload behind a +// capability view (see views_gen.go), never directly. +type Conn struct { + drv *Driver + ID int + DSN string + caps Caps + + closed atomic.Bool + closeCount atomic.Int64 + valid atomic.Bool // inverted: true means IsValid returns false +} + +// Closed reports whether the conn was closed. +func (c *Conn) Closed() bool { return c.closed.Load() } + +// CloseCount reports how many times Close was called on the conn; hotload +// must never close an underlying conn twice. +func (c *Conn) CloseCount() int64 { return c.closeCount.Load() } + +// SetInvalid makes a CapValidator conn report IsValid() == false. +func (c *Conn) SetInvalid() { c.valid.Store(true) } + +func (c *Conn) log(method, detail string) { c.drv.Log.add(c.ID, method, detail) } + +func (c *Conn) doPrepare(query string) (driver.Stmt, error) { + c.log("Prepare", query) + return viewStmt(&Stmt{conn: c, query: query}), nil +} + +func (c *Conn) doPrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + c.log("PrepareContext", query) + if err := ctx.Err(); err != nil { + return nil, err + } + return viewStmt(&Stmt{conn: c, query: query}), nil +} + +func (c *Conn) doClose() error { + c.closeCount.Add(1) + c.closed.Store(true) + c.log("Close", "") + return nil +} + +func (c *Conn) doBegin() (driver.Tx, error) { + c.log("Begin", "") + return &Tx{conn: c}, nil +} + +func (c *Conn) doBeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + c.log("BeginTx", "") + if err := ctx.Err(); err != nil { + return nil, err + } + return &Tx{conn: c}, nil +} + +func (c *Conn) doExec(ctx context.Context, method, query string, args []driver.NamedValue) (driver.Result, error) { + c.log(method, query) + if c.drv.ExecFn != nil { + return c.drv.ExecFn(c, ctx, query, args) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return driver.RowsAffected(1), nil +} + +func (c *Conn) doQuery(ctx context.Context, method, query string, args []driver.NamedValue) (driver.Rows, error) { + c.log(method, query) + if c.drv.QueryFn != nil { + return c.drv.QueryFn(c, ctx, query, args) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return &Rows{cols: []string{"dsn"}, rows: [][]driver.Value{{c.DSN}}}, nil +} + +func (c *Conn) doPing(ctx context.Context) error { + c.log("Ping", "") + return ctx.Err() +} + +func (c *Conn) doCheckNamedValue(nv *driver.NamedValue) error { + c.log("CheckNamedValue", fmt.Sprint(nv.Value)) + return nil +} + +func (c *Conn) doResetSession(ctx context.Context) error { + c.log("ResetSession", "") + return nil +} + +func (c *Conn) doIsValid() bool { + c.log("IsValid", "") + return !c.valid.Load() +} + +func namedToValues(args []driver.Value) []driver.NamedValue { + out := make([]driver.NamedValue, len(args)) + for i, v := range args { + out[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return out +} + +// fvBase implements the mandatory driver.Conn methods for every view. +type fvBase struct{ c *Conn } + +func (v fvBase) Prepare(query string) (driver.Stmt, error) { return v.c.doPrepare(query) } +func (v fvBase) Close() error { return v.c.doClose() } +func (v fvBase) Begin() (driver.Tx, error) { return v.c.doBegin() } + +// Unwrap exposes the fake conn behind a view for test assertions. +func (v fvBase) Unwrap() *Conn { return v.c } + +// Unwrapper is implemented by every conn view; tests use it to reach the +// underlying fake conn. +type Unwrapper interface{ Unwrap() *Conn } + +type fvExecer struct{ c *Conn } + +func (v fvExecer) Exec(query string, args []driver.Value) (driver.Result, error) { + return v.c.doExec(context.Background(), "Exec", query, namedToValues(args)) +} + +type fvExecerCtx struct{ c *Conn } + +func (v fvExecerCtx) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return v.c.doExec(ctx, "ExecContext", query, args) +} + +type fvQueryer struct{ c *Conn } + +func (v fvQueryer) Query(query string, args []driver.Value) (driver.Rows, error) { + return v.c.doQuery(context.Background(), "Query", query, namedToValues(args)) +} + +type fvQueryerCtx struct{ c *Conn } + +func (v fvQueryerCtx) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return v.c.doQuery(ctx, "QueryContext", query, args) +} + +type fvPinger struct{ c *Conn } + +func (v fvPinger) Ping(ctx context.Context) error { return v.c.doPing(ctx) } + +type fvNVChecker struct{ c *Conn } + +func (v fvNVChecker) CheckNamedValue(nv *driver.NamedValue) error { return v.c.doCheckNamedValue(nv) } + +// fvClassA groups the optional interfaces hotload always synthesizes when +// the underlying conn lacks them: ConnPrepareContext, ConnBeginTx, +// SessionResetter and Validator. +type fvClassA struct{ c *Conn } + +func (v fvClassA) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + return v.c.doPrepareContext(ctx, query) +} + +func (v fvClassA) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + return v.c.doBeginTx(ctx, opts) +} + +func (v fvClassA) ResetSession(ctx context.Context) error { return v.c.doResetSession(ctx) } +func (v fvClassA) IsValid() bool { return v.c.doIsValid() } + +// Stmt is the fake prepared statement, always handed out behind a +// capability view. +type Stmt struct { + conn *Conn + query string +} + +func (s *Stmt) doClose() error { + s.conn.log("StmtClose", s.query) + return nil +} + +func (s *Stmt) doNumInput() int { return -1 } + +func (s *Stmt) doExec(ctx context.Context, method string, args []driver.NamedValue) (driver.Result, error) { + return s.conn.doExec(ctx, method, s.query, args) +} + +func (s *Stmt) doQuery(ctx context.Context, method string, args []driver.NamedValue) (driver.Rows, error) { + return s.conn.doQuery(ctx, method, s.query, args) +} + +type fsBase struct{ s *Stmt } + +func (v fsBase) Close() error { return v.s.doClose() } +func (v fsBase) NumInput() int { return v.s.doNumInput() } +func (v fsBase) Exec(args []driver.Value) (driver.Result, error) { + return v.s.doExec(context.Background(), "StmtExec", namedToValues(args)) +} +func (v fsBase) Query(args []driver.Value) (driver.Rows, error) { + return v.s.doQuery(context.Background(), "StmtQuery", namedToValues(args)) +} + +type fsExecerCtx struct{ s *Stmt } + +func (v fsExecerCtx) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return v.s.doExec(ctx, "StmtExecContext", args) +} + +type fsQueryerCtx struct{ s *Stmt } + +func (v fsQueryerCtx) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return v.s.doQuery(ctx, "StmtQueryContext", args) +} + +type fsColConv struct{ s *Stmt } + +func (v fsColConv) ColumnConverter(idx int) driver.ValueConverter { + return driver.DefaultParameterConverter +} + +type fsNVChecker struct{ s *Stmt } + +func (v fsNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return v.s.conn.doCheckNamedValue(nv) +} + +// Tx is the fake transaction. +type Tx struct{ conn *Conn } + +func (t *Tx) Commit() error { + t.conn.log("Commit", "") + return nil +} + +func (t *Tx) Rollback() error { + t.conn.log("Rollback", "") + return nil +} + +// Rows is the fake result set. +type Rows struct { + cols []string + rows [][]driver.Value + pos int +} + +func (r *Rows) Columns() []string { return r.cols } +func (r *Rows) Close() error { return nil } + +func (r *Rows) Next(dest []driver.Value) error { + if r.pos >= len(r.rows) { + return io.EOF + } + copy(dest, r.rows[r.pos]) + r.pos++ + return nil +} + +// Call is one recorded driver operation. +type Call struct { + ConnID int + Method string + Detail string +} + +// CallLog records driver operations across all conns of a Driver. +type CallLog struct { + mu sync.Mutex + calls []Call +} + +func (l *CallLog) add(connID int, method, detail string) { + l.mu.Lock() + defer l.mu.Unlock() + l.calls = append(l.calls, Call{ConnID: connID, Method: method, Detail: detail}) +} + +// Calls returns a snapshot of all recorded calls in order. +func (l *CallLog) Calls() []Call { + l.mu.Lock() + defer l.mu.Unlock() + out := make([]Call, len(l.calls)) + copy(out, l.calls) + return out +} + +// Count returns how many times the named method was called. +func (l *CallLog) Count(method string) int { + l.mu.Lock() + defer l.mu.Unlock() + n := 0 + for _, c := range l.calls { + if c.Method == method { + n++ + } + } + return n +} + +// Reset clears the log. +func (l *CallLog) Reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.calls = nil +} diff --git a/internal/dbfake/views_gen.go b/internal/dbfake/views_gen.go new file mode 100644 index 0000000..c3fedbc --- /dev/null +++ b/internal/dbfake/views_gen.go @@ -0,0 +1,1403 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package dbfake + +import "database/sql/driver" + +type fconn_bare struct { + fvBase +} + +type fconn_e struct { + fvBase + fvExecer +} + +type fconn_E struct { + fvBase + fvExecerCtx +} + +type fconn_eE struct { + fvBase + fvExecer + fvExecerCtx +} + +type fconn_q struct { + fvBase + fvQueryer +} + +type fconn_eq struct { + fvBase + fvExecer + fvQueryer +} + +type fconn_Eq struct { + fvBase + fvExecerCtx + fvQueryer +} + +type fconn_eEq struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer +} + +type fconn_Q struct { + fvBase + fvQueryerCtx +} + +type fconn_eQ struct { + fvBase + fvExecer + fvQueryerCtx +} + +type fconn_EQ struct { + fvBase + fvExecerCtx + fvQueryerCtx +} + +type fconn_eEQ struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx +} + +type fconn_qQ struct { + fvBase + fvQueryer + fvQueryerCtx +} + +type fconn_eqQ struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx +} + +type fconn_EqQ struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx +} + +type fconn_eEqQ struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx +} + +type fconn_P struct { + fvBase + fvPinger +} + +type fconn_eP struct { + fvBase + fvExecer + fvPinger +} + +type fconn_EP struct { + fvBase + fvExecerCtx + fvPinger +} + +type fconn_eEP struct { + fvBase + fvExecer + fvExecerCtx + fvPinger +} + +type fconn_qP struct { + fvBase + fvQueryer + fvPinger +} + +type fconn_eqP struct { + fvBase + fvExecer + fvQueryer + fvPinger +} + +type fconn_EqP struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger +} + +type fconn_eEqP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger +} + +type fconn_QP struct { + fvBase + fvQueryerCtx + fvPinger +} + +type fconn_eQP struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger +} + +type fconn_EQP struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger +} + +type fconn_eEQP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger +} + +type fconn_qQP struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_eqQP struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_EqQP struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_eEqQP struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger +} + +type fconn_N struct { + fvBase + fvNVChecker +} + +type fconn_eN struct { + fvBase + fvExecer + fvNVChecker +} + +type fconn_EN struct { + fvBase + fvExecerCtx + fvNVChecker +} + +type fconn_eEN struct { + fvBase + fvExecer + fvExecerCtx + fvNVChecker +} + +type fconn_qN struct { + fvBase + fvQueryer + fvNVChecker +} + +type fconn_eqN struct { + fvBase + fvExecer + fvQueryer + fvNVChecker +} + +type fconn_EqN struct { + fvBase + fvExecerCtx + fvQueryer + fvNVChecker +} + +type fconn_eEqN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvNVChecker +} + +type fconn_QN struct { + fvBase + fvQueryerCtx + fvNVChecker +} + +type fconn_eQN struct { + fvBase + fvExecer + fvQueryerCtx + fvNVChecker +} + +type fconn_EQN struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvNVChecker +} + +type fconn_eEQN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvNVChecker +} + +type fconn_qQN struct { + fvBase + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_eqQN struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_EqQN struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_eEqQN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker +} + +type fconn_PN struct { + fvBase + fvPinger + fvNVChecker +} + +type fconn_ePN struct { + fvBase + fvExecer + fvPinger + fvNVChecker +} + +type fconn_EPN struct { + fvBase + fvExecerCtx + fvPinger + fvNVChecker +} + +type fconn_eEPN struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvNVChecker +} + +type fconn_qPN struct { + fvBase + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_eqPN struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_EqPN struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_eEqPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker +} + +type fconn_QPN struct { + fvBase + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eQPN struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_EQPN struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eEQPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_qQPN struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eqQPN struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_EqQPN struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_eEqQPN struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker +} + +type fconn_A struct { + fvBase + fvClassA +} + +type fconn_eA struct { + fvBase + fvExecer + fvClassA +} + +type fconn_EA struct { + fvBase + fvExecerCtx + fvClassA +} + +type fconn_eEA struct { + fvBase + fvExecer + fvExecerCtx + fvClassA +} + +type fconn_qA struct { + fvBase + fvQueryer + fvClassA +} + +type fconn_eqA struct { + fvBase + fvExecer + fvQueryer + fvClassA +} + +type fconn_EqA struct { + fvBase + fvExecerCtx + fvQueryer + fvClassA +} + +type fconn_eEqA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvClassA +} + +type fconn_QA struct { + fvBase + fvQueryerCtx + fvClassA +} + +type fconn_eQA struct { + fvBase + fvExecer + fvQueryerCtx + fvClassA +} + +type fconn_EQA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvClassA +} + +type fconn_eEQA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvClassA +} + +type fconn_qQA struct { + fvBase + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_eqQA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_EqQA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_eEqQA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvClassA +} + +type fconn_PA struct { + fvBase + fvPinger + fvClassA +} + +type fconn_ePA struct { + fvBase + fvExecer + fvPinger + fvClassA +} + +type fconn_EPA struct { + fvBase + fvExecerCtx + fvPinger + fvClassA +} + +type fconn_eEPA struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvClassA +} + +type fconn_qPA struct { + fvBase + fvQueryer + fvPinger + fvClassA +} + +type fconn_eqPA struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvClassA +} + +type fconn_EqPA struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvClassA +} + +type fconn_eEqPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvClassA +} + +type fconn_QPA struct { + fvBase + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eQPA struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_EQPA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eEQPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_qQPA struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eqQPA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_EqQPA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_eEqQPA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvClassA +} + +type fconn_NA struct { + fvBase + fvNVChecker + fvClassA +} + +type fconn_eNA struct { + fvBase + fvExecer + fvNVChecker + fvClassA +} + +type fconn_ENA struct { + fvBase + fvExecerCtx + fvNVChecker + fvClassA +} + +type fconn_eENA struct { + fvBase + fvExecer + fvExecerCtx + fvNVChecker + fvClassA +} + +type fconn_qNA struct { + fvBase + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_eqNA struct { + fvBase + fvExecer + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_EqNA struct { + fvBase + fvExecerCtx + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_eEqNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvNVChecker + fvClassA +} + +type fconn_QNA struct { + fvBase + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eQNA struct { + fvBase + fvExecer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_EQNA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eEQNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_qQNA struct { + fvBase + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eqQNA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_EqQNA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_eEqQNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvNVChecker + fvClassA +} + +type fconn_PNA struct { + fvBase + fvPinger + fvNVChecker + fvClassA +} + +type fconn_ePNA struct { + fvBase + fvExecer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EPNA struct { + fvBase + fvExecerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEPNA struct { + fvBase + fvExecer + fvExecerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_qPNA struct { + fvBase + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eqPNA struct { + fvBase + fvExecer + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EqPNA struct { + fvBase + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEqPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvPinger + fvNVChecker + fvClassA +} + +type fconn_QPNA struct { + fvBase + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eQPNA struct { + fvBase + fvExecer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EQPNA struct { + fvBase + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEQPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_qQPNA struct { + fvBase + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eqQPNA struct { + fvBase + fvExecer + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_EqQPNA struct { + fvBase + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fconn_eEqQPNA struct { + fvBase + fvExecer + fvExecerCtx + fvQueryer + fvQueryerCtx + fvPinger + fvNVChecker + fvClassA +} + +type fstmt_bare struct { + fsBase +} + +type fstmt_E struct { + fsBase + fsExecerCtx +} + +type fstmt_Q struct { + fsBase + fsQueryerCtx +} + +type fstmt_EQ struct { + fsBase + fsExecerCtx + fsQueryerCtx +} + +type fstmt_C struct { + fsBase + fsColConv +} + +type fstmt_EC struct { + fsBase + fsExecerCtx + fsColConv +} + +type fstmt_QC struct { + fsBase + fsQueryerCtx + fsColConv +} + +type fstmt_EQC struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsColConv +} + +type fstmt_N struct { + fsBase + fsNVChecker +} + +type fstmt_EN struct { + fsBase + fsExecerCtx + fsNVChecker +} + +type fstmt_QN struct { + fsBase + fsQueryerCtx + fsNVChecker +} + +type fstmt_EQN struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsNVChecker +} + +type fstmt_CN struct { + fsBase + fsColConv + fsNVChecker +} + +type fstmt_ECN struct { + fsBase + fsExecerCtx + fsColConv + fsNVChecker +} + +type fstmt_QCN struct { + fsBase + fsQueryerCtx + fsColConv + fsNVChecker +} + +type fstmt_EQCN struct { + fsBase + fsExecerCtx + fsQueryerCtx + fsColConv + fsNVChecker +} + +// viewConn returns c behind a view exposing exactly the optional +// interfaces selected by c.caps. The four interfaces hotload always +// synthesizes (ConnPrepareContext, ConnBeginTx, SessionResetter, +// Validator) are grouped: if any of their caps is set the view exposes +// all four. +func viewConn(c *Conn) driver.Conn { + var f int + if c.caps&CapExecer != 0 { + f |= 1 << 0 + } + if c.caps&CapExecerContext != 0 { + f |= 1 << 1 + } + if c.caps&CapQueryer != 0 { + f |= 1 << 2 + } + if c.caps&CapQueryerContext != 0 { + f |= 1 << 3 + } + if c.caps&CapPinger != 0 { + f |= 1 << 4 + } + if c.caps&CapNamedValueChecker != 0 { + f |= 1 << 5 + } + if c.caps&(CapConnPrepareContext|CapConnBeginTx|CapSessionResetter|CapValidator) != 0 { + f |= 1 << 6 + } + switch f { + case 0: + return fconn_bare{fvBase{c}} + case 1: + return fconn_e{fvBase{c}, fvExecer{c}} + case 2: + return fconn_E{fvBase{c}, fvExecerCtx{c}} + case 3: + return fconn_eE{fvBase{c}, fvExecer{c}, fvExecerCtx{c}} + case 4: + return fconn_q{fvBase{c}, fvQueryer{c}} + case 5: + return fconn_eq{fvBase{c}, fvExecer{c}, fvQueryer{c}} + case 6: + return fconn_Eq{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}} + case 7: + return fconn_eEq{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}} + case 8: + return fconn_Q{fvBase{c}, fvQueryerCtx{c}} + case 9: + return fconn_eQ{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}} + case 10: + return fconn_EQ{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}} + case 11: + return fconn_eEQ{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}} + case 12: + return fconn_qQ{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 13: + return fconn_eqQ{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 14: + return fconn_EqQ{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 15: + return fconn_eEqQ{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}} + case 16: + return fconn_P{fvBase{c}, fvPinger{c}} + case 17: + return fconn_eP{fvBase{c}, fvExecer{c}, fvPinger{c}} + case 18: + return fconn_EP{fvBase{c}, fvExecerCtx{c}, fvPinger{c}} + case 19: + return fconn_eEP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}} + case 20: + return fconn_qP{fvBase{c}, fvQueryer{c}, fvPinger{c}} + case 21: + return fconn_eqP{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}} + case 22: + return fconn_EqP{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}} + case 23: + return fconn_eEqP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}} + case 24: + return fconn_QP{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}} + case 25: + return fconn_eQP{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 26: + return fconn_EQP{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}} + case 27: + return fconn_eEQP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}} + case 28: + return fconn_qQP{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 29: + return fconn_eqQP{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 30: + return fconn_EqQP{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 31: + return fconn_eEqQP{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}} + case 32: + return fconn_N{fvBase{c}, fvNVChecker{c}} + case 33: + return fconn_eN{fvBase{c}, fvExecer{c}, fvNVChecker{c}} + case 34: + return fconn_EN{fvBase{c}, fvExecerCtx{c}, fvNVChecker{c}} + case 35: + return fconn_eEN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvNVChecker{c}} + case 36: + return fconn_qN{fvBase{c}, fvQueryer{c}, fvNVChecker{c}} + case 37: + return fconn_eqN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvNVChecker{c}} + case 38: + return fconn_EqN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}} + case 39: + return fconn_eEqN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}} + case 40: + return fconn_QN{fvBase{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 41: + return fconn_eQN{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 42: + return fconn_EQN{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 43: + return fconn_eEQN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 44: + return fconn_qQN{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 45: + return fconn_eqQN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 46: + return fconn_EqQN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 47: + return fconn_eEqQN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}} + case 48: + return fconn_PN{fvBase{c}, fvPinger{c}, fvNVChecker{c}} + case 49: + return fconn_ePN{fvBase{c}, fvExecer{c}, fvPinger{c}, fvNVChecker{c}} + case 50: + return fconn_EPN{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 51: + return fconn_eEPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 52: + return fconn_qPN{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 53: + return fconn_eqPN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 54: + return fconn_EqPN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 55: + return fconn_eEqPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}} + case 56: + return fconn_QPN{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 57: + return fconn_eQPN{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 58: + return fconn_EQPN{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 59: + return fconn_eEQPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 60: + return fconn_qQPN{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 61: + return fconn_eqQPN{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 62: + return fconn_EqQPN{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 63: + return fconn_eEqQPN{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}} + case 64: + return fconn_A{fvBase{c}, fvClassA{c}} + case 65: + return fconn_eA{fvBase{c}, fvExecer{c}, fvClassA{c}} + case 66: + return fconn_EA{fvBase{c}, fvExecerCtx{c}, fvClassA{c}} + case 67: + return fconn_eEA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvClassA{c}} + case 68: + return fconn_qA{fvBase{c}, fvQueryer{c}, fvClassA{c}} + case 69: + return fconn_eqA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvClassA{c}} + case 70: + return fconn_EqA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvClassA{c}} + case 71: + return fconn_eEqA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvClassA{c}} + case 72: + return fconn_QA{fvBase{c}, fvQueryerCtx{c}, fvClassA{c}} + case 73: + return fconn_eQA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 74: + return fconn_EQA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvClassA{c}} + case 75: + return fconn_eEQA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvClassA{c}} + case 76: + return fconn_qQA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 77: + return fconn_eqQA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 78: + return fconn_EqQA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 79: + return fconn_eEqQA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvClassA{c}} + case 80: + return fconn_PA{fvBase{c}, fvPinger{c}, fvClassA{c}} + case 81: + return fconn_ePA{fvBase{c}, fvExecer{c}, fvPinger{c}, fvClassA{c}} + case 82: + return fconn_EPA{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvClassA{c}} + case 83: + return fconn_eEPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvClassA{c}} + case 84: + return fconn_qPA{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 85: + return fconn_eqPA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 86: + return fconn_EqPA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 87: + return fconn_eEqPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvClassA{c}} + case 88: + return fconn_QPA{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 89: + return fconn_eQPA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 90: + return fconn_EQPA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 91: + return fconn_eEQPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 92: + return fconn_qQPA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 93: + return fconn_eqQPA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 94: + return fconn_EqQPA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 95: + return fconn_eEqQPA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvClassA{c}} + case 96: + return fconn_NA{fvBase{c}, fvNVChecker{c}, fvClassA{c}} + case 97: + return fconn_eNA{fvBase{c}, fvExecer{c}, fvNVChecker{c}, fvClassA{c}} + case 98: + return fconn_ENA{fvBase{c}, fvExecerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 99: + return fconn_eENA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 100: + return fconn_qNA{fvBase{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 101: + return fconn_eqNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 102: + return fconn_EqNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 103: + return fconn_eEqNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvNVChecker{c}, fvClassA{c}} + case 104: + return fconn_QNA{fvBase{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 105: + return fconn_eQNA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 106: + return fconn_EQNA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 107: + return fconn_eEQNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 108: + return fconn_qQNA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 109: + return fconn_eqQNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 110: + return fconn_EqQNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 111: + return fconn_eEqQNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvNVChecker{c}, fvClassA{c}} + case 112: + return fconn_PNA{fvBase{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 113: + return fconn_ePNA{fvBase{c}, fvExecer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 114: + return fconn_EPNA{fvBase{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 115: + return fconn_eEPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 116: + return fconn_qPNA{fvBase{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 117: + return fconn_eqPNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 118: + return fconn_EqPNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 119: + return fconn_eEqPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 120: + return fconn_QPNA{fvBase{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 121: + return fconn_eQPNA{fvBase{c}, fvExecer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 122: + return fconn_EQPNA{fvBase{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 123: + return fconn_eEQPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 124: + return fconn_qQPNA{fvBase{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 125: + return fconn_eqQPNA{fvBase{c}, fvExecer{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 126: + return fconn_EqQPNA{fvBase{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + case 127: + return fconn_eEqQPNA{fvBase{c}, fvExecer{c}, fvExecerCtx{c}, fvQueryer{c}, fvQueryerCtx{c}, fvPinger{c}, fvNVChecker{c}, fvClassA{c}} + } + panic("unreachable") +} + +// viewStmt returns s behind a view exposing exactly the optional stmt +// interfaces selected by the conn's caps. +func viewStmt(s *Stmt) driver.Stmt { + var f int + if s.conn.caps&CapStmtExecContext != 0 { + f |= 1 << 0 + } + if s.conn.caps&CapStmtQueryContext != 0 { + f |= 1 << 1 + } + if s.conn.caps&CapColumnConverter != 0 { + f |= 1 << 2 + } + if s.conn.caps&CapStmtNamedValueChecker != 0 { + f |= 1 << 3 + } + switch f { + case 0: + return fstmt_bare{fsBase{s}} + case 1: + return fstmt_E{fsBase{s}, fsExecerCtx{s}} + case 2: + return fstmt_Q{fsBase{s}, fsQueryerCtx{s}} + case 3: + return fstmt_EQ{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}} + case 4: + return fstmt_C{fsBase{s}, fsColConv{s}} + case 5: + return fstmt_EC{fsBase{s}, fsExecerCtx{s}, fsColConv{s}} + case 6: + return fstmt_QC{fsBase{s}, fsQueryerCtx{s}, fsColConv{s}} + case 7: + return fstmt_EQC{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsColConv{s}} + case 8: + return fstmt_N{fsBase{s}, fsNVChecker{s}} + case 9: + return fstmt_EN{fsBase{s}, fsExecerCtx{s}, fsNVChecker{s}} + case 10: + return fstmt_QN{fsBase{s}, fsQueryerCtx{s}, fsNVChecker{s}} + case 11: + return fstmt_EQN{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsNVChecker{s}} + case 12: + return fstmt_CN{fsBase{s}, fsColConv{s}, fsNVChecker{s}} + case 13: + return fstmt_ECN{fsBase{s}, fsExecerCtx{s}, fsColConv{s}, fsNVChecker{s}} + case 14: + return fstmt_QCN{fsBase{s}, fsQueryerCtx{s}, fsColConv{s}, fsNVChecker{s}} + case 15: + return fstmt_EQCN{fsBase{s}, fsExecerCtx{s}, fsQueryerCtx{s}, fsColConv{s}, fsNVChecker{s}} + } + panic("unreachable") +} diff --git a/internal/depbudget/main.go b/internal/depbudget/main.go new file mode 100644 index 0000000..8472df6 --- /dev/null +++ b/internal/depbudget/main.go @@ -0,0 +1,31 @@ +// Command depbudget reads `go mod edit -json` output on stdin and prints +// the module's direct (non-indirect) requirements, one per line. The +// Makefile's dep-budget target uses it to assert that the hotload core's +// only direct dependency is fsnotify. +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" +) + +type goMod struct { + Require []struct { + Path string + Indirect bool + } +} + +func main() { + var mod goMod + if err := json.NewDecoder(os.Stdin).Decode(&mod); err != nil { + log.Fatalf("depbudget: decoding go mod json: %v", err) + } + for _, req := range mod.Require { + if !req.Indirect { + fmt.Println(req.Path) + } + } +} diff --git a/internal/gen/main.go b/internal/gen/main.go new file mode 100644 index 0000000..3e99325 --- /dev/null +++ b/internal/gen/main.go @@ -0,0 +1,279 @@ +// Command gen generates the optional-interface combination wrappers for +// hotload (conn_combos_gen.go, stmt_combos_gen.go) and the capability views +// for the dbfake test driver (internal/dbfake/views_gen.go). +// +// Hotload must expose an optional driver interface on a wrapped conn or stmt +// if and only if the underlying object implements it: database/sql discovers +// capabilities with type assertions, so the method must actually exist or +// not exist. One struct per interface subset is required for that, and this +// program emits them so adding an interface later is a regen instead of +// hand-editing dozens of types. +// +// It is invoked from the repository root via go:generate (see conn.go). +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "os" +) + +// pick names the piece type embedded for one optional interface, the letter +// used in generated type-name suffixes, and the interfaces the combination +// gains when the piece is present. +type pick struct { + letter string + piece string + ifaces []string +} + +func main() { + writeFile("conn_combos_gen.go", genHotloadCombos( + "conn", "*baseConn", + "// wrapConn wraps b so that the returned driver.Conn exposes an optional\n"+ + "// interface if and only if the underlying conn supports the capability.\n"+ + "// The legacy Execer/Queryer interfaces are collapsed into their context\n"+ + "// flavors: the wrapper only ever exposes ExecerContext/QueryerContext and\n"+ + "// replicates database/sql's legacy fallback internally (see\n"+ + "// baseConn.execContext and baseConn.queryContext).\n"+ + "func connFlags(c driver.Conn) uint8 {\n"+ + " var f uint8\n"+ + " if _, ok := c.(driver.ExecerContext); ok {\n"+ + " f |= 1 << 0\n"+ + " } else if _, ok := c.(driver.Execer); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 0\n"+ + " }\n"+ + " if _, ok := c.(driver.QueryerContext); ok {\n"+ + " f |= 1 << 1\n"+ + " } else if _, ok := c.(driver.Queryer); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 1\n"+ + " }\n"+ + " if _, ok := c.(driver.Pinger); ok {\n"+ + " f |= 1 << 2\n"+ + " }\n"+ + " if _, ok := c.(driver.NamedValueChecker); ok {\n"+ + " f |= 1 << 3\n"+ + " }\n"+ + " return f\n"+ + "}\n", + "wrapConn", "b", "driver.Conn", "connFlags(b.inner)", + []pick{ + {"E", "cExecer", []string{"driver.ExecerContext"}}, + {"Q", "cQueryer", []string{"driver.QueryerContext"}}, + {"P", "cPinger", []string{"driver.Pinger"}}, + {"N", "cNVChecker", []string{"driver.NamedValueChecker"}}, + }, + )) + + writeFile("stmt_combos_gen.go", genHotloadCombos( + "stmt", "*baseStmt", + "// stmtFlags reports which optional interfaces the wrapped stmt must expose.\n"+ + "func stmtFlags(s driver.Stmt) uint8 {\n"+ + " var f uint8\n"+ + " if _, ok := s.(driver.StmtExecContext); ok {\n"+ + " f |= 1 << 0\n"+ + " }\n"+ + " if _, ok := s.(driver.StmtQueryContext); ok {\n"+ + " f |= 1 << 1\n"+ + " }\n"+ + " if _, ok := s.(driver.ColumnConverter); ok { //nolint:staticcheck // legacy interface intentionally supported\n"+ + " f |= 1 << 2\n"+ + " }\n"+ + " if _, ok := s.(driver.NamedValueChecker); ok {\n"+ + " f |= 1 << 3\n"+ + " }\n"+ + " return f\n"+ + "}\n", + "wrapStmt", "b", "driver.Stmt", "stmtFlags(b.inner)", + []pick{ + {"E", "sExecer", []string{"driver.StmtExecContext"}}, + {"Q", "sQueryer", []string{"driver.StmtQueryContext"}}, + {"C", "sColConv", []string{"driver.ColumnConverter"}}, + {"N", "sNVChecker", []string{"driver.NamedValueChecker"}}, + }, + )) + + writeFile("internal/dbfake/views_gen.go", genDbfakeViews()) +} + +func writeFile(path string, src []byte) { + formatted, err := format.Source(src) + if err != nil { + log.Fatalf("formatting %s: %v\n%s", path, err, src) + } + if err := os.WriteFile(path, formatted, 0o644); err != nil { + log.Fatalf("writing %s: %v", path, err) + } + fmt.Printf("wrote %s\n", path) +} + +func header(pkg string) string { + return "// Code generated by internal/gen. DO NOT EDIT.\n\npackage " + pkg + "\n\nimport \"database/sql/driver\"\n\n" +} + +// suffix builds the type-name suffix for a flag mask, e.g. mask 0b0011 over +// conn picks -> "EQ". +func suffix(mask int, picks []pick) string { + s := "" + for i, p := range picks { + if mask&(1< 0 { - return errors.New(diffStr) - } - return nil -} diff --git a/internal/secret_sink.go b/internal/secret_sink.go index 8afa669..48e2180 100644 --- a/internal/secret_sink.go +++ b/internal/secret_sink.go @@ -1,10 +1,10 @@ package internal import ( + "crypto/rand" + "encoding/hex" "sync" "time" - - "github.com/google/uuid" ) const ( @@ -69,11 +69,11 @@ func (rss *RandomSecretSink) Add(actualSecret string) (randomSecret string, err delete(rss.secretStore, oldestData.actualSecret) } - guid, err := uuid.NewRandom() - if err != nil { + buf := make([]byte, randomLen/2) + if _, err := rand.Read(buf); err != nil { return "", err } - randomSecret = guid.String()[:randomLen] + randomSecret = hex.EncodeToString(buf) secretData = &RandomSecretData{ actualSecret: actualSecret, diff --git a/internal/testutil/leak.go b/internal/testutil/leak.go new file mode 100644 index 0000000..412f3e8 --- /dev/null +++ b/internal/testutil/leak.go @@ -0,0 +1,87 @@ +// Package testutil holds small stdlib-only helpers for hotload's tests. +package testutil + +import ( + "runtime" + "strings" + "testing" + "time" +) + +// NoLeaks registers a cleanup that fails the test if goroutines are still +// running once the test — including cleanups registered after this call — +// has finished. Call it first in the test, before opening any resources, so +// its cleanup runs last (cleanups run in LIFO order). +// +// Transient goroutines get a grace period: the check retries until the +// stragglers exit or a deadline passes. +func NoLeaks(t *testing.T) { + t.Helper() + t.Cleanup(func() { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + var extra []string + for { + extra = leakedGoroutines() + if len(extra) == 0 { + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(5 * time.Millisecond) + } + t.Errorf("leaked %d goroutine(s):\n%s", len(extra), strings.Join(extra, "\n\n")) + }) +} + +// allowedStackFragments mark goroutines that belong to the test harness or +// the runtime rather than the code under test. +var allowedStackFragments = []string{ + "testing.", // test runner goroutines (tRunner, parallel subtests, …) + "runtime.goexit", // never matches a frame list alone; kept for safety + "os/signal.", + "runtime/trace.", + "runtime.ReadTrace", +} + +func leakedGoroutines() []string { + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + stacks := strings.Split(strings.TrimSpace(string(buf[:n])), "\n\n") + + var out []string + for i, s := range stacks { + if i == 0 { + // The goroutine running this check. + continue + } + if isAllowed(s) { + continue + } + out = append(out, s) + } + return out +} + +func isAllowed(stack string) bool { + for _, frag := range allowedStackFragments { + if strings.Contains(stack, frag) { + return true + } + } + return false +} + +// WaitFor polls cond every millisecond until it returns true or the timeout +// elapses, failing the test in the latter case. +func WaitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out after %v waiting for %s", timeout, what) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/url_test.go b/internal/url_test.go index 48ab8a4..ee5089f 100644 --- a/internal/url_test.go +++ b/internal/url_test.go @@ -3,43 +3,61 @@ package internal import ( "fmt" "net/url" + "regexp" + "strings" "testing" - - "github.com/google/uuid" ) func TestRedactUrl(t *testing.T) { - nrr := NewNonRandomReader(1) - uuid.SetRand(nrr) - testcases := []struct { - inputDsn string - expectDsn string + inputDsn string + // expectPattern matches the redacted DSN; the password is replaced + // with a random token, asserted separately. + expectPattern string }{ { - inputDsn: "qwerty", - expectDsn: "//u---r:01020304@qwerty", + inputDsn: "qwerty", + expectPattern: `^//u---r:[0-9a-f]{8}@qwerty$`, }, { - inputDsn: "mysql://u:p@amazon.rds.com:5432/contacts", - expectDsn: "mysql://u---u:11121314@amazon.rds.com:5432/contacts", + inputDsn: "mysql://u:p@amazon.rds.com:5432/contacts", + expectPattern: `^mysql://u---u:[0-9a-f]{8}@amazon\.rds\.com:5432/contacts$`, }, { - inputDsn: "postgresql://admin:test@localhost:5432/hotload_test?sslmode=disable", - expectDsn: "postgresql://a---n:21222324@localhost:5432/hotload_test?sslmode=disable", + inputDsn: "postgresql://admin:test@localhost:5432/hotload_test?sslmode=disable", + expectPattern: `^postgresql://a---n:[0-9a-f]{8}@localhost:5432/hotload_test\?sslmode=disable$`, }, } for _, tt := range testcases { t.Run(tt.inputDsn, func(t *testing.T) { gotDsn := RedactUrl(tt.inputDsn) - if gotDsn != tt.expectDsn { - t.Errorf("expectDsn='%s' gotDsn='%s'", tt.expectDsn, gotDsn) + re := regexp.MustCompile(tt.expectPattern) + if !re.MatchString(gotDsn) { + t.Errorf("RedactUrl(%q) = %q, want match for %q", tt.inputDsn, gotDsn, tt.expectPattern) + } + if strings.Contains(gotDsn, ":test@") { + t.Errorf("RedactUrl(%q) = %q leaked the password", tt.inputDsn, gotDsn) } }) } } +// TestRedactUrlStablePassword verifies that the same password maps to the +// same random token across calls, so log lines remain correlatable. +func TestRedactUrlStablePassword(t *testing.T) { + first := RedactUrl("postgresql://admin:hunter2@localhost/db") + second := RedactUrl("postgresql://admin:hunter2@localhost/db") + if first != second { + t.Errorf("redaction not stable for identical input: %q vs %q", first, second) + } + + other := RedactUrl("postgresql://admin:different@localhost/db") + if other == first { + t.Errorf("different passwords redacted to the same DSN: %q", other) + } +} + func TestUrlEncodedQueryParams(t *testing.T) { testcases := []struct { inputParams url.Values diff --git a/k8ssecret/e2e_test.go b/k8ssecret/e2e_test.go new file mode 100644 index 0000000..17e49d0 --- /dev/null +++ b/k8ssecret/e2e_test.go @@ -0,0 +1,156 @@ +package k8ssecret + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "sync" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// TestEndToEndThroughSQLOpen drives the strategy through the real hotload +// core and database/sql: registration via this package's init, DSN parsing +// (including the leading slash the core puts on the path component), watch +// establishment at sql.Open, and update propagation into the pool. +func TestEndToEndThroughSQLOpen(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + + // Swap the package-registered strategy instance for one bound to the + // fake clientset; restore a pristine instance afterwards. + hotload.UnregisterStrategy("k8ssecret") + s := NewStrategyWithClientset(cs) + s.backoff = 20 * time.Millisecond + hotload.RegisterStrategy("k8ssecret", s) + t.Cleanup(func() { + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategy()) + }) + + registerFakeDriverOnce() + + db, err := sql.Open("hotload", "k8ssecret://k8sfake/mydb?namespace=prod&dsn=dsn") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + defer db.Close() + + queryDSN := func() (string, error) { + var dsn string + err := db.QueryRow("SELECT dsn").Scan(&dsn) + return dsn, err + } + + got, err := queryDSN() + if err != nil { + t.Fatal(err) + } + if got != "dsn-1" { + t.Fatalf("initial query routed to %q, want dsn-1", got) + } + + // The fake clientset cannot replay events from before a watch is + // registered (a real API server can, via the resource version), so wait + // for the strategy's watch before rotating the secret. + awaitWatchReady(t, ready) + + // Rotate the secret; the pool must converge on the new DSN. + if _, err := cs.CoreV1().Secrets("prod").Update(context.Background(), + makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(5 * time.Second) + for { + got, err := queryDSN() + if err == nil && got == "dsn-2" { + return + } + if time.Now().After(deadline) { + t.Fatalf("pool did not converge on dsn-2 (last: %q, err: %v)", got, err) + } + time.Sleep(10 * time.Millisecond) + } +} + +// TestSQLOpenErrorsOnMissingSecret: configuration problems surface at +// sql.Open (the hotload core starts the watch there). +func TestSQLOpenErrorsOnMissingSecret(t *testing.T) { + cs, _ := fakeClientset(t) + + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategyWithClientset(cs)) + t.Cleanup(func() { + hotload.UnregisterStrategy("k8ssecret") + hotload.RegisterStrategy("k8ssecret", NewStrategy()) + }) + + registerFakeDriverOnce() + + if _, err := sql.Open("hotload", "k8ssecret://k8sfake/nosuchsecret?namespace=prod"); err == nil { + t.Fatal("expected sql.Open to fail for a missing secret") + } +} + +// A minimal driver whose conns answer "SELECT dsn" with the DSN they were +// opened with, so tests can observe which generation served a query. (The +// hotload core's richer fake lives in its internal packages, which this +// module cannot import.) +var registerFakeDriver sync.Once + +func registerFakeDriverOnce() { + registerFakeDriver.Do(func() { + hotload.RegisterSQLDriver("k8sfake", fakeDriver{}) + }) +} + +type fakeDriver struct{} + +func (fakeDriver) Open(dsn string) (driver.Conn, error) { return &fakeConn{dsn: dsn}, nil } + +type fakeConn struct{ dsn string } + +func (c *fakeConn) Prepare(query string) (driver.Stmt, error) { return &fakeStmt{conn: c}, nil } +func (c *fakeConn) Close() error { return nil } +func (c *fakeConn) Begin() (driver.Tx, error) { return fakeTx{}, nil } + +func (c *fakeConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return &fakeRows{dsn: c.dsn}, nil +} + +type fakeStmt struct{ conn *fakeConn } + +func (s *fakeStmt) Close() error { return nil } +func (s *fakeStmt) NumInput() int { return -1 } +func (s *fakeStmt) Exec(args []driver.Value) (driver.Result, error) { + return driver.RowsAffected(1), nil +} +func (s *fakeStmt) Query(args []driver.Value) (driver.Rows, error) { + return &fakeRows{dsn: s.conn.dsn}, nil +} + +type fakeTx struct{} + +func (fakeTx) Commit() error { return nil } +func (fakeTx) Rollback() error { return nil } + +type fakeRows struct { + dsn string + done bool +} + +func (r *fakeRows) Columns() []string { return []string{"dsn"} } +func (r *fakeRows) Close() error { return nil } +func (r *fakeRows) Next(dest []driver.Value) error { + if r.done { + return io.EOF + } + dest[0] = r.dsn + r.done = true + return nil +} diff --git a/k8ssecret/go.mod b/k8ssecret/go.mod new file mode 100644 index 0000000..fc8de02 --- /dev/null +++ b/k8ssecret/go.mod @@ -0,0 +1,52 @@ +module github.com/infobloxopen/hotload/k8ssecret + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + k8s.io/api v0.32.3 + k8s.io/apimachinery v0.32.3 + k8s.io/client-go v0.32.3 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace github.com/infobloxopen/hotload/v3 => ../ diff --git a/k8ssecret/go.sum b/k8ssecret/go.sum new file mode 100644 index 0000000..da2dc2f --- /dev/null +++ b/k8ssecret/go.sum @@ -0,0 +1,154 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= +k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/k8ssecret/strategy.go b/k8ssecret/strategy.go new file mode 100644 index 0000000..9045ba9 --- /dev/null +++ b/k8ssecret/strategy.go @@ -0,0 +1,363 @@ +// Package k8ssecret implements a hotload strategy that watches Kubernetes +// Secrets for database connection string changes. +// +// In multi-namespace deployments a service sometimes needs credentials from +// a Secret in another team's namespace. Kubernetes does not allow mounting +// Secrets across namespaces, so the volume-based fsnotify strategy cannot +// see them; this strategy watches the Secret through the Kubernetes API +// instead. +// +// This package is a separate Go module +// (github.com/infobloxopen/hotload/k8ssecret) so client-go and its +// transitive dependencies stay out of the hotload core. +// +// # DSN format +// +// k8ssecret:///?namespace=&dsn= +// +// Parameters: +// - secret-name: the Kubernetes Secret name (the path component) +// - namespace: the namespace containing the Secret (default: the pod's +// namespace from the service account mount, else "default") +// - dsn: the data key within the Secret holding the connection string +// (default: "dsn.txt") +// +// The hotload parameters (forceKill, killWindow) may appear in the same +// query string. +// +// # Usage +// +// Import the package for its side effect of registering the strategy: +// +// import _ "github.com/infobloxopen/hotload/k8ssecret" +// +// db, err := sql.Open("hotload", "k8ssecret://pgx/myapp-db?namespace=prod&dsn=dsn.txt") +// +// The pod's service account needs get and watch permissions on the Secret. +package k8ssecret + +import ( + "context" + "fmt" + "net/url" + "os" + "path" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/logger" +) + +func init() { + hotload.RegisterStrategy("k8ssecret", NewStrategy()) +} + +const ( + defaultKey = "dsn.txt" + defaultBackoff = 2 * time.Second +) + +// ClientsetFunc constructs the Kubernetes clientset used by strategies that +// were not given one explicitly (including the instance registered by this +// package's init). It is consulted lazily on the first Watch; replace it +// before opening connections to use out-of-cluster config or fakes. +var ClientsetFunc = defaultClientset + +func defaultClientset() (kubernetes.Interface, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("k8ssecret: in-cluster config: %w", err) + } + return kubernetes.NewForConfig(cfg) +} + +// Strategy implements hotload.Strategy by watching Kubernetes Secrets. +type Strategy struct { + mu sync.Mutex + clientset kubernetes.Interface + watches map[watchKey]*secretWatch + backoff time.Duration +} + +// watchKey identifies one watched value. The data key is part of the +// identity: two DSNs reading different keys of the same Secret carry +// different values and must not share state. +type watchKey struct { + namespace string + name string + key string +} + +// secretWatch fans one watched Secret value out to its subscriptions. +// Subscription channels have capacity 1 and are written with drop-oldest +// semantics, so a subscriber always converges on the latest value and a +// slow subscriber can never block delivery. All channel sends and closes +// happen under Strategy.mu, so they cannot race. +type secretWatch struct { + cancel context.CancelFunc + value string + subs map[*subscription]struct{} +} + +// subscription is one active watch handed out by Watch (it implements +// hotload.Watchable). Closing the last subscription of a Secret value stops +// the underlying API watch. +type subscription struct { + strat *Strategy + wk watchKey + ch chan string + closed bool // guarded by strat.mu + stopAfter func() bool // detaches the ctx-cancel hook installed by Watch +} + +// Values implements hotload.Watchable. +func (sub *subscription) Values() <-chan string { + return sub.ch +} + +// Close implements hotload.Watchable. +func (sub *subscription) Close() error { + s := sub.strat + s.mu.Lock() + defer s.mu.Unlock() + if sub.closed { + return nil + } + sub.closed = true + sub.stopAfter() + close(sub.ch) + sw, ok := s.watches[sub.wk] + if !ok { + return nil + } + delete(sw.subs, sub) + if len(sw.subs) == 0 { + sw.cancel() + delete(s.watches, sub.wk) + } + return nil +} + +// NewStrategy creates a strategy that builds its clientset lazily from +// ClientsetFunc on first use. +func NewStrategy() *Strategy { + return &Strategy{ + watches: make(map[watchKey]*secretWatch), + backoff: defaultBackoff, + } +} + +// NewStrategyWithClientset creates a strategy using the given clientset; +// useful for tests (client-go's fake clientset) and out-of-cluster use. +func NewStrategyWithClientset(cs kubernetes.Interface) *Strategy { + s := NewStrategy() + s.clientset = cs + return s +} + +// secretName extracts the Secret name from the path component the hotload +// core passes to Watch. For "k8ssecret://pgx/myapp-db" that component is +// "/myapp-db" — with a leading slash that is not part of the name. +func secretName(pth string) string { + return strings.TrimPrefix(path.Clean(strings.TrimSpace(pth)), "/") +} + +// parseParams extracts the namespace and data key from the encoded query +// parameters of the hotload DSN. +func parseParams(pathQry string) (namespace, key string, err error) { + params, err := url.ParseQuery(strings.TrimSpace(pathQry)) + if err != nil { + return "", "", fmt.Errorf("k8ssecret: parse query %q: %w", pathQry, err) + } + namespace = params.Get("namespace") + if namespace == "" { + namespace = podNamespace() + } + key = params.Get("dsn") + if key == "" { + key = defaultKey + } + return namespace, key, nil +} + +// Watch implements hotload.Strategy. pth is the Secret name; pathQry +// carries the namespace and dsn parameters (see the package documentation). +// Every call returns an independent watch; watches on the same Secret value +// share one underlying API watch. +func (s *Strategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { + name := secretName(pth) + namespace, key, err := parseParams(strings.TrimSpace(pathQry)) + if err != nil { + return "", nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.clientset == nil { + cs, err := ClientsetFunc() + if err != nil { + return "", nil, err + } + s.clientset = cs + } + if s.watches == nil { + // A zero-value Strategy works too. + s.watches = make(map[watchKey]*secretWatch) + } + + wk := watchKey{namespace: namespace, name: name, key: key} + sw, exists := s.watches[wk] + if !exists { + secret, err := s.clientset.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", nil, fmt.Errorf("k8ssecret: get secret %s/%s: %w", namespace, name, err) + } + val, ok := secret.Data[key] + if !ok { + return "", nil, fmt.Errorf("k8ssecret: secret %s/%s has no key %q", namespace, name, key) + } + + watchCtx, cancel := context.WithCancel(context.Background()) + sw = &secretWatch{ + cancel: cancel, + value: string(val), + subs: make(map[*subscription]struct{}), + } + s.watches[wk] = sw + go s.runWatch(watchCtx, wk) + } + + sub := &subscription{strat: s, wk: wk, ch: make(chan string, 1)} + sw.subs[sub] = struct{}{} + sub.stopAfter = context.AfterFunc(ctx, func() { sub.Close() }) + return sw.value, sub, nil +} + +// runWatch maintains the API watch for one Secret value until its context +// is canceled. Every (re)connect first re-reads the Secret and delivers any +// value missed while disconnected, then watches from that read's resource +// version — so no modification is lost between the read and the watch, or +// while a dropped watch was reconnecting. +func (s *Strategy) runWatch(ctx context.Context, wk watchKey) { + for { + rv, err := s.catchUp(ctx, wk) + if err == nil { + err = s.consumeWatch(ctx, wk, rv) + } + if ctx.Err() != nil { + return + } + logger.ErrLogf("k8ssecret.runWatch:", "watch %s/%s interrupted: %v, reconnecting in %s", + wk.namespace, wk.name, err, s.backoff) + select { + case <-ctx.Done(): + return + case <-time.After(s.backoff): + } + } +} + +// catchUp reads the Secret's current value, delivers it if it changed, and +// returns the resource version to start the watch from. +func (s *Strategy) catchUp(ctx context.Context, wk watchKey) (string, error) { + secret, err := s.clientset.CoreV1().Secrets(wk.namespace).Get(ctx, wk.name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("get: %w", err) + } + if val, ok := secret.Data[wk.key]; ok { + s.deliver(wk, string(val)) + } + return secret.ResourceVersion, nil +} + +// consumeWatch processes watch events until the watch drops or the context +// is canceled. +func (s *Strategy) consumeWatch(ctx context.Context, wk watchKey, rv string) error { + watcher, err := s.clientset.CoreV1().Secrets(wk.namespace).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + wk.name, + ResourceVersion: rv, + }) + if err != nil { + return fmt.Errorf("start watch: %w", err) + } + defer watcher.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-watcher.ResultChan(): + if !ok { + return fmt.Errorf("watch channel closed") + } + switch event.Type { + case watch.Added, watch.Modified: + // Added matters: a Secret deleted and recreated comes back + // as Added, and so does the first event after a reconnect. + case watch.Error: + return fmt.Errorf("watch error event: %v", event.Object) + default: + // Deleted: keep serving the last known value, exactly like + // the fsnotify strategy when the watched file disappears; + // the recreate arrives as Added. + continue + } + secret, ok := event.Object.(*corev1.Secret) + if !ok || secret.Name != wk.name { + // The fake clientset (and old API servers) can ignore field + // selectors, so filter by name here too. + continue + } + if val, ok := secret.Data[wk.key]; ok { + s.deliver(wk, string(val)) + } + } + } +} + +// deliver pushes a changed value to every subscription of the watch. +// Channels have capacity 1; when full, the stale queued value is dropped so +// the subscriber always converges on the latest one (dropping the new value +// instead would leave a slow subscriber permanently stale). +func (s *Strategy) deliver(wk watchKey, val string) { + s.mu.Lock() + defer s.mu.Unlock() + sw, ok := s.watches[wk] + if !ok || sw.value == val { + return + } + sw.value = val + for sub := range sw.subs { + select { + case sub.ch <- val: + continue + default: + } + select { + case <-sub.ch: // drop the stale queued value + default: + } + select { + case sub.ch <- val: + default: + } + } +} + +// podNamespace returns the namespace of the current pod from the service +// account mount, or "default" when not running in-cluster. +func podNamespace() string { + if ns, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + return strings.TrimSpace(string(ns)) + } + return "default" +} diff --git a/k8ssecret/strategy_test.go b/k8ssecret/strategy_test.go new file mode 100644 index 0000000..87aeec0 --- /dev/null +++ b/k8ssecret/strategy_test.go @@ -0,0 +1,414 @@ +package k8ssecret + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func makeSecret(namespace, name string, data map[string]string) *corev1.Secret { + bs := make(map[string][]byte, len(data)) + for k, v := range data { + bs[k] = []byte(v) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: bs, + } +} + +// fakeClientset returns a fake clientset pre-populated with secrets, plus a +// channel that signals every time a secret watch has been registered with +// the tracker. Unlike a real API server, the fake ignores the resource +// version in watch options and only delivers events to already-registered +// watchers — so tests MUST receive from ready before mutating a secret, or +// the mutation can race the strategy's watch establishment and be lost (in +// production the watch replays from the resource version and no such gap +// exists). +func fakeClientset(t *testing.T, secrets ...*corev1.Secret) (kubernetes.Interface, <-chan struct{}) { + t.Helper() + cs := fake.NewClientset() + for _, sec := range secrets { + if _, err := cs.CoreV1().Secrets(sec.Namespace).Create(context.Background(), sec, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + ready := make(chan struct{}, 16) + cs.PrependWatchReactor("secrets", func(action k8stesting.Action) (bool, watch.Interface, error) { + w, err := cs.Tracker().Watch(action.GetResource(), action.GetNamespace()) + if err != nil { + return false, nil, err + } + ready <- struct{}{} // the watcher is registered; mutations are now visible to it + return true, w, nil + }) + return cs, ready +} + +func awaitWatchReady(t *testing.T, ready <-chan struct{}) { + t.Helper() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the strategy to establish its watch") + } +} + +// newTestStrategy returns a strategy on a fake clientset with a short +// reconnect backoff. Watches are tied to per-test contexts (see testCtx), +// so cleanup happens by cancellation. +func newTestStrategy(t *testing.T, cs kubernetes.Interface) *Strategy { + t.Helper() + s := NewStrategyWithClientset(cs) + s.backoff = 20 * time.Millisecond + return s +} + +// testCtx returns a context canceled when the test ends, closing every +// watch established with it. +func testCtx(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + +func awaitValue(t *testing.T, ch <-chan string, want string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case got, ok := <-ch: + if !ok { + t.Fatalf("update channel closed while waiting for %q", want) + } + t.Logf("update: %q", got) + if got == want { + return + } + case <-deadline: + t.Fatalf("timed out waiting for value %q", want) + } + } +} + +func awaitClosed(t *testing.T, ch <-chan string) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("timed out waiting for channel close") + } + } +} + +func updateSecret(t *testing.T, cs kubernetes.Interface, sec *corev1.Secret) { + t.Helper() + if _, err := cs.CoreV1().Secrets(sec.Namespace).Update(context.Background(), sec, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } +} + +func TestWatchInitialValue(t *testing.T) { + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "postgres://host/db"})) + s := newTestStrategy(t, cs) + + // The hotload core passes uri.Path, which has a leading slash. + val, w, err := s.Watch(testCtx(t), "/mydb", "dsn=dsn&namespace=prod") + if err != nil { + t.Fatalf("Watch: %v", err) + } + if val != "postgres://host/db" { + t.Errorf("initial value = %q, want postgres://host/db", val) + } + if w == nil || w.Values() == nil { + t.Fatal("expected non-nil watch and channel") + } +} + +func TestWatchSecretNotFound(t *testing.T) { + cs, _ := fakeClientset(t) + s := newTestStrategy(t, cs) + if _, _, err := s.Watch(testCtx(t), "/missing", "namespace=prod"); err == nil { + t.Fatal("expected error for missing secret") + } +} + +func TestWatchKeyNotFound(t *testing.T) { + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"password": "hunter2"})) + s := newTestStrategy(t, cs) + if _, _, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn"); err == nil { + t.Fatal("expected error for missing key") + } +} + +func TestWatchDefaults(t *testing.T) { + // Default key is dsn.txt; default namespace outside a pod is "default". + cs, _ := fakeClientset(t, makeSecret("default", "mydb", map[string]string{"dsn.txt": "postgres://host/db"})) + s := newTestStrategy(t, cs) + + val, _, err := s.Watch(testCtx(t), "/mydb", "") + if err != nil { + t.Fatalf("Watch: %v", err) + } + if val != "postgres://host/db" { + t.Errorf("value = %q, want postgres://host/db", val) + } +} + +func TestWatchSeesUpdates(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + awaitWatchReady(t, ready) + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, w.Values(), "dsn-2") + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-3"})) + awaitValue(t, w.Values(), "dsn-3") +} + +// TestWatchSeesDeleteAndRecreate: a deleted Secret keeps serving the last +// value; the recreate arrives as an Added event and propagates. +func TestWatchSeesDeleteAndRecreate(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + awaitWatchReady(t, ready) + + if err := cs.CoreV1().Secrets("prod").Delete(context.Background(), "mydb", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + if _, err := cs.CoreV1().Secrets("prod").Create(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"}), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + awaitValue(t, w.Values(), "dsn-2") +} + +// TestWatchCatchesUpAfterReconnect: a change made while the API watch is +// down must be delivered by the reconnect's catch-up read — the gap that +// loses updates when reconnects only resume watching from "now". +func TestWatchCatchesUpAfterReconnect(t *testing.T) { + cs := fake.NewClientset() + if _, err := cs.CoreV1().Secrets("prod").Create(context.Background(), makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"}), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + // Hand the strategy a controllable watcher, then kill it. + firstWatch := watch.NewFake() + watchCalls := make(chan struct{}, 10) + first := true + cs.PrependWatchReactor("secrets", func(action k8stesting.Action) (bool, watch.Interface, error) { + watchCalls <- struct{}{} + if first { + first = false + return true, firstWatch, nil + } + return false, nil, nil // fall through to the default tracker watch + }) + + s := newTestStrategy(t, cs) + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + <-watchCalls // first watch established + + // Change the secret while the (about to die) first watch sees nothing, + // then drop the watch. + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + firstWatch.Stop() + + // The reconnect's catch-up Get must deliver the missed value. + awaitValue(t, w.Values(), "dsn-2") + <-watchCalls // and a second watch was established +} + +// TestSlowSubscriberConvergesOnLatest: when a subscriber is not draining +// its channel, intermediate values may drop but the latest must win. +func TestSlowSubscriberConvergesOnLatest(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-0"})) + s := newTestStrategy(t, cs) + + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + awaitWatchReady(t, ready) + + // Push several updates without reading; the channel has capacity 1. + for i := 1; i <= 5; i++ { + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-x"})) + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-final"})) + } + // Now drain: the last value seen must be dsn-final, not a stale one + // stuck in the buffer. + awaitValue(t, w.Values(), "dsn-final") +} + +// TestMultipleSubscribersOneSecret: watches established by separate Watch +// calls get independent channels fed from one underlying API watch — even +// for an identical path and query. +func TestMultipleSubscribersOneSecret(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + _, w2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + awaitWatchReady(t, ready) + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, w1.Values(), "dsn-2") + awaitValue(t, w2.Values(), "dsn-2") +} + +// TestDistinctKeysAreIndependent: two watchers reading different data keys +// of the same Secret must each get their own key's value. +func TestDistinctKeysAreIndependent(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{ + "primary": "dsn-primary-1", + "replica": "dsn-replica-1", + })) + s := newTestStrategy(t, cs) + + ctx := testCtx(t) + vp, wP, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=primary") + if err != nil { + t.Fatal(err) + } + vr, wR, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=replica") + if err != nil { + t.Fatal(err) + } + if vp != "dsn-primary-1" || vr != "dsn-replica-1" { + t.Fatalf("initial values = %q / %q, want dsn-primary-1 / dsn-replica-1", vp, vr) + } + awaitWatchReady(t, ready) // primary key watch + awaitWatchReady(t, ready) // replica key watch + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{ + "primary": "dsn-primary-2", + "replica": "dsn-replica-2", + })) + awaitValue(t, wP.Values(), "dsn-primary-2") + awaitValue(t, wR.Values(), "dsn-replica-2") +} + +func TestCloseClosesChannel(t *testing.T) { + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + _, w, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + awaitClosed(t, w.Values()) + + // Close is idempotent. + if err := w.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } +} + +// TestCloseKeepsOtherSubscribers: closing one watch leaves the other one +// live. +func TestCloseKeepsOtherSubscribers(t *testing.T) { + cs, ready := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx := testCtx(t) + _, w1, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn&forceKill=true") + if err != nil { + t.Fatal(err) + } + _, w2, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + + awaitWatchReady(t, ready) + if err := w1.Close(); err != nil { + t.Fatal(err) + } + awaitClosed(t, w1.Values()) + + updateSecret(t, cs, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-2"})) + awaitValue(t, w2.Values(), "dsn-2") +} + +// TestCtxCancelClosesWatch: canceling the Watch context releases the watch, +// exactly like Close; the strategy stays usable and re-establishes the API +// watch for a subsequent Watch. +func TestCtxCancelClosesWatch(t *testing.T) { + cs, _ := fakeClientset(t, makeSecret("prod", "mydb", map[string]string{"dsn": "dsn-1"})) + s := newTestStrategy(t, cs) + + ctx, cancel := context.WithCancel(context.Background()) + _, w, err := s.Watch(ctx, "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatal(err) + } + cancel() + awaitClosed(t, w.Values()) + + val, w2, err := s.Watch(testCtx(t), "/mydb", "namespace=prod&dsn=dsn") + if err != nil { + t.Fatalf("Watch after cancel: %v", err) + } + if val != "dsn-1" { + t.Errorf("value after reopen = %q, want dsn-1", val) + } + if err := w2.Close(); err != nil { + t.Fatal(err) + } +} + +// TestSecretName covers the path normalization between the hotload DSN and +// the Kubernetes Secret name. +func TestSecretName(t *testing.T) { + cases := map[string]string{ + "/mydb": "mydb", + "mydb": "mydb", + " /mydb ": "mydb", + "/a/b": "a/b", // invalid as a Secret name; surfaces as a Get error + } + for in, want := range cases { + if got := secretName(in); got != want { + t.Errorf("secretName(%q) = %q, want %q", in, got, want) + } + } +} + +var _ runtime.Object = (*corev1.Secret)(nil) diff --git a/lifecycle_test.go b/lifecycle_test.go new file mode 100644 index 0000000..13ee944 --- /dev/null +++ b/lifecycle_test.go @@ -0,0 +1,396 @@ +package hotload_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "sync" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/dbfake" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// TestGracefulSwap: after a config change without forceKill, the pool +// discards old-generation conns on reuse and dials the new DSN; the old +// underlying conn is closed exactly once. +func TestGracefulSwap(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + + fx.pushAndWait("dsn-2") + + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN after change = %q, want dsn-2", got) + } + + conns := fx.drv.Conns() + if len(conns) < 2 { + t.Fatalf("expected a second conn to be dialed, got %d conns", len(conns)) + } + old := conns[0] + testutil.WaitFor(t, 2*time.Second, "old conn to close", old.Closed) + if n := old.CloseCount(); n != 1 { + t.Errorf("old conn closed %d times, want exactly 1", n) + } +} + +// TestGracefulGracePeriod: in graceful mode the previous generation's idle +// conns survive one change (the grace period) and are killed on the next +// change, without the pool ever touching them. +func TestGracefulGracePeriod(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() // dial conn 1 on dsn-1 + c1 := fx.drv.Conns()[0] + + fx.pushAndWait("dsn-2") + // Grace period: conn 1 must not be force-closed by this change. + time.Sleep(20 * time.Millisecond) + if c1.Closed() { + t.Fatal("conn from previous generation was closed during its grace period") + } + + fx.pushAndWait("dsn-3") + // Now conn 1 belonged to the generation before the previous one — killed. + testutil.WaitFor(t, 2*time.Second, "prev-prev conn to close", c1.Closed) + if n := c1.CloseCount(); n != 1 { + t.Errorf("conn closed %d times, want exactly 1", n) + } + + if got := fx.queryDSN(); got != "dsn-3" { + t.Fatalf("queryDSN = %q, want dsn-3", got) + } +} + +// TestSwapBackToOriginal: a -> b -> a is two real changes and ends with +// fresh conns on the original DSN. +func TestSwapBackToOriginal(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.pushAndWait("dsn-2") + ev := fx.pushAndWait("dsn-1") + if ev.GroupName != fx.dsn { + t.Errorf("event group = %q, want %q", ev.GroupName, fx.dsn) + } + + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + if len(fx.drv.Conns()) < 2 { + t.Errorf("expected a fresh conn after swapping back, got %d conns", len(fx.drv.Conns())) + } +} + +// TestUnchangedValueIgnored: pushing the same value is not a change. +func TestUnchangedValueIgnored(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.push("dsn-1") // identical; processed but ignored + fx.pushAndWait("dsn-2") + fx.noPendingChange() + + c1 := fx.drv.Conns()[0] + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } + _ = c1 +} + +// TestForceKillCancelsInflightExec: with forceKill, an exec blocked in the +// driver is canceled by the config change, the error surfaces as ErrHotSwap, +// and the conn is closed exactly once. +func TestForceKillCancelsInflightExec(t *testing.T) { + testutil.NoLeaks(t) + + started := make(chan struct{}, 10) + fx := newFixture(t, fxCfg{ + params: "forceKill=true", + execFn: func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + started <- struct{}{} + <-ctx.Done() + return nil, context.Cause(ctx) + }, + }) + + errCh := make(chan error, 1) + go func() { + _, err := fx.db.Exec("UPDATE x") + errCh <- err + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("exec never reached the driver") + } + + fx.pushAndWait("dsn-2") + + select { + case err := <-errCh: + if !errors.Is(err, hotload.ErrHotSwap) { + t.Fatalf("exec error = %v, want ErrHotSwap", err) + } + case <-time.After(2 * time.Second): + t.Fatal("exec did not return after forceKill change") + } + + old := fx.drv.Conns()[0] + testutil.WaitFor(t, 2*time.Second, "old conn to close", old.Closed) + if n := old.CloseCount(); n != 1 { + t.Errorf("old conn closed %d times, want exactly 1", n) + } + + fx.drv.ExecFn = nil // restore default behavior for the new generation + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } +} + +// TestForceKillBoundedByKillWindow: a driver that ignores context +// cancellation cannot wedge the run loop — after killWindow elapses the conn +// is force-closed and new work proceeds on the new DSN while the old +// operation is still parked. +func TestForceKillBoundedByKillWindow(t *testing.T) { + testutil.NoLeaks(t) + + started := make(chan struct{}, 10) + release := make(chan struct{}) + fx := newFixture(t, fxCfg{ + params: "forceKill=true&killWindow=30ms", + execFn: func(c *dbfake.Conn, ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + started <- struct{}{} + <-release // deliberately ignores ctx + return driver.RowsAffected(1), nil + }, + }) + defer close(release) + + errCh := make(chan error, 1) + go func() { + _, err := fx.db.Exec("UPDATE x") + errCh <- err + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("exec never reached the driver") + } + + fx.pushAndWait("dsn-2") + + old := fx.drv.Conns()[0] + testutil.WaitFor(t, 2*time.Second, "old conn to be force-closed", old.Closed) + + // The group must remain responsive with the old exec still parked. + fx.drv.ExecFn = nil + if got := fx.queryDSN(); got != "dsn-2" { + t.Fatalf("queryDSN = %q, want dsn-2", got) + } + select { + case err := <-errCh: + t.Logf("parked exec returned early: %v", err) + default: + // Still parked, as expected; released by the deferred close. + } +} + +// TestRunLoopNotBlockedBySlowDial: dials happen outside group locks, so a +// slow dial must not delay config-change processing (hotload v1 serialized +// these on one mutex). +func TestRunLoopNotBlockedBySlowDial(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + fx.drv.OpenDelay = 300 * time.Millisecond + + dialDone := make(chan struct{}) + go func() { + defer close(dialDone) + fx.queryDSN() // triggers the slow dial + }() + + time.Sleep(20 * time.Millisecond) // let the dial start + t0 := time.Now() + fx.pushAndWait("dsn-2") + if elapsed := time.Since(t0); elapsed > 200*time.Millisecond { + t.Errorf("config change took %v while a dial was in flight; the run loop appears blocked by dialing", elapsed) + } + <-dialDone +} + +// TestOpenErrors: bad DSNs and missing registrations fail at sql.Open (the +// connector is created there), with the sentinel errors preserved from v1. +func TestOpenErrors(t *testing.T) { + t.Run("unsupported strategy", func(t *testing.T) { + _, err := sql.Open("hotload", "nosuchstrategy://nosuchdriver/cfg") + if !errors.Is(err, hotload.ErrUnsupportedStrategy) { + t.Fatalf("err = %v, want ErrUnsupportedStrategy", err) + } + }) + + t.Run("unknown driver", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + // Reuse the registered strategy but point at an unregistered driver. + dsnURL := fmt.Sprintf("%s://%s%s", schemeOf(t, fx.dsn), "nosuchdriver", fixturePath) + _, err := sql.Open("hotload", dsnURL) + if !errors.Is(err, hotload.ErrUnknownDriver) { + t.Fatalf("err = %v, want ErrUnknownDriver", err) + } + }) + + t.Run("malformed killWindow", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + _, err := sql.Open("hotload", fx.dsn+"?killWindow=bogus") + if !errors.Is(err, hotload.ErrMalformedConnectionString) { + t.Fatalf("err = %v, want ErrMalformedConnectionString", err) + } + }) + + t.Run("watch error", func(t *testing.T) { + fx := newFixture(t, fxCfg{noDB: true}) + _, err := sql.Open("hotload", fx.dsn+"nosuchpath") + if err == nil { + t.Fatal("expected watch error for unknown path") + } + }) +} + +// TestGroupTeardownAndSharing: two sql.DB handles on the same DSN share one +// watch; the watch survives the first Close and stops after the last one, +// leaving no goroutines behind. +func TestGroupTeardownAndSharing(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + db2, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatal(err) + } + + if n := fx.strat.Watches(); n != 1 { + t.Fatalf("watches = %d, want 1 (groups must be shared per DSN)", n) + } + + if err := db2.Close(); err != nil { + t.Fatal(err) + } + if n := fx.strat.Watches(); n != 1 { + t.Fatalf("watches after first close = %d, want 1", n) + } + + if err := fx.db.Close(); err != nil { + t.Fatal(err) + } + testutil.WaitFor(t, 2*time.Second, "watch teardown", func() bool { + return fx.strat.Watches() == 0 + }) +} + +// TestReopenWhileClosing races the close of a DSN's last sql.DB against a +// fresh sql.Open of the same DSN. Whatever the interleaving, the surviving +// handle must keep receiving config changes: every Watch call gets its own +// update channel, and the dying group's watch teardown is serialized with +// new watch creation under the driver lock. +func TestReopenWhileClosing(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{noDB: true}) + + db, err := sql.Open("hotload", fx.dsn) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 20; i++ { + var ( + wg sync.WaitGroup + db2 *sql.DB + ) + wg.Add(2) + go func() { + defer wg.Done() + db.Close() + }() + go func() { + defer wg.Done() + var err error + db2, err = sql.Open("hotload", fx.dsn) + if err != nil { + t.Errorf("reopen: %v", err) + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // The surviving handle must observe a config change. + want := fmt.Sprintf("dsn-rw-%d", i) + fx.push(want) + testutil.WaitFor(t, 5*time.Second, "change to propagate to the reopened db", func() bool { + var got string + return db2.QueryRow("SELECT dsn").Scan(&got) == nil && got == want + }) + db = db2 + } + db.Close() +} + +// TestStrategyChannelClose: when the strategy closes its update channel the +// run loop exits, existing connections keep serving the last value, and a +// later db.Close still tears down cleanly. +func TestStrategyChannelClose(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + fx.strat.CloseChan(fixturePath) + + // Existing conns keep working on the last known value. + testutil.WaitFor(t, 2*time.Second, "queries to keep working", func() bool { + return fx.queryDSN() == "dsn-1" + }) +} + +// TestValidatorEviction: an underlying conn reporting IsValid()==false is +// discarded by the pool and replaced. +func TestValidatorEviction(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + + fx.queryDSN() + c1 := fx.drv.Conns()[0] + c1.SetInvalid() + + // The pool validates on put-back/reuse; the next queries must succeed + // on a fresh conn. + if got := fx.queryDSN(); got != "dsn-1" { + t.Fatalf("queryDSN = %q, want dsn-1", got) + } + testutil.WaitFor(t, 2*time.Second, "invalid conn to be discarded", c1.Closed) +} + +func schemeOf(t *testing.T, dsn string) string { + t.Helper() + for i := range dsn { + if dsn[i] == ':' { + return dsn[:i] + } + } + t.Fatalf("no scheme in %q", dsn) + return "" +} diff --git a/metrics/pathchksum_metrics.go b/metrics/pathchksum_metrics.go deleted file mode 100644 index a77616a..0000000 --- a/metrics/pathchksum_metrics.go +++ /dev/null @@ -1,153 +0,0 @@ -package metrics - -import ( - "errors" - "hash/crc64" - "os" - "path" - "strings" - "sync" - "time" - - "github.com/colega/gaugefuncvec" - "github.com/infobloxopen/hotload/logger" - "github.com/prometheus/client_golang/prometheus" -) - -const ( - PathChksumMetricsEnableEnvVar = "HOTLOAD_PATH_CHKSUM_METRICS_ENABLE" -) - -var ( - ErrDuplicatePath = errors.New("duplicate path") - ErrPathNotFound = errors.New("path not found") - - HotloadPathChksumTimestampSecondsName = "hotload_path_chksum_timestamp_seconds" - HotloadPathChksumTimestampSecondsHelp = "Hotload path checksum last changed (unix timestamp), by path" - HotloadPathChksumTimestampSecondsGaugeFuncVec = gaugefuncvec.New(prometheus.GaugeOpts{ - Name: HotloadPathChksumTimestampSecondsName, - Help: HotloadPathChksumTimestampSecondsHelp, - }, []string{PathKey}) - - crc64Table = crc64.MakeTable(crc64.ECMA) - - defaultPathChksum *pathChksum -) - -func init() { - defaultPathChksum = newPathChksum(DefaultFileHasher) - prometheus.MustRegister(HotloadPathChksumTimestampSecondsGaugeFuncVec) -} - -// AddToDefaultPathChksum adds a path to the global defaultPathChksum for checksum metrics -func AddToDefaultPathChksum(pathStr string) error { - return defaultPathChksum.addPath(pathStr) -} - -type pathChksum struct { - sync.RWMutex // used to synchronize changes to the set of paths being monitored - enabled bool - fileHasher FileHasher - paths map[string]*pathRecord -} - -type pathRecord struct { - path string - crc64 uint64 - lastChanged int64 -} - -// Define FileHasher type so we can mock it for unit-testing -type FileHasher func(filePath string) (uint64, error) - -// DefaultFileHasher hashes file contents using CRC64 -func DefaultFileHasher(filePath string) (uint64, error) { - pathBytes, err := os.ReadFile(filePath) - if err != nil { - logger.ErrLogf("DefaultFileHasher", "ReadFile(%s) err=%s", filePath, err) - return 0, err - } - - newCrc64 := crc64.Checksum(pathBytes, crc64Table) - return newCrc64, nil -} - -// newPathChksum returns a new PathChksum -func newPathChksum(fileHasher FileHasher) *pathChksum { - if fileHasher == nil { - panic("nil FileHasher") - } - - enabledStr := strings.ToLower(strings.TrimSpace(os.Getenv(PathChksumMetricsEnableEnvVar))) - enabledFlg := false - switch enabledStr { - case "1", "true", "yes": - enabledFlg = true - } - - pthm := &pathChksum{ - enabled: enabledFlg, - fileHasher: fileHasher, - paths: make(map[string]*pathRecord), - } - return pthm -} - -// addPath adds a path to be checksum'd for change in contents, -// and registers path for metrics collection -func (pthm *pathChksum) addPath(pathStr string) error { - if !pthm.enabled { - return nil - } - - pathStr = CleanPath(pathStr) - - pthm.Lock() - defer pthm.Unlock() - - pathRec, found := pthm.paths[pathStr] - if found { - return ErrDuplicatePath - } - - pathRec = &pathRecord{ - path: pathStr, - } - pthm.paths[pathStr] = pathRec - - scraperFn := func() float64 { - if !pthm.enabled { - return float64(0) - } - - newCrc64, err := pthm.fileHasher(pathRec.path) - if err != nil { - // log error, but continue - logger.ErrLogf("PathChksum.scraper", "fileHasher(%s) err=%s", pathRec.path, err) - } else if pathRec.crc64 != newCrc64 { - pathRec.crc64 = newCrc64 - pathRec.lastChanged = time.Now().Unix() - } - - return float64(pathRec.lastChanged) - } - - HotloadPathChksumTimestampSecondsGaugeFuncVec.MustRegister( - prometheus.Labels{PathKey: pathStr}, - scraperFn, - ) - - return nil -} - -// CleanPath cleans and trimspaces path strings -func CleanPath(pathStr string) string { - return path.Clean(strings.TrimSpace(pathStr)) -} - -var ExpectHotloadPathChksumTimestampSecondsPreamble = ` -# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path -# TYPE hotload_path_chksum_timestamp_seconds gauge` - -var ExpectHotloadPathChksumTimestampSecondsRegexp = ` -hotload_path_chksum_timestamp_seconds{path="%s"} \d\.\d+e\+\d+` diff --git a/metrics/pathchksum_test.go b/metrics/pathchksum_test.go deleted file mode 100644 index 3810f34..0000000 --- a/metrics/pathchksum_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package metrics - -import ( - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/infobloxopen/hotload/internal" - "github.com/prometheus/client_golang/prometheus/testutil" -) - -var delayDur = 616789 * time.Microsecond - -func MyTestFileHasher(filePath string) (uint64, error) { - return uint64(time.Now().UnixMicro()), nil -} - -func TestPathChksumMetricsDisabled(t *testing.T) { - os.Unsetenv(PathChksumMetricsEnableEnvVar) - pthmDisabled := newPathChksum(MyTestFileHasher) - - time.Sleep(delayDur) - err := testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (zero paths) err=%v", err) - } - - time.Sleep(delayDur) - rwDsn := "/env/unset/db-dsn/dsn.txt" - err = pthmDisabled.addPath(rwDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", rwDsn, err) - } - err = testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (one path) err=%v", err) - } - - time.Sleep(delayDur) - roDsn := "/env/unset/db-dsn/ro-dsn.txt" - err = pthmDisabled.addPath("/env/unset/db-dsn/ro-dsn.txt") - if err != nil { - t.Errorf("addPath(%s) err=%v", roDsn, err) - } - err = testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (two paths) err=%v", err) - } -} - -func TestPathChksumMetricsEnabled(t *testing.T) { - os.Setenv(PathChksumMetricsEnableEnvVar, "true") - pthmEnabled := newPathChksum(MyTestFileHasher) - - time.Sleep(delayDur) - err := testutil.CollectAndCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader("")) - if err != nil { - t.Errorf("CollectAndCompare (zero paths) err=%v", err) - } - - time.Sleep(delayDur) - rwDsn := "/env/true/db-dsn/dsn.txt" - err = pthmEnabled.addPath(rwDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", rwDsn, err) - } - err = internal.CollectAndRegexpCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, rwDsn)), - HotloadPathChksumTimestampSecondsName) - if err != nil { - t.Errorf("CollectAndRegexpCompare (one path) err=%v", err) - } - - time.Sleep(delayDur) - roDsn := "/env/true/db-dsn/ro-dsn.txt" - err = pthmEnabled.addPath(roDsn) - if err != nil { - t.Errorf("addPath(%s) err=%v", roDsn, err) - } - err = internal.CollectAndRegexpCompare(HotloadPathChksumTimestampSecondsGaugeFuncVec, - strings.NewReader(ExpectHotloadPathChksumTimestampSecondsPreamble+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, rwDsn)+ - fmt.Sprintf(ExpectHotloadPathChksumTimestampSecondsRegexp, roDsn)), - HotloadPathChksumTimestampSecondsName) - if err != nil { - t.Errorf("CollectAndRegexpCompare (two paths) err=%v", err) - } - - err = pthmEnabled.addPath(rwDsn) - if err != ErrDuplicatePath { - t.Errorf("addPath(%s): expecting ErrDuplicatePath, but got err=%v", rwDsn, err) - } -} diff --git a/metrics/prometheus.go b/metrics/prometheus.go deleted file mode 100644 index 6f841e2..0000000 --- a/metrics/prometheus.go +++ /dev/null @@ -1,85 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -const ( - GRPCMethodKey = "grpc_method" - GRPCServiceKey = "grpc_service" - StatementKey = "stmt" // either exec or query - ExecStatement = "exec" - QueryStatement = "query" - - StrategyKey = "strategy" - PathKey = "path" - UrlKey = "url" -) - -// SqlStmtsSummary is a prometheus metric to keep track of the number of times -// a sql statement is called in a transaction by statement type per grpc service -var SqlStmtsSummaryName = "transaction_sql_stmts" -var SqlStmtsSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Name: SqlStmtsSummaryName, - Help: "The number of sql stmts called in a transaction by statement type per grpc service and method", -}, []string{GRPCServiceKey, GRPCMethodKey, StatementKey}) - -// HotloadModtimeLatencyHistogram is modtime latency histogram (in seconds) -// ie: each sample datapoint is time.Now().Sub(Modtime) -var HotloadModtimeLatencyHistogramName = "hotload_modtime_latency_histogram" -var HotloadModtimeLatencyHistogramHelp = "Hotload modtime latency histogram (seconds) by strategy and path" -var HotloadModtimeLatencyHistogramDefBuckets = []float64{900, 1800, 2700, 3600, 4500, 5400, 7200, 10800, 14400, 28800, 86400} -var HotloadModtimeLatencyHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: HotloadModtimeLatencyHistogramName, - Help: HotloadModtimeLatencyHistogramHelp, - Buckets: HotloadModtimeLatencyHistogramDefBuckets, -}, []string{StrategyKey, PathKey}) - -func ObserveHotloadModtimeLatencyHistogram(strategy, path string, val float64) { - HotloadModtimeLatencyHistogram.WithLabelValues(strategy, path).Observe(val) -} - -// HotloadChangeTotal is count of changes detected by hotload -var HotloadChangeTotalName = "hotload_change_total" -var HotloadChangeTotalHelp = "Hotload change total by url" -var HotloadChangeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: HotloadChangeTotalName, - Help: HotloadChangeTotalHelp, -}, []string{UrlKey}) - -func IncHotloadChangeTotal(url string) { - HotloadChangeTotal.WithLabelValues(url).Inc() -} - -// HotloadLastChangedTimestampSeconds is timestamp when hotload last detected change (unix timestamp) -var HotloadLastChangedTimestampSecondsName = "hotload_last_changed_timestamp_seconds" -var HotloadLastChangedTimestampSecondsHelp = "Hotload last changed (unix timestamp), by url" -var HotloadLastChangedTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: HotloadLastChangedTimestampSecondsName, - Help: HotloadLastChangedTimestampSecondsHelp, -}, []string{UrlKey}) - -func SetHotloadLastChangedTimestampSeconds(url string, val float64) { - HotloadLastChangedTimestampSeconds.WithLabelValues(url).Set(val) -} - -func GetCollectors() []prometheus.Collector { - return []prometheus.Collector{ - SqlStmtsSummary, - HotloadModtimeLatencyHistogram, - HotloadChangeTotal, - HotloadLastChangedTimestampSeconds, - } -} - -// ResetCollectors is useful for testing -func ResetCollectors() { - SqlStmtsSummary.Reset() - HotloadModtimeLatencyHistogram.Reset() - HotloadChangeTotal.Reset() - HotloadLastChangedTimestampSeconds.Reset() -} - -func init() { - prometheus.MustRegister(GetCollectors()...) -} diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go deleted file mode 100644 index ff49903..0000000 --- a/metrics/prometheus_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package metrics - -import ( - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" -) - -var _ = Describe("PrometheusMetric", func() { - It("Should register a prometheus metric", func() { - // This test is a placeholder for a real test - err := prometheus.Register(SqlStmtsSummary) - Expect(err).Should(HaveOccurred()) - Expect(errors.As(err, &prometheus.AlreadyRegisteredError{})).Should(BeTrue()) - }) -}) diff --git a/modtime/modtime_monitor.go b/modtime/modtime_monitor.go index 274d34b..414426c 100644 --- a/modtime/modtime_monitor.go +++ b/modtime/modtime_monitor.go @@ -11,8 +11,8 @@ import ( "sync/atomic" "time" - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/logger" ) var ( @@ -155,9 +155,12 @@ func (mtm *ModTimeMonitor) checkPathModTimes(ctx context.Context, nowTime time.T pathRec.modTime.Store(newTime) } - latencyNano := nowTime.Sub(pathRec.modTime.Load().(time.Time)) - latencySecs := latencyNano.Seconds() - metrics.ObserveHotloadModtimeLatencyHistogram(pkey.strategy, pkey.path, latencySecs) + latency := nowTime.Sub(pathRec.modTime.Load().(time.Time)) + hotload.EmitModTimeEvent(hotload.ModTimeEvent{ + Strategy: pkey.strategy, + Path: pkey.path, + Latency: latency, + }) } } diff --git a/modtime/modtime_options.go b/modtime/modtime_options.go index de491a1..8ef3865 100644 --- a/modtime/modtime_options.go +++ b/modtime/modtime_options.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/infobloxopen/hotload/logger" + "github.com/infobloxopen/hotload/v3/logger" ) var ( diff --git a/modtime/modtime_test.go b/modtime/modtime_test.go index e53acd1..21d9273 100644 --- a/modtime/modtime_test.go +++ b/modtime/modtime_test.go @@ -3,377 +3,170 @@ package modtime import ( "context" "fmt" - "log" - "math" - "strings" + "sync" "testing" "testing/fstest" "time" - internal "github.com/infobloxopen/hotload/internal" - "github.com/infobloxopen/hotload/metrics" + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal" + "github.com/infobloxopen/hotload/v3/internal/testutil" ) -var fsnotifyStrategy = "fsnotify" +const testStrategy = "fsnotify" -// TestAgainstUnixFS verifies that the use of io/fs.FS in the implementation -// works against the real Unix FS -func TestAgainstUnixFS(t *testing.T) { - var zeroTime time.Time - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() - - metrics.ResetCollectors() - - logfn := func(args ...any) { - log.Println(args...) +func mustParseRFC3339(t *testing.T, s string) time.Time { + t.Helper() + ts, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) } + return ts +} - // Create ModTimeMonitor that monitors the (default) real host Unix FS - mtm := NewModTimeMonitor(ctx, - WithCheckInterval(time.Millisecond*200), - WithLogger(logfn), - WithErrLogger(logfn), - ) - - // Add well-known Unix path to monitor the mod-time of - pth := "/dev/null" - mtm.AddMonitoredPath(fsnotifyStrategy, pth) +// eventRecorder captures ModTimeEvents emitted through hotload hooks. +type eventRecorder struct { + mu sync.Mutex + events []hotload.ModTimeEvent +} - // Give time for ModTimeMonitor background thread to check mod-times - time.Sleep(time.Millisecond * 500) +func newEventRecorder(t *testing.T) *eventRecorder { + t.Helper() + r := &eventRecorder{} + hotload.RegisterHooks(hotload.Hooks{ + OnModTimeCheck: func(ev hotload.ModTimeEvent) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, ev) + }, + }) + return r +} - // Verify valid mod-time has been retrieved - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if zeroTime.After(sts.ModTime) { - t.Errorf("GetPathStatus(%s): unexpected ModTime=%+v", pth, sts.ModTime) +func (r *eventRecorder) forPath(path string) []hotload.ModTimeEvent { + r.mu.Lock() + defer r.mu.Unlock() + var out []hotload.ModTimeEvent + for _, ev := range r.events { + if ev.Path == path { + out = append(out, ev) } } + return out +} - // Unfortunately for some reason, os.Chtimes() returns - // "chtimes /dev/null: operation not permitted" - // so we can't update /dev/null modtime for additional testing - - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 200) +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + testutil.WaitFor(t, timeout, what, cond) } -// TestAgainstMapFS verifies using MapFS mock FS -func TestAgainstMapFS(t *testing.T) { - var zeroTime time.Time - prevTime := zeroTime +// TestAgainstUnixFS verifies the monitor works against the real filesystem. +func TestAgainstUnixFS(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() + mtm := NewModTimeMonitor(ctx, WithCheckInterval(20*time.Millisecond)) + pth := "/dev/null" + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Fatal(err) + } - metrics.ResetCollectors() + waitFor(t, 2*time.Second, "a modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && !sts.ModTime.IsZero() + }) +} + +// TestAgainstMapFS drives the monitor with a mock filesystem and verifies +// path status updates and latency events emitted through hotload hooks. +func TestAgainstMapFS(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rec := newEventRecorder(t) - // Create MapFS mock FS pth := "/foo/bar" mfs := internal.NewSafeMapFS() - mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: MustParseRFC3339("0002-02-02T02:02:02Z")}) - - checkIntv := time.Millisecond * 1000 - - // Create ModTimeMonitor that monitors mock FS - mtm := NewModTimeMonitor(ctx, - WithStatFS(mfs), - WithCheckInterval(checkIntv), - WithLogger(func(args ...any) { - log.Println(args...) - }), - ) - - // Add mock path to monitor the mod-time of. - mtm.AddMonitoredPath(fsnotifyStrategy, pth) - - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 2 cycles of checks. - time.Sleep(2*checkIntv + 10*time.Millisecond) - - // Verify valid mod-time has been retrieved - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.Before(prevTime) || sts.ModTime.Equal(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly not updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime - } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpInitial), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) + start := mustParseRFC3339(t, "2026-01-02T03:04:05Z") + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: start}); err != nil { + t.Fatal(err) } - // Update mock path mod-time - mapf, err := mfs.GetMapFile(pth) - if err != nil { - t.Errorf("GetMapFile(%s): unexpected err=%s", pth, err) + mtm := NewModTimeMonitor(ctx, WithStatFS(mfs), WithCheckInterval(20*time.Millisecond)) + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Fatal(err) + } + if err := mtm.AddMonitoredPath(testStrategy, pth); err != ErrDuplicatePath { + t.Errorf("second AddMonitoredPath err = %v, want ErrDuplicatePath", err) } - mapf.ModTime = time.Now() - mfs.UpsertMapFile(pth, mapf) - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 1 cycle of checks. - time.Sleep(1*checkIntv + 10*time.Millisecond) + waitFor(t, 2*time.Second, "first modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && sts.ModTime.Equal(start) + }) - // Verify modtime updated - sts, err = mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.Before(prevTime) || sts.ModTime.Equal(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly not updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime - } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpAfterModtimeUpdated), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) + // Bump the file's modtime; the monitor must observe it and the latency + // events must shrink accordingly. + now := time.Now() + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: now}); err != nil { + t.Fatal(err) } + waitFor(t, 2*time.Second, "updated modtime sample", func() bool { + sts, err := mtm.GetPathStatus(testStrategy, pth) + return err == nil && sts.ModTime.Equal(now) + }) - // Give time for ModTimeMonitor background thread to check mod-times. - // Wait 1 cycle of checks. - time.Sleep(1*checkIntv + 10*time.Millisecond) - - // Verify modtime NOT updated because mock path modtime was NOT updated - sts, err = mtm.GetPathStatus(fsnotifyStrategy, pth) - if err != nil { - t.Errorf("GetPathStatus(%s): unexpected err=%s", pth, err) - } else { - t.Logf("sts=%+v", sts) - if sts.ModTime.After(prevTime) { - t.Errorf("GetPathStatus(%s): unexpectedly updated: ModTime=%+v", pth, sts.ModTime) - } - prevTime = sts.ModTime + waitFor(t, 2*time.Second, "latency events", func() bool { + return len(rec.forPath(pth)) >= 2 + }) + evs := rec.forPath(pth) + first, last := evs[0], evs[len(evs)-1] + if first.Strategy != testStrategy { + t.Errorf("event strategy = %q, want %q", first.Strategy, testStrategy) } - err = internal.CollectAndRegexpCompare(metrics.HotloadModtimeLatencyHistogram, - strings.NewReader(expectMetricsRegexpAfterModtimeNotUpdated), - metrics.HotloadModtimeLatencyHistogramName) - if err != nil { - t.Errorf("CollectAndRegexpCompare(): unexpected err=\n%s", err) + // The first sample's latency is measured against the old modtime (huge); + // after the update the latency must be small. + if first.Latency < 24*time.Hour { + t.Errorf("first latency = %v, want large (old modtime)", first.Latency) + } + if last.Latency > time.Hour { + t.Errorf("latency after modtime update = %v, want small", last.Latency) } - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 200) + if _, err := mtm.GetPathStatus(testStrategy, "/no/such/path"); err != ErrPathNotFound { + t.Errorf("GetPathStatus(unknown) err = %v, want ErrPathNotFound", err) + } } -// TestConcurrency verifies thread-safety by spawning multiple -// threads all running with the same interval. -// Should be tested with go test -race flag. +// TestConcurrency exercises concurrent AddMonitoredPath/GetPathStatus calls +// against a running monitor; meaningful under -race. func TestConcurrency(t *testing.T) { - var zeroTime time.Time - ctx, cancelCtxFn := context.WithCancel(context.Background()) - defer cancelCtxFn() - - metrics.ResetCollectors() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - commonIntv := time.Millisecond * 100 - - // Create MapFS mock FS mfs := internal.NewSafeMapFS() - - modTimePaths := []struct { - pathStr string - startMtime time.Time - }{ - { - pathStr: "/concurrency/sub10", - startMtime: time.Now(), - }, - { - pathStr: "/concurrency/sub20", - startMtime: zeroTime, - }, - { - pathStr: "/concurrency/sub30", - startMtime: MustParseRFC3339("2010-04-01T23:07:59Z"), - }, - } - - // Add paths to mock FS - for _, pathRec := range modTimePaths { - mfs.UpsertMapFile(pathRec.pathStr, &fstest.MapFile{ModTime: pathRec.startMtime}) - } - - // Create ModTimeMonitor that monitors mock FS - mtm := NewModTimeMonitor(context.Background(), - WithStatFS(mfs), - WithCheckInterval(commonIntv), - WithLogger(func(args ...any) { - log.Println(args...) - }), - ) - - modTimeUpdaterLoop := func(ctx context.Context, t *testing.T, pathStr string, updateIntv time.Duration) { - mtm.log(fmt.Sprintf("modTimeUpdaterLoop(%s) started", pathStr)) - updateTicker := time.NewTicker(updateIntv) - defer updateTicker.Stop() - loop: - for { - select { - case <-ctx.Done(): - break loop - case curTime := <-updateTicker.C: - mapf, err := mfs.GetMapFile(pathStr) - if err != nil { - t.Errorf("modTimeUpdaterLoop(%s): GetMapFile() err=%s", pathStr, err) - } else { - mapf.ModTime = curTime - mfs.UpsertMapFile(pathStr, mapf) - } + mtm := NewModTimeMonitor(ctx, WithStatFS(mfs), WithCheckInterval(5*time.Millisecond)) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + pth := fmt.Sprintf("/concurrency/sub%d", i) + if err := mfs.UpsertMapFile(pth, &fstest.MapFile{ModTime: time.Now()}); err != nil { + t.Error(err) + return } - } - mtm.log(fmt.Sprintf("modTimeUpdaterLoop(%s) terminated", pathStr)) - } - - modTimeReaderLoop := func(ctx context.Context, t *testing.T, pathStr string, readIntv time.Duration) { - mtm.log(fmt.Sprintf("modTimeReaderLoop(%s) started", pathStr)) - readTicker := time.NewTicker(readIntv) - defer readTicker.Stop() - loop: - for { - select { - case <-ctx.Done(): - break loop - case <-readTicker.C: - sts, err := mtm.GetPathStatus(fsnotifyStrategy, pathStr) - if err != nil { - t.Errorf("modTimeReaderLoop(%s): GetPathStatus() err=%s", pathStr, err) - } else { - t.Logf("sts=%+v", sts) + if err := mtm.AddMonitoredPath(testStrategy, pth); err != nil { + t.Error(err) + return + } + for j := 0; j < 50; j++ { + if _, err := mtm.GetPathStatus(testStrategy, pth); err != nil { + t.Error(err) + return } } - } - mtm.log(fmt.Sprintf("modTimeReaderLoop(%s) terminated", pathStr)) - } - - // Add paths to monitor - for _, pathRec := range modTimePaths { - mtm.AddMonitoredPath(fsnotifyStrategy, pathRec.pathStr) - go modTimeUpdaterLoop(ctx, t, pathRec.pathStr, commonIntv) - go modTimeReaderLoop(ctx, t, pathRec.pathStr, commonIntv) - } - - // Give time for background threads to do their thing - time.Sleep(time.Millisecond * 3000) - - // Cancel ctx and give time for background threads to terminate - cancelCtxFn() - time.Sleep(time.Millisecond * 1000) -} - -// TestMaxTimeSubtractionDuration verifies that time.subtraction -// difference that is too large to be represented by time.Duration -// results in the max int64 value (approx 290 years), and does not fail. -func TestMaxTimeSubtractionDuration(t *testing.T) { - var zeroTime time.Time - modTime := MustParseRFC3339("2025-04-01T23:07:59Z") - hugeDuration := modTime.Sub(zeroTime) - t.Logf("zeroTime=%s", zeroTime) - t.Logf("modTime=%s", modTime) - t.Logf("hugeDuration=modTime.Sub(zeroTime)=0x%x=%d=%s", - int64(hugeDuration), hugeDuration, hugeDuration) - if int64(hugeDuration) != math.MaxInt64 { - t.Errorf("unexpected hugeDuration=%x, should be %x", int64(hugeDuration), math.MaxInt64) + }(i) } + wg.Wait() } - -func TestTimeComparisons(t *testing.T) { - modTime := MustParseRFC3339("2025-04-01T23:07:59Z") - isEqual := modTime.Equal(modTime) - isAfter := modTime.After(modTime) - isBefore := modTime.Before(modTime) - t.Logf("modTime=%s", modTime) - t.Logf("modTime.Equal(modTime)=%v, modTime.After(modTime)=%v, modTime.Before(modTime)=%v", - isEqual, isAfter, isBefore) - if !isEqual { - t.Errorf("modTime.Equal(modTime)=false, should be true") - } - if isAfter { - t.Errorf("modTime.After(modTime)=true, should be false") - } - if isBefore { - t.Errorf("modTime.Before(modTime)=true, should be false") - } -} - -// MustParseRFC3339 calls time.Parse(time.RFC3339,...), -// and panics on error, otherwise returns parsed time.Time result -func MustParseRFC3339(str string) time.Time { - t, err := time.Parse(time.RFC3339, str) - if err != nil { - panic(fmt.Sprintf("time.Parse(%s) err=%s", str, err)) - } - return t -} - -var expectMetricsRegexpInitial = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 0 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 2 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 2 -` - -var expectMetricsRegexpAfterModtimeUpdated = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 1 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 3 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 3 -` - -var expectMetricsRegexpAfterModtimeNotUpdated = ` -# HELP hotload_modtime_latency_histogram Hotload modtime latency histogram \(seconds\) by strategy and path -# TYPE hotload_modtime_latency_histogram histogram -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="900"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="1800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="2700"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="3600"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="4500"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="5400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="7200"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="10800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="14400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="28800"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="86400"} 2 -hotload_modtime_latency_histogram_bucket{path="/foo/bar",strategy="fsnotify",le="\+Inf"} 4 -hotload_modtime_latency_histogram_sum{path="/foo/bar",strategy="fsnotify"} 1.8446744\d*e\+10 -hotload_modtime_latency_histogram_count{path="/foo/bar",strategy="fsnotify"} 4 -` diff --git a/observability/go.mod b/observability/go.mod new file mode 100644 index 0000000..669a7c5 --- /dev/null +++ b/observability/go.mod @@ -0,0 +1,22 @@ +module github.com/infobloxopen/hotload/observability + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + github.com/prometheus/client_golang v1.20.0 + github.com/prometheus/common v0.55.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) + +replace github.com/infobloxopen/hotload/v3 => ../ diff --git a/observability/go.sum b/observability/go.sum new file mode 100644 index 0000000..2923745 --- /dev/null +++ b/observability/go.sum @@ -0,0 +1,24 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/observability/pathchksum.go b/observability/pathchksum.go new file mode 100644 index 0000000..e4a5f2c --- /dev/null +++ b/observability/pathchksum.go @@ -0,0 +1,121 @@ +package observability + +import ( + "hash/crc64" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/infobloxopen/hotload/v3/logger" +) + +// PathChksumMetricsEnableEnvVar gates checksum collection: hashing every +// watched file on each scrape is not free, so it is opt-in, exactly as in +// hotload v1. +const PathChksumMetricsEnableEnvVar = "HOTLOAD_PATH_CHKSUM_METRICS_ENABLE" + +// HotloadPathChksumTimestampSecondsName is the metric name, identical to +// hotload v1. +const HotloadPathChksumTimestampSecondsName = "hotload_path_chksum_timestamp_seconds" + +// FileHasher hashes a file's contents; replaceable for unit tests. +type FileHasher func(filePath string) (uint64, error) + +var crc64Table = crc64.MakeTable(crc64.ECMA) + +// DefaultFileHasher hashes file contents using CRC64. +func DefaultFileHasher(filePath string) (uint64, error) { + pathBytes, err := os.ReadFile(filePath) + if err != nil { + return 0, err + } + return crc64.Checksum(pathBytes, crc64Table), nil +} + +// PathChksumCollector is a prometheus.Collector reporting, per watched +// path, the unix timestamp at which the file's content checksum last +// changed. The checksum is computed at scrape time, so the metric stays +// accurate without a background poller. It replaces hotload v1's +// gaugefuncvec-based implementation. +type PathChksumCollector struct { + desc *prometheus.Desc + hasher FileHasher + enabled bool + + mu sync.Mutex + paths map[string]*chksumRecord +} + +type chksumRecord struct { + crc64 uint64 + lastChanged int64 +} + +// NewPathChksumCollector creates a collector using hasher (DefaultFileHasher +// when nil). Collection is enabled by PathChksumMetricsEnableEnvVar. +func NewPathChksumCollector(hasher FileHasher) *PathChksumCollector { + if hasher == nil { + hasher = DefaultFileHasher + } + enabled := false + switch strings.ToLower(strings.TrimSpace(os.Getenv(PathChksumMetricsEnableEnvVar))) { + case "1", "true", "yes": + enabled = true + } + return &PathChksumCollector{ + desc: prometheus.NewDesc( + HotloadPathChksumTimestampSecondsName, + "Hotload path checksum last changed (unix timestamp), by path", + []string{PathKey}, nil, + ), + hasher: hasher, + enabled: enabled, + paths: make(map[string]*chksumRecord), + } +} + +// AddPath starts reporting the checksum timestamp of pathStr, which must be +// a local file path. Duplicate adds are ignored. Fed automatically for +// fsnotify watches when the collector is wired through Collectors.Hooks; +// custom file-backed strategies should call it directly. +func (p *PathChksumCollector) AddPath(pathStr string) { + if !p.enabled { + return + } + pathStr = cleanPath(pathStr) + p.mu.Lock() + defer p.mu.Unlock() + if _, found := p.paths[pathStr]; !found { + p.paths[pathStr] = &chksumRecord{} + } +} + +// Describe implements prometheus.Collector. +func (p *PathChksumCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- p.desc +} + +// Collect implements prometheus.Collector. Each watched file is hashed; a +// changed checksum bumps the path's last-changed timestamp. +func (p *PathChksumCollector) Collect(ch chan<- prometheus.Metric) { + p.mu.Lock() + defer p.mu.Unlock() + for pathStr, rec := range p.paths { + newCrc, err := p.hasher(pathStr) + if err != nil { + logger.ErrLogf("PathChksumCollector", "hashing %s failed: %v", pathStr, err) + } else if rec.crc64 != newCrc { + rec.crc64 = newCrc + rec.lastChanged = time.Now().Unix() + } + ch <- prometheus.MustNewConstMetric(p.desc, prometheus.GaugeValue, float64(rec.lastChanged), pathStr) + } +} + +func cleanPath(pathStr string) string { + return path.Clean(strings.TrimSpace(pathStr)) +} diff --git a/observability/pathchksum_test.go b/observability/pathchksum_test.go new file mode 100644 index 0000000..411b736 --- /dev/null +++ b/observability/pathchksum_test.go @@ -0,0 +1,81 @@ +package observability + +import ( + "fmt" + "strings" + "sync/atomic" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/infobloxopen/hotload/observability/promtest" +) + +func TestPathChksumCollector(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "true") + + var hash atomic.Uint64 + hash.Store(1) + c := NewPathChksumCollector(func(filePath string) (uint64, error) { + return hash.Load(), nil + }) + + c.AddPath("/etc/dsn") + c.AddPath("/etc/dsn") // duplicate adds are ignored + + expect := ` +# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path +# TYPE hotload_path_chksum_timestamp_seconds gauge +hotload_path_chksum_timestamp_seconds\{path="/etc/dsn"\} 1\.\d+e\+09 +` + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(expect), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("first scrape diff:\n%s", err) + } + + // An unchanged checksum keeps the timestamp; a changed one bumps it. + first := collectValue(t, c) + if v := collectValue(t, c); v != first { + t.Errorf("timestamp changed without content change: %v -> %v", first, v) + } + hash.Store(2) + if v := collectValue(t, c); v < first { + t.Errorf("timestamp went backwards after content change: %v -> %v", first, v) + } +} + +func TestPathChksumCollectorDisabled(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "") + + c := NewPathChksumCollector(func(filePath string) (uint64, error) { return 1, nil }) + c.AddPath("/etc/dsn") + + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(""), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("disabled collector should produce no series, got diff:\n%s", err) + } +} + +func TestPathChksumCollectorHashError(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "yes") + + c := NewPathChksumCollector(func(filePath string) (uint64, error) { + return 0, fmt.Errorf("boom") + }) + c.AddPath("/etc/dsn") + + // Hash errors keep the last value (zero here) rather than dropping the + // series or panicking. + expect := ` +# HELP hotload_path_chksum_timestamp_seconds Hotload path checksum last changed \(unix timestamp\), by path +# TYPE hotload_path_chksum_timestamp_seconds gauge +hotload_path_chksum_timestamp_seconds\{path="/etc/dsn"\} 0 +` + if err := promtest.CollectAndRegexpCompare(c, strings.NewReader(expect), HotloadPathChksumTimestampSecondsName); err != nil { + t.Errorf("unexpected diff:\n%s", err) + } +} + +// collectValue scrapes the single-series collector and returns its value. +func collectValue(t *testing.T, c *PathChksumCollector) float64 { + t.Helper() + return testutil.ToFloat64(c) +} diff --git a/observability/prometheus.go b/observability/prometheus.go new file mode 100644 index 0000000..8a386e7 --- /dev/null +++ b/observability/prometheus.go @@ -0,0 +1,192 @@ +// Package observability exports hotload activity as prometheus metrics. +// +// Hotload v3's core has no metrics dependency; it emits events through +// hotload.Hooks. This module adapts those events to prometheus collectors +// that preserve the metric names of hotload v1, so existing dashboards keep +// working. Unlike v1, registration is explicit: +// +// import "github.com/infobloxopen/hotload/observability" +// +// func main() { +// observability.MustEnablePrometheus(nil) // nil = prometheus.DefaultRegisterer +// ... +// } +// +// With a nil registerer the call is idempotent, so a library and its caller +// can both enable metrics without coordinating. +package observability + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// Label keys and values, identical to hotload v1's metrics package. +const ( + GRPCMethodKey = "grpc_method" + GRPCServiceKey = "grpc_service" + StatementKey = "stmt" // either exec or query + ExecStatement = "exec" + QueryStatement = "query" + + StrategyKey = "strategy" + PathKey = "path" + UrlKey = "url" +) + +// Metric names, identical to hotload v1. +const ( + SqlStmtsSummaryName = "transaction_sql_stmts" + HotloadChangeTotalName = "hotload_change_total" + HotloadLastChangedTimestampSecondsName = "hotload_last_changed_timestamp_seconds" + HotloadModtimeLatencyHistogramName = "hotload_modtime_latency_histogram" +) + +// HotloadModtimeLatencyHistogramDefBuckets are the default buckets (seconds) +// of the modtime latency histogram, identical to hotload v1. +var HotloadModtimeLatencyHistogramDefBuckets = []float64{900, 1800, 2700, 3600, 4500, 5400, 7200, 10800, 14400, 28800, 86400} + +// Collectors bundles the prometheus collectors fed by hotload hooks. +type Collectors struct { + // SqlStmtsSummary tracks the number of sql statements per transaction + // by statement type and the grpc service/method labels carried by the + // transaction context (see hotload.ContextWithExecLabels). + SqlStmtsSummary *prometheus.SummaryVec + // HotloadChangeTotal counts config changes per hotload DSN. + HotloadChangeTotal *prometheus.CounterVec + // HotloadLastChangedTimestampSeconds is the unix timestamp of the last + // config change per hotload DSN. + HotloadLastChangedTimestampSeconds *prometheus.GaugeVec + // HotloadModtimeLatencyHistogram tracks how stale watched files are, + // fed by the modtime monitor. + HotloadModtimeLatencyHistogram *prometheus.HistogramVec + // HotloadPathChksumTimestampSeconds reports when each watched file's + // content checksum last changed, computed at scrape time. Gated by the + // HOTLOAD_PATH_CHKSUM_METRICS_ENABLE environment variable. + HotloadPathChksumTimestampSeconds *PathChksumCollector +} + +// NewCollectors creates unregistered collectors with hotload v1's metric +// names. +func NewCollectors() *Collectors { + return &Collectors{ + SqlStmtsSummary: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Name: SqlStmtsSummaryName, + Help: "The number of sql stmts called in a transaction by statement type per grpc service and method", + }, []string{GRPCServiceKey, GRPCMethodKey, StatementKey}), + HotloadChangeTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: HotloadChangeTotalName, + Help: "Hotload change total by url", + }, []string{UrlKey}), + HotloadLastChangedTimestampSeconds: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: HotloadLastChangedTimestampSecondsName, + Help: "Hotload last changed (unix timestamp), by url", + }, []string{UrlKey}), + HotloadModtimeLatencyHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: HotloadModtimeLatencyHistogramName, + Help: "Hotload modtime latency histogram (seconds) by strategy and path", + Buckets: HotloadModtimeLatencyHistogramDefBuckets, + }, []string{StrategyKey, PathKey}), + HotloadPathChksumTimestampSeconds: NewPathChksumCollector(DefaultFileHasher), + } +} + +// All returns every collector, for manual registration. +func (c *Collectors) All() []prometheus.Collector { + return []prometheus.Collector{ + c.SqlStmtsSummary, + c.HotloadChangeTotal, + c.HotloadLastChangedTimestampSeconds, + c.HotloadModtimeLatencyHistogram, + c.HotloadPathChksumTimestampSeconds, + } +} + +// Hooks returns the hotload hooks that feed the collectors. Register them +// with hotload.RegisterHooks (EnablePrometheus does this for you). +func (c *Collectors) Hooks() hotload.Hooks { + return hotload.Hooks{ + OnConfigChange: func(ev hotload.ConfigChangeEvent) { + c.HotloadChangeTotal.WithLabelValues(ev.GroupName).Inc() + c.HotloadLastChangedTimestampSeconds.WithLabelValues(ev.GroupName).Set(float64(ev.At.Unix())) + }, + OnTxComplete: func(ev hotload.TxEvent) { + labels := hotload.GetExecLabelsFromContext(ev.Ctx) + service := labels[GRPCServiceKey] + method := labels[GRPCMethodKey] + c.SqlStmtsSummary.WithLabelValues(service, method, ExecStatement).Observe(float64(ev.ExecStmts)) + c.SqlStmtsSummary.WithLabelValues(service, method, QueryStatement).Observe(float64(ev.QueryStmts)) + }, + OnModTimeCheck: func(ev hotload.ModTimeEvent) { + c.HotloadModtimeLatencyHistogram.WithLabelValues(ev.Strategy, ev.Path).Observe(ev.Latency.Seconds()) + }, + OnWatch: func(ev hotload.WatchEvent) { + // Only fsnotify watches local files; other strategies' paths + // (Kubernetes Secret names, etcd keys, ...) are not hashable. + // File-backed strategies outside this repo can call + // Collectors.HotloadPathChksumTimestampSeconds.AddPath directly. + if !ev.Closed && ev.Strategy == "fsnotify" { + c.HotloadPathChksumTimestampSeconds.AddPath(ev.Path) + } + }, + } +} + +// defaultMu guards the idempotent default-registerer path of +// EnablePrometheus; defaultCollectors holds its result. +var ( + defaultMu sync.Mutex + defaultCollectors *Collectors +) + +// EnablePrometheus creates the collectors, registers them with reg, and +// registers the hooks that feed them with hotload. Call it during program +// initialization, before opening hotload connections. +// +// When reg is nil (or prometheus.DefaultRegisterer) the call is idempotent: +// the first call registers collectors and hooks with the default registerer +// and later calls return that same *Collectors — so an application and a +// library it uses can both enable hotload metrics defensively without +// tripping duplicate-registration errors. Calls with any other registerer +// create and register fresh collectors every time; managing their lifetime +// is the caller's job (this is the path tests use with throwaway +// registries). +func EnablePrometheus(reg prometheus.Registerer) (*Collectors, error) { + if reg != nil && reg != prometheus.DefaultRegisterer { + return enablePrometheus(reg) + } + defaultMu.Lock() + defer defaultMu.Unlock() + if defaultCollectors == nil { + c, err := enablePrometheus(prometheus.DefaultRegisterer) + if err != nil { + return nil, err + } + defaultCollectors = c + } + return defaultCollectors, nil +} + +func enablePrometheus(reg prometheus.Registerer) (*Collectors, error) { + c := NewCollectors() + for _, collector := range c.All() { + if err := reg.Register(collector); err != nil { + return nil, err + } + } + hotload.RegisterHooks(c.Hooks()) + return c, nil +} + +// MustEnablePrometheus is EnablePrometheus, panicking on registration +// errors. +func MustEnablePrometheus(reg prometheus.Registerer) *Collectors { + c, err := EnablePrometheus(reg) + if err != nil { + panic(err) + } + return c +} diff --git a/observability/prometheus_test.go b/observability/prometheus_test.go new file mode 100644 index 0000000..c40ed34 --- /dev/null +++ b/observability/prometheus_test.go @@ -0,0 +1,136 @@ +package observability + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/infobloxopen/hotload/observability/promtest" + hotload "github.com/infobloxopen/hotload/v3" +) + +func TestConfigChangeMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + at := time.Unix(1764000000, 0) + ev := hotload.ConfigChangeEvent{GroupName: "fsnotify://postgres/etc/dsn", At: at} + h.OnConfigChange(ev) + h.OnConfigChange(hotload.ConfigChangeEvent{GroupName: "fsnotify://postgres/etc/dsn", At: at.Add(time.Minute)}) + + if got := testutil.ToFloat64(c.HotloadChangeTotal.WithLabelValues(ev.GroupName)); got != 2 { + t.Errorf("hotload_change_total = %v, want 2", got) + } + want := float64(at.Add(time.Minute).Unix()) + if got := testutil.ToFloat64(c.HotloadLastChangedTimestampSeconds.WithLabelValues(ev.GroupName)); got != want { + t.Errorf("hotload_last_changed_timestamp_seconds = %v, want %v", got, want) + } +} + +func TestTxMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + ctx := hotload.ContextWithExecLabels(context.Background(), map[string]string{ + GRPCServiceKey: "svc", + GRPCMethodKey: "m", + }) + h.OnTxComplete(hotload.TxEvent{Ctx: ctx, ExecStmts: 3, QueryStmts: 1, Committed: true}) + + expect := ` +# HELP transaction_sql_stmts The number of sql stmts called in a transaction by statement type per grpc service and method +# TYPE transaction_sql_stmts summary +transaction_sql_stmts_sum\{grpc_method="m",grpc_service="svc",stmt="exec"\} 3 +transaction_sql_stmts_count\{grpc_method="m",grpc_service="svc",stmt="exec"\} 1 +transaction_sql_stmts_sum\{grpc_method="m",grpc_service="svc",stmt="query"\} 1 +transaction_sql_stmts_count\{grpc_method="m",grpc_service="svc",stmt="query"\} 1 +` + err := promtest.CollectAndRegexpCompare(c.SqlStmtsSummary, strings.NewReader(expect), SqlStmtsSummaryName) + if err != nil { + t.Errorf("unexpected metrics diff:\n%s", err) + } +} + +func TestModTimeMetrics(t *testing.T) { + c := NewCollectors() + h := c.Hooks() + + h.OnModTimeCheck(hotload.ModTimeEvent{Strategy: "fsnotify", Path: "/etc/dsn", Latency: 1000 * time.Second}) + + if got := testutil.CollectAndCount(c.HotloadModtimeLatencyHistogram, HotloadModtimeLatencyHistogramName); got != 1 { + t.Errorf("histogram series count = %d, want 1", got) + } +} + +func TestEnablePrometheus(t *testing.T) { + reg := prometheus.NewRegistry() + c, err := EnablePrometheus(reg) + if err != nil { + t.Fatal(err) + } + + c.Hooks().OnConfigChange(hotload.ConfigChangeEvent{GroupName: "g", At: time.Now()}) + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + found := false + for _, mf := range families { + if mf.GetName() == HotloadChangeTotalName { + found = true + } + } + if !found { + t.Errorf("%s not gatherable from registry", HotloadChangeTotalName) + } + + // Double registration must error, not panic. + if _, err := EnablePrometheus(reg); err == nil { + t.Error("second EnablePrometheus on same registry should error") + } +} + +// TestPathChksumOnlyFsnotify: watch events register chksum paths only for +// the fsnotify strategy — other strategies' paths (Secret names, etcd keys) +// are not local files and must not be hashed at scrape time. +func TestPathChksumOnlyFsnotify(t *testing.T) { + t.Setenv(PathChksumMetricsEnableEnvVar, "true") + c := NewCollectors() + h := c.Hooks() + + h.OnWatch(hotload.WatchEvent{Strategy: "fsnotify", Path: "/etc/dsn"}) + h.OnWatch(hotload.WatchEvent{Strategy: "k8ssecret", Path: "/mydb"}) + h.OnWatch(hotload.WatchEvent{Strategy: "fsnotify", Path: "/etc/other", Closed: true}) + + col := c.HotloadPathChksumTimestampSeconds + col.mu.Lock() + defer col.mu.Unlock() + if _, ok := col.paths["/etc/dsn"]; !ok { + t.Error("fsnotify watch path not registered for chksum") + } + if _, ok := col.paths["/mydb"]; ok { + t.Error("k8ssecret path must not be registered for chksum") + } + if _, ok := col.paths["/etc/other"]; ok { + t.Error("closed watch event must not register a path") + } +} + +// TestEnablePrometheusDefaultIdempotent: hotload v1 enabled metrics as an +// import side effect, so migrated code may enable defensively in more than +// one place; with the default registerer the second call must return the +// same collectors instead of a duplicate-registration panic. +func TestEnablePrometheusDefaultIdempotent(t *testing.T) { + c1 := MustEnablePrometheus(nil) + c2 := MustEnablePrometheus(nil) + if c1 != c2 { + t.Error("second MustEnablePrometheus(nil) returned different collectors") + } + if c3 := MustEnablePrometheus(prometheus.DefaultRegisterer); c3 != c1 { + t.Error("MustEnablePrometheus(DefaultRegisterer) should take the idempotent default path") + } +} diff --git a/observability/promtest/promtest.go b/observability/promtest/promtest.go new file mode 100644 index 0000000..64de3c0 --- /dev/null +++ b/observability/promtest/promtest.go @@ -0,0 +1,79 @@ +// Package promtest provides helpers for asserting prometheus metric output +// against regexp patterns. Hotload v1 shipped these in its internal +// package; they live here so the hotload core stays free of prometheus +// dependencies. +package promtest + +import ( + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/expfmt" +) + +// CollectAndRegexpCompare is similar to testutil.CollectAndCompare() +// but the expected lines are regexp patterns. +// Note that unlike testutil.CollectAndCompare(), +// the metricName MUST be specified to get any collected result. +func CollectAndRegexpCompare(colltor prometheus.Collector, expectRdr io.Reader, metricNames ...string) error { + expectBytes, err := io.ReadAll(expectRdr) + if err != nil { + return err + } + + collectBytes, err := testutil.CollectAndFormat(colltor, expfmt.TypeTextPlain, metricNames...) + if err != nil { + return err + } + + expectStr := strings.TrimSpace(string(expectBytes)) + collectStr := strings.TrimSpace(string(collectBytes)) + + expectSplit := strings.Split(expectStr, "\n") + collectSplit := strings.Split(collectStr, "\n") + + diffStr := strings.TrimSpace(SimpleRegexpLineDiff(expectSplit, collectSplit)) + if len(diffStr) > 0 { + return errors.New(diffStr) + } + return nil +} + +// SimpleRegexpLineDiff performs a simple/dumb line-by-line diff +// between two arrays of lines. The expected array of lines are regexp +// patterns. Returns line(s) which diff. Empty string is returned if there +// are no diffs. +func SimpleRegexpLineDiff(regexpLines []string, gotLines []string) string { + maxLen := max(len(regexpLines), len(gotLines)) + + for len(regexpLines) < maxLen { + regexpLines = append(regexpLines, "") + } + for len(gotLines) < maxLen { + gotLines = append(gotLines, "") + } + + var diffBuf strings.Builder + for k := 0; k < maxLen; k++ { + expStr := strings.TrimSpace(regexpLines[k]) + gotStr := strings.TrimSpace(gotLines[k]) + expPat := `^` + expStr + `$` + + matched, err := regexp.MatchString(expPat, gotStr) + if err != nil { + return err.Error() + } + + if !matched { + fmt.Fprintf(&diffBuf, "-%s\n", expStr) + fmt.Fprintf(&diffBuf, "+%s\n", gotStr) + } + } + + return diffBuf.String() +} diff --git a/observer.go b/observer.go new file mode 100644 index 0000000..bf5fd5b --- /dev/null +++ b/observer.go @@ -0,0 +1,157 @@ +package hotload + +import ( + "context" + "sync" + "time" +) + +// ConfigChangeEvent is emitted when a strategy reports a new connection +// string for a watched hotload DSN. +type ConfigChangeEvent struct { + // GroupName is the full hotload DSN (e.g. "fsnotify://postgres/etc/dsn"). + GroupName string + // OldRedactedDSN and NewRedactedDSN are the previous and new underlying + // connection strings with credentials redacted. + OldRedactedDSN string + NewRedactedDSN string + // ForceKill reports whether the group closes old connections immediately. + ForceKill bool + At time.Time +} + +// ConnEvent is emitted when hotload opens or closes an underlying connection. +type ConnEvent struct { + GroupName string + RedactedDSN string + // Killed is true on close events caused by a config change (rather than + // the pool retiring the connection). + Killed bool +} + +// TxEvent is emitted when a transaction completes. ExecStmts and QueryStmts +// are the number of exec and query statements observed on the connection +// since the previous transaction completed. +type TxEvent struct { + // Ctx is the context the transaction was started with; adapters can + // extract labels from it with GetExecLabelsFromContext. + Ctx context.Context + ExecStmts int64 + QueryStmts int64 + Committed bool +} + +// WatchEvent is emitted when a strategy watch is established or closed. +type WatchEvent struct { + GroupName string + Strategy string + Path string + Closed bool +} + +// ModTimeEvent is emitted by the modtime monitor when it samples the +// modification time of a watched path. +type ModTimeEvent struct { + Strategy string + Path string + // Latency is the time elapsed since the file was last modified. + Latency time.Duration +} + +// Hooks receives notifications about hotload activity. All fields are +// optional; nil fields are skipped. Hooks must be fast and must not call +// back into hotload. Adapters (e.g. the observability module) use Hooks to +// export metrics without hotload depending on any metrics library. +type Hooks struct { + OnConfigChange func(ConfigChangeEvent) + OnConnOpen func(ConnEvent) + OnConnClose func(ConnEvent) + OnTxComplete func(TxEvent) + OnWatch func(WatchEvent) + OnModTimeCheck func(ModTimeEvent) +} + +var hooksMu sync.RWMutex +var hooks []Hooks + +// RegisterHooks adds h to the set of registered hooks. Hooks cannot be +// unregistered; register them once during program initialization, before +// opening connections. +func RegisterHooks(h Hooks) { + hooksMu.Lock() + defer hooksMu.Unlock() + hooks = append(hooks, h) +} + +// resetHooks removes all registered hooks. For tests. +func resetHooks() { + hooksMu.Lock() + defer hooksMu.Unlock() + hooks = nil +} + +// hooksRegistered reports whether any hooks have been registered. +func hooksRegistered() bool { + hooksMu.RLock() + defer hooksMu.RUnlock() + return len(hooks) != 0 +} + +func snapshotHooks() []Hooks { + hooksMu.RLock() + defer hooksMu.RUnlock() + return hooks +} + +func emitConfigChange(ev ConfigChangeEvent) { + for _, h := range snapshotHooks() { + if h.OnConfigChange != nil { + h.OnConfigChange(ev) + } + } +} + +func emitConnOpen(ev ConnEvent) { + for _, h := range snapshotHooks() { + if h.OnConnOpen != nil { + h.OnConnOpen(ev) + } + } +} + +func emitConnClose(ev ConnEvent) { + for _, h := range snapshotHooks() { + if h.OnConnClose != nil { + h.OnConnClose(ev) + } + } +} + +func emitTxComplete(ev TxEvent) { + for _, h := range snapshotHooks() { + if h.OnTxComplete != nil { + h.OnTxComplete(ev) + } + } +} + +// EmitWatchEvent notifies registered hooks that a strategy watch was +// established or closed. It is exported for strategy implementations +// (e.g. the fsnotify subpackage). +func EmitWatchEvent(ev WatchEvent) { + for _, h := range snapshotHooks() { + if h.OnWatch != nil { + h.OnWatch(ev) + } + } +} + +// EmitModTimeEvent notifies registered hooks of a modtime sample. It is +// exported for the modtime subpackage. +func EmitModTimeEvent(ev ModTimeEvent) { + for _, h := range snapshotHooks() { + if h.OnModTimeCheck != nil { + h.OnModTimeCheck(ev) + } + } +} diff --git a/stmt.go b/stmt.go new file mode 100644 index 0000000..2393e37 --- /dev/null +++ b/stmt.go @@ -0,0 +1,69 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// baseStmt wraps an underlying driver.Stmt prepared on a baseConn. It +// implements the mandatory driver.Stmt methods; the optional interfaces +// (StmtExecContext, StmtQueryContext, ColumnConverter, NamedValueChecker) +// are exposed only when the underlying stmt supports them, via the generated +// combination wrappers returned by wrapStmt. +type baseStmt struct { + inner driver.Stmt + conn *baseConn +} + +func (s *baseStmt) Close() error { + return s.inner.Close() +} + +func (s *baseStmt) NumInput() int { + return s.inner.NumInput() +} + +func (s *baseStmt) Exec(args []driver.Value) (driver.Result, error) { + s.conn.execStmts.Add(1) + return s.inner.Exec(args) +} + +func (s *baseStmt) Query(args []driver.Value) (driver.Rows, error) { + s.conn.queryStmts.Add(1) + return s.inner.Query(args) +} + +// execContext backs the generated StmtExecContext wrappers; only reachable +// when the underlying stmt implements driver.StmtExecContext. +func (s *baseStmt) execContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + octx, release := s.conn.opCtx(ctx) + if octx == nil { + return nil, driver.ErrBadConn + } + defer release() + s.conn.execStmts.Add(1) + return s.inner.(driver.StmtExecContext).ExecContext(octx, args) +} + +// queryContext backs the generated StmtQueryContext wrappers. As with +// baseConn.queryContext, the caller's context passes through unmerged +// because the returned driver.Rows captures it. +func (s *baseStmt) queryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if s.conn.gen.ctx.Err() != nil { + return nil, driver.ErrBadConn + } + s.conn.queryStmts.Add(1) + return s.inner.(driver.StmtQueryContext).QueryContext(ctx, args) +} + +// columnConverter backs the generated ColumnConverter wrappers; only +// reachable when the underlying stmt implements driver.ColumnConverter. +func (s *baseStmt) columnConverter(idx int) driver.ValueConverter { + return s.inner.(driver.ColumnConverter).ColumnConverter(idx) +} + +// checkNamedValue backs the generated NamedValueChecker wrappers; only +// reachable when the underlying stmt implements driver.NamedValueChecker. +func (s *baseStmt) checkNamedValue(nv *driver.NamedValue) error { + return s.inner.(driver.NamedValueChecker).CheckNamedValue(nv) +} diff --git a/stmt_combos_gen.go b/stmt_combos_gen.go new file mode 100644 index 0000000..9edec85 --- /dev/null +++ b/stmt_combos_gen.go @@ -0,0 +1,244 @@ +// Code generated by internal/gen. DO NOT EDIT. + +package hotload + +import "database/sql/driver" + +// stmtFlags reports which optional interfaces the wrapped stmt must expose. +func stmtFlags(s driver.Stmt) uint8 { + var f uint8 + if _, ok := s.(driver.StmtExecContext); ok { + f |= 1 << 0 + } + if _, ok := s.(driver.StmtQueryContext); ok { + f |= 1 << 1 + } + if _, ok := s.(driver.ColumnConverter); ok { //nolint:staticcheck // legacy interface intentionally supported + f |= 1 << 2 + } + if _, ok := s.(driver.NamedValueChecker); ok { + f |= 1 << 3 + } + return f +} + +type stmt_E struct { + *baseStmt + sExecer +} + +var ( + _ driver.Stmt = stmt_E{} + _ driver.StmtExecContext = stmt_E{} +) + +type stmt_Q struct { + *baseStmt + sQueryer +} + +var ( + _ driver.Stmt = stmt_Q{} + _ driver.StmtQueryContext = stmt_Q{} +) + +type stmt_EQ struct { + *baseStmt + sExecer + sQueryer +} + +var ( + _ driver.Stmt = stmt_EQ{} + _ driver.StmtExecContext = stmt_EQ{} + _ driver.StmtQueryContext = stmt_EQ{} +) + +type stmt_C struct { + *baseStmt + sColConv +} + +var ( + _ driver.Stmt = stmt_C{} + _ driver.ColumnConverter = stmt_C{} +) + +type stmt_EC struct { + *baseStmt + sExecer + sColConv +} + +var ( + _ driver.Stmt = stmt_EC{} + _ driver.StmtExecContext = stmt_EC{} + _ driver.ColumnConverter = stmt_EC{} +) + +type stmt_QC struct { + *baseStmt + sQueryer + sColConv +} + +var ( + _ driver.Stmt = stmt_QC{} + _ driver.StmtQueryContext = stmt_QC{} + _ driver.ColumnConverter = stmt_QC{} +) + +type stmt_EQC struct { + *baseStmt + sExecer + sQueryer + sColConv +} + +var ( + _ driver.Stmt = stmt_EQC{} + _ driver.StmtExecContext = stmt_EQC{} + _ driver.StmtQueryContext = stmt_EQC{} + _ driver.ColumnConverter = stmt_EQC{} +) + +type stmt_N struct { + *baseStmt + sNVChecker +} + +var ( + _ driver.Stmt = stmt_N{} + _ driver.NamedValueChecker = stmt_N{} +) + +type stmt_EN struct { + *baseStmt + sExecer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EN{} + _ driver.StmtExecContext = stmt_EN{} + _ driver.NamedValueChecker = stmt_EN{} +) + +type stmt_QN struct { + *baseStmt + sQueryer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_QN{} + _ driver.StmtQueryContext = stmt_QN{} + _ driver.NamedValueChecker = stmt_QN{} +) + +type stmt_EQN struct { + *baseStmt + sExecer + sQueryer + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EQN{} + _ driver.StmtExecContext = stmt_EQN{} + _ driver.StmtQueryContext = stmt_EQN{} + _ driver.NamedValueChecker = stmt_EQN{} +) + +type stmt_CN struct { + *baseStmt + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_CN{} + _ driver.ColumnConverter = stmt_CN{} + _ driver.NamedValueChecker = stmt_CN{} +) + +type stmt_ECN struct { + *baseStmt + sExecer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_ECN{} + _ driver.StmtExecContext = stmt_ECN{} + _ driver.ColumnConverter = stmt_ECN{} + _ driver.NamedValueChecker = stmt_ECN{} +) + +type stmt_QCN struct { + *baseStmt + sQueryer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_QCN{} + _ driver.StmtQueryContext = stmt_QCN{} + _ driver.ColumnConverter = stmt_QCN{} + _ driver.NamedValueChecker = stmt_QCN{} +) + +type stmt_EQCN struct { + *baseStmt + sExecer + sQueryer + sColConv + sNVChecker +} + +var ( + _ driver.Stmt = stmt_EQCN{} + _ driver.StmtExecContext = stmt_EQCN{} + _ driver.StmtQueryContext = stmt_EQCN{} + _ driver.ColumnConverter = stmt_EQCN{} + _ driver.NamedValueChecker = stmt_EQCN{} +) + +func wrapStmt(b *baseStmt) driver.Stmt { + switch stmtFlags(b.inner) { + case 1: + return stmt_E{b, sExecer{b}} + case 2: + return stmt_Q{b, sQueryer{b}} + case 3: + return stmt_EQ{b, sExecer{b}, sQueryer{b}} + case 4: + return stmt_C{b, sColConv{b}} + case 5: + return stmt_EC{b, sExecer{b}, sColConv{b}} + case 6: + return stmt_QC{b, sQueryer{b}, sColConv{b}} + case 7: + return stmt_EQC{b, sExecer{b}, sQueryer{b}, sColConv{b}} + case 8: + return stmt_N{b, sNVChecker{b}} + case 9: + return stmt_EN{b, sExecer{b}, sNVChecker{b}} + case 10: + return stmt_QN{b, sQueryer{b}, sNVChecker{b}} + case 11: + return stmt_EQN{b, sExecer{b}, sQueryer{b}, sNVChecker{b}} + case 12: + return stmt_CN{b, sColConv{b}, sNVChecker{b}} + case 13: + return stmt_ECN{b, sExecer{b}, sColConv{b}, sNVChecker{b}} + case 14: + return stmt_QCN{b, sQueryer{b}, sColConv{b}, sNVChecker{b}} + case 15: + return stmt_EQCN{b, sExecer{b}, sQueryer{b}, sColConv{b}, sNVChecker{b}} + default: + return b + } +} diff --git a/stmt_pieces.go b/stmt_pieces.go new file mode 100644 index 0000000..617b003 --- /dev/null +++ b/stmt_pieces.go @@ -0,0 +1,37 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// The piece types below each carry exactly one optional driver.Stmt method, +// mirroring the conn pieces in conn_pieces.go. See that file for the design. + +// sExecer carries ExecContext (driver.StmtExecContext). +type sExecer struct{ b *baseStmt } + +func (p sExecer) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return p.b.execContext(ctx, args) +} + +// sQueryer carries QueryContext (driver.StmtQueryContext). +type sQueryer struct{ b *baseStmt } + +func (p sQueryer) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return p.b.queryContext(ctx, args) +} + +// sColConv carries ColumnConverter (driver.ColumnConverter). +type sColConv struct{ b *baseStmt } + +func (p sColConv) ColumnConverter(idx int) driver.ValueConverter { + return p.b.columnConverter(idx) +} + +// sNVChecker carries CheckNamedValue (driver.NamedValueChecker). +type sNVChecker struct{ b *baseStmt } + +func (p sNVChecker) CheckNamedValue(nv *driver.NamedValue) error { + return p.b.checkNamedValue(nv) +} diff --git a/strategy_fake_test.go b/strategy_fake_test.go new file mode 100644 index 0000000..83c4fba --- /dev/null +++ b/strategy_fake_test.go @@ -0,0 +1,109 @@ +package hotload_test + +import ( + "context" + "fmt" + "sync" + + hotload "github.com/infobloxopen/hotload/v3" +) + +// fakeStrategy is a hotload.Strategy driven by tests. It lives in the test +// package rather than in internal/dbfake because the Strategy interface +// names the hotload.Watchable type, and dbfake must not import hotload (the +// internal tests import dbfake). +type fakeStrategy struct { + mu sync.Mutex + initial map[string]string + subs map[string][]*fakeWatch + watches int +} + +// newFakeStrategy returns a fakeStrategy that answers Watch with the given +// initial values keyed by path. +func newFakeStrategy(initial map[string]string) *fakeStrategy { + cp := make(map[string]string, len(initial)) + for k, v := range initial { + cp[k] = v + } + return &fakeStrategy{initial: cp, subs: make(map[string][]*fakeWatch)} +} + +// Watch implements hotload.Strategy; every call opens an independent watch. +func (s *fakeStrategy) Watch(ctx context.Context, pth string, pathQry string) (string, hotload.Watchable, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.initial[pth] + if !ok { + return "", nil, fmt.Errorf("fakeStrategy: no initial value for path %q", pth) + } + w := &fakeWatch{strat: s, path: pth, ch: make(chan string)} + s.subs[pth] = append(s.subs[pth], w) + s.watches++ + context.AfterFunc(ctx, func() { w.Close() }) + return value, w, nil +} + +// Push delivers a new value to every watcher of path. It blocks until each +// hotload run loop receives it, which makes change injection deterministic. +func (s *fakeStrategy) Push(path, value string) { + s.mu.Lock() + watches := append([]*fakeWatch(nil), s.subs[path]...) + s.mu.Unlock() + if len(watches) == 0 { + panic("fakeStrategy: Push on unwatched path " + path) + } + for _, w := range watches { + w.ch <- value + } +} + +// CloseChan closes the update channels of every watch on path, simulating a +// strategy that stops watching on its own. +func (s *fakeStrategy) CloseChan(path string) { + s.mu.Lock() + defer s.mu.Unlock() + for _, w := range s.subs[path] { + w.closed = true + close(w.ch) + s.watches-- + } + delete(s.subs, path) +} + +// Watches reports how many watches are currently open. +func (s *fakeStrategy) Watches() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.watches +} + +// fakeWatch is the hotload.Watchable handed out by fakeStrategy.Watch. +type fakeWatch struct { + strat *fakeStrategy + path string + ch chan string + closed bool // guarded by strat.mu +} + +func (w *fakeWatch) Values() <-chan string { return w.ch } + +func (w *fakeWatch) Close() error { + s := w.strat + s.mu.Lock() + defer s.mu.Unlock() + if w.closed { + return nil + } + w.closed = true + close(w.ch) + s.watches-- + watches := s.subs[w.path] + for i, other := range watches { + if other == w { + s.subs[w.path] = append(watches[:i], watches[i+1:]...) + break + } + } + return nil +} diff --git a/test/integration/docker/docker-compose.yaml b/test/integration/docker/docker-compose.yaml new file mode 100644 index 0000000..8544d64 --- /dev/null +++ b/test/integration/docker/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + db: + image: postgres:10.3 + environment: + - POSTGRES_USER=admin + - POSTGRES_PASSWORD=test + - POSTGRES_DB=hldatabase + - HOTLOAD_PATH_CHKSUM_METRICS_ENABLE=true + ports: + - '5432:5432' + # `docker compose up --wait` blocks on this. The TCP host target + # matters: the postgres entrypoint runs init scripts against a + # temporary unix-socket-only server and then restarts; a plain + # pg_isready would report ready during that phase. + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U admin -d hldatabase"] + interval: 2s + timeout: 3s + retries: 30 + volumes: + # https://github.com/docker-library/docs/tree/master/postgres#initialization-scripts + - ./intgtest_init.sql:/docker-entrypoint-initdb.d/intgtest_init.sql diff --git a/integrationtests/docker/intgtest_init.sql b/test/integration/docker/intgtest_init.sql similarity index 100% rename from integrationtests/docker/intgtest_init.sql rename to test/integration/docker/intgtest_init.sql diff --git a/test/integration/go.mod b/test/integration/go.mod new file mode 100644 index 0000000..cc0ec19 --- /dev/null +++ b/test/integration/go.mod @@ -0,0 +1,17 @@ +module github.com/infobloxopen/hotload/test/integration + +go 1.23.0 + +require ( + github.com/infobloxopen/hotload/v3 v3.0.0-rc.1 + github.com/lib/pq v1.10.9 +) + +require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + golang.org/x/sys v0.35.0 // indirect +) + +// This module exists only to test the sibling modules in this repository; +// it is never tagged or imported, so the replace directive is permanent. +replace github.com/infobloxopen/hotload/v3 => ../../ diff --git a/test/integration/go.sum b/test/integration/go.sum new file mode 100644 index 0000000..0447aa2 --- /dev/null +++ b/test/integration/go.sum @@ -0,0 +1,7 @@ +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/test/integration/hotload_test.go b/test/integration/hotload_test.go new file mode 100644 index 0000000..8337d7e --- /dev/null +++ b/test/integration/hotload_test.go @@ -0,0 +1,303 @@ +package integration + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// TestSwitchDatabase: changing the watched DSN reroutes new connections to +// another database, observable via current_database(). +func TestSwitchDatabase(t *testing.T) { + requirePostgres(t) + + f := newDsnFile(t, adminDsn("hotload_test")) + db := openHotload(t, f, "") + + waitForQueryValue(t, db, "SELECT current_database()", "hotload_test", 5*time.Second) + + f.set(adminDsn("hotload_test1")) + waitForQueryValue(t, db, "SELECT current_database()", "hotload_test1", 10*time.Second) +} + +// TestPasswordRotationGraceful: rotating the database password and the DSN +// must transparently move the pool to fresh credentials; in-flight long +// operations on old connections complete undisturbed. +func TestPasswordRotationGraceful(t *testing.T) { + requirePostgres(t) + + ops := map[string]func(db *sql.DB) error{ + "Exec": func(db *sql.DB) error { + _, err := db.Exec("SELECT pg_sleep(2)") + return err + }, + "ExecContext": func(db *sql.DB) error { + _, err := db.ExecContext(context.Background(), "SELECT pg_sleep(2)") + return err + }, + "Query": func(db *sql.DB) error { + rows, err := db.Query("SELECT pg_sleep(2)") + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() + }, + "QueryContext": func(db *sql.DB) error { + rows, err := db.QueryContext(context.Background(), "SELECT pg_sleep(2)") + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() + }, + } + + for name, op := range ops { + t.Run(name, func(t *testing.T) { + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + + errCh := make(chan error, 1) + go func() { errCh <- op(db) }() + time.Sleep(300 * time.Millisecond) // let the long op reach the server + + newPass := rotatePassword(t) + f.set(userDsn(newPass)) + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("graceful mode must let the in-flight %s finish, got: %v", name, err) + } + case <-time.After(15 * time.Second): + t.Fatal("long operation did not return") + } + + // New connections must authenticate with the new password. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + }) + } +} + +// TestPasswordRotationForceKill: with forceKill the in-flight operation is +// cut when the DSN changes, and the pool recovers on the new credentials. +func TestPasswordRotationForceKill(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "forceKill=true") + + errCh := make(chan error, 1) + go func() { + _, err := db.Exec("SELECT pg_sleep(30)") + errCh <- err + }() + time.Sleep(300 * time.Millisecond) + + newPass := rotatePassword(t) + f.set(userDsn(newPass)) + + select { + case err := <-errCh: + if err == nil { + t.Fatal("forceKill mode must cancel the in-flight exec, got nil error") + } + t.Logf("in-flight exec canceled with: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("forceKill did not cancel the in-flight exec (30s sleep still running)") + } + + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) +} + +// TestLongTransactionAcrossChangeGraceful: a transaction spanning a config +// change completes on its original connection. +func TestLongTransactionAcrossChangeGraceful(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + + tag := fmt.Sprintf("txn-%d", time.Now().UnixNano()) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (1, $1)", tag); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("SELECT pg_sleep(1)"); err != nil { + t.Fatal(err) + } + + f.set(userDsn(rotatePassword(t))) + time.Sleep(300 * time.Millisecond) // let the change land mid-transaction + + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (2, $1)", tag); err != nil { + t.Fatalf("exec after change in graceful txn: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit after change in graceful txn: %v", err) + } + + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM test WHERE csource = $1", tag).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("committed rows = %d, want 2", n) + } +} + +// TestLongTransactionAcrossChangeForceKill: with forceKill a transaction +// holding a connection across a change is rolled back by the kill. +func TestLongTransactionAcrossChangeForceKill(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "forceKill=true") + + tag := fmt.Sprintf("txnfk-%d", time.Now().UnixNano()) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (1, $1)", tag); err != nil { + t.Fatal(err) + } + + f.set(userDsn(rotatePassword(t))) + time.Sleep(500 * time.Millisecond) // change lands; connection is killed + + err = func() error { + if _, err := tx.Exec("INSERT INTO test (cnum, csource) VALUES (2, $1)", tag); err != nil { + return err + } + return tx.Commit() + }() + if err == nil { + t.Fatal("transaction across a forceKill change should fail") + } + t.Logf("transaction failed as expected: %v", err) + + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM test WHERE csource = $1", tag).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("rows from killed transaction = %d, want 0 (rolled back)", n) + } +} + +// TestContextCancellation: caller-driven cancellation behaves identically +// in both modes and leaves the pool healthy. +func TestContextCancellation(t *testing.T) { + requirePostgres(t) + + for _, params := range []string{"", "forceKill=true"} { + name := "graceful" + if params != "" { + name = "forceKill" + } + t.Run(name, func(t *testing.T) { + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, params) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + _, err := db.ExecContext(ctx, "SELECT pg_sleep(10)") + if err == nil { + t.Fatal("expected context cancellation error") + } + if !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(err.Error(), "cancel") { + t.Fatalf("err = %v, want context cancellation", err) + } + + // The pool must remain healthy after the cancellation. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + }) + } +} + +// TestPreparedStatementAcrossChange: a prepared statement survives a +// graceful change — database/sql re-prepares it on a new connection. +func TestPreparedStatementAcrossChange(t *testing.T) { + requirePostgres(t) + + f := newDsnFile(t, adminDsn("hotload_test")) + db := openHotload(t, f, "") + + stmt, err := db.Prepare("SELECT current_database()") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + var got string + if err := stmt.QueryRow().Scan(&got); err != nil { + t.Fatal(err) + } + if got != "hotload_test" { + t.Fatalf("before change: %q, want hotload_test", got) + } + + f.set(adminDsn("hotload_test1")) + deadline := time.Now().Add(10 * time.Second) + for { + if err := stmt.QueryRow().Scan(&got); err != nil { + t.Fatalf("prepared stmt after change: %v", err) + } + if got == "hotload_test1" { + break + } + if time.Now().After(deadline) { + t.Fatalf("prepared stmt still routed to %q", got) + } + time.Sleep(50 * time.Millisecond) + } +} + +// TestMultipleSequentialLongExecs: a sequence of long operations, each +// overlapping one password rotation, all complete in graceful mode (each +// in-flight operation stays within its generation's grace period; only the +// generation before the previous one is killed). +func TestMultipleSequentialLongExecs(t *testing.T) { + requirePostgres(t) + + pass := rotatePassword(t) + f := newDsnFile(t, userDsn(pass)) + db := openHotload(t, f, "") + db.SetMaxOpenConns(10) + + for i := 0; i < 5; i++ { + errCh := make(chan error, 1) + go func() { + _, err := db.Exec("SELECT pg_sleep(1)") + errCh <- err + }() + time.Sleep(150 * time.Millisecond) // let the exec reach the server + + f.set(userDsn(rotatePassword(t))) + + if err := <-errCh; err != nil { + t.Fatalf("round %d: long exec failed in graceful mode: %v", i, err) + } + // Confirm the swap landed before the next round dials; transient + // auth failures while the new DSN propagates are retried here. + waitForQueryValue(t, db, "SELECT current_user", testDbUser, 10*time.Second) + } +} diff --git a/test/integration/main_test.go b/test/integration/main_test.go new file mode 100644 index 0000000..2d2dfad --- /dev/null +++ b/test/integration/main_test.go @@ -0,0 +1,208 @@ +// Package integration tests hotload against a real PostgreSQL server using +// the lib/pq driver and the fsnotify strategy. +// +// Start the database with `make postgres-docker-compose-up` (or point the +// HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST/PORT environment variables at an +// existing server provisioned with docker/intgtest_init.sql). Tests skip +// themselves when no server is reachable. +package integration + +import ( + "database/sql" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + _ "github.com/infobloxopen/hotload/v3/fsnotify" + "github.com/lib/pq" +) + +var ( + postgresHost = "localhost" + postgresPort = "5432" + + adminUser = "admin" + adminPass = "test" + testDbUser = "uuser" + + // passSeq makes every password rotation unique across tests. + passSeq atomic.Int64 + + setupOnce sync.Once + setupErr error + adminDB *sql.DB +) + +func TestMain(m *testing.M) { + hotload.RegisterSQLDriver("postgres", &pq.Driver{}) + + if h := strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST")); h != "" { + postgresHost = h + } + if p := strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT")); p != "" { + postgresPort = p + } + + os.Exit(m.Run()) +} + +func adminDsn(database string) string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", + adminUser, adminPass, postgresHost, postgresPort, database) +} + +func userDsn(pass string) string { + return fmt.Sprintf("postgresql://%s:%s@%s:%s/hldatabase?sslmode=disable", + testDbUser, pass, postgresHost, postgresPort) +} + +// enabled reports whether the integration tests were explicitly requested: +// either HOTLOAD_INTEGRATION_TESTS is truthy (set by `make +// local-integration-tests`) or a postgres host/port was pointed at via the +// environment. An implicit "is something listening on 5432" probe is not +// enough — an unrelated local postgres would fail authentication instead of +// skipping. +func enabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("HOTLOAD_INTEGRATION_TESTS"))) { + case "1", "true", "yes": + return true + } + return os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_HOST") != "" || + os.Getenv("HOTLOAD_INTEGRATION_TEST_POSTGRES_PORT") != "" +} + +// requirePostgres skips the test unless integration testing was requested +// and the server is reachable; it lazily provisions the test user and table. +func requirePostgres(t *testing.T) { + t.Helper() + if !enabled() { + t.Skip("skipping: integration tests not requested (run via `make local-integration-tests`, or set HOTLOAD_INTEGRATION_TESTS=1)") + } + addr := net.JoinHostPort(postgresHost, postgresPort) + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + t.Skipf("skipping: postgres not reachable at %s (start it with `make postgres-docker-compose-up`): %v", addr, err) + } + conn.Close() + + setupOnce.Do(func() { + adminDB, setupErr = sql.Open("postgres", adminDsn("hldatabase")) + if setupErr != nil { + return + } + // The TCP probe above only proves something is listening; postgres + // may still be initializing (its docker entrypoint restarts the + // server after running init scripts). Retry until it answers. + deadline := time.Now().Add(60 * time.Second) + for { + setupErr = adminDB.Ping() + if setupErr == nil { + break + } + if time.Now().After(deadline) { + setupErr = fmt.Errorf("postgres never became ready: %w", setupErr) + return + } + time.Sleep(500 * time.Millisecond) + } + stmts := []string{ + "DROP TABLE IF EXISTS test", + "DROP USER IF EXISTS " + testDbUser, + fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", testDbUser, nextPass(0)), + "CREATE TABLE IF NOT EXISTS test (cnum INT, csource TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)", + "GRANT ALL ON test TO PUBLIC", + } + for _, stmt := range stmts { + if _, setupErr = adminDB.Exec(stmt); setupErr != nil { + setupErr = fmt.Errorf("setup %q: %w", stmt, setupErr) + return + } + } + }) + if setupErr != nil { + t.Fatalf("postgres setup failed: %v", setupErr) + } +} + +// nextPass returns the password for rotation n (or the next unique one when +// n is 0). +func nextPass(n int64) string { + if n == 0 { + n = passSeq.Add(1) + } + return fmt.Sprintf("ppass%d", n) +} + +// rotatePassword sets a fresh password for the test user on the server and +// returns it. Existing sessions stay valid; only new dials use the new one. +func rotatePassword(t *testing.T) string { + t.Helper() + pass := nextPass(0) + if _, err := adminDB.Exec(fmt.Sprintf("ALTER USER %s WITH PASSWORD '%s'", testDbUser, pass)); err != nil { + t.Fatalf("rotatePassword: %v", err) + } + return pass +} + +// dsnFile manages the config file watched by the fsnotify strategy. Each +// test gets its own path, hence its own hotload group. +type dsnFile struct { + t *testing.T + path string +} + +func newDsnFile(t *testing.T, dsn string) *dsnFile { + t.Helper() + f := &dsnFile{t: t, path: filepath.Join(t.TempDir(), "dsn.txt")} + f.set(dsn) + return f +} + +func (f *dsnFile) set(dsn string) { + f.t.Helper() + if err := os.WriteFile(f.path, []byte(dsn), 0o644); err != nil { + f.t.Fatalf("writing dsn file: %v", err) + } +} + +// openHotload opens a hotload sql.DB watching f. +func openHotload(t *testing.T, f *dsnFile, params string) *sql.DB { + t.Helper() + url := "fsnotify://postgres" + f.path + if params != "" { + url += "?" + params + } + db, err := sql.Open("hotload", url) + if err != nil { + t.Fatalf("sql.Open(%q): %v", url, err) + } + t.Cleanup(func() { db.Close() }) + if err := db.Ping(); err != nil { + t.Fatalf("ping through hotload: %v", err) + } + return db +} + +// waitForQueryValue polls query until it returns want or the timeout +// elapses. +func waitForQueryValue(t *testing.T, db *sql.DB, query, want string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last string + var lastErr error + for time.Now().Before(deadline) { + lastErr = db.QueryRow(query).Scan(&last) + if lastErr == nil && last == want { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for %q to return %q (last value %q, last err %v)", query, want, last, lastErr) +} diff --git a/transaction.go b/transaction.go deleted file mode 100644 index 3761f11..0000000 --- a/transaction.go +++ /dev/null @@ -1,82 +0,0 @@ -package hotload - -import ( - "context" - "database/sql/driver" - - "github.com/infobloxopen/hotload/logger" - "github.com/infobloxopen/hotload/metrics" -) - -// managedTx wraps a sql/driver.Tx so that it can store the context of the -// transaction and clean up the execqueryCallsCounter on Commit or Rollback. -type managedTx struct { - tx driver.Tx - conn *managedConn - ctx context.Context -} - -func (t *managedTx) Commit() error { - var log = logger.GetLogger() - log("managedTx.Commit") - err := t.tx.Commit() - t.cleanup() - return err -} - -func (t *managedTx) Rollback() error { - var log = logger.GetLogger() - log("managedTx.Rollback") - err := t.tx.Rollback() - t.cleanup() - return err -} - -func observeSQLStmtsSummary(ctx context.Context, execStmtsCounter, queryStmtsCounter int64) { - labels := GetExecLabelsFromContext(ctx) - service := labels[metrics.GRPCServiceKey] - method := labels[metrics.GRPCMethodKey] - - metrics.SqlStmtsSummary.WithLabelValues(service, method, metrics.ExecStatement).Observe(float64(execStmtsCounter)) - metrics.SqlStmtsSummary.WithLabelValues(service, method, metrics.QueryStatement).Observe(float64(queryStmtsCounter)) -} - -func (t *managedTx) cleanup() { - observeSQLStmtsSummary(t.ctx, t.conn.execStmtsCounter.Load(), t.conn.queryStmtsCounter.Load()) - t.conn.resetExecStmtsCounter() - t.conn.resetQueryStmtsCounter() -} - -type promLabelKeyType struct{} - -var promLabelKey = promLabelKeyType{} - -func ContextWithExecLabels(ctx context.Context, labels map[string]string) context.Context { - var log = logger.GetLogger() - if labels == nil { - log("ContextWithExecLabels called with nil label set") - return ctx - } - return context.WithValue(ctx, promLabelKey, labels) -} - -func GetExecLabelsFromContext(ctx context.Context) map[string]string { - var log = logger.GetLogger() - if ctx == nil { - log("No context provided, returning") - return nil - } - - value := ctx.Value(promLabelKey) - if value == nil { - log("No value for promLabelKey, returning") - return nil - } - labelMap, ok := value.(map[string]string) - if !ok { - log("Bad value type used for promLabelKey, conversion error") - return nil - } - - return labelMap -} diff --git a/tx.go b/tx.go new file mode 100644 index 0000000..cceaac9 --- /dev/null +++ b/tx.go @@ -0,0 +1,37 @@ +package hotload + +import ( + "context" + "database/sql/driver" +) + +// managedTx wraps a driver.Tx so the statement counters accumulated on the +// conn can be reported to hooks when the transaction completes. +type managedTx struct { + tx driver.Tx + conn *baseConn + ctx context.Context +} + +func (t *managedTx) Commit() error { + err := t.tx.Commit() + t.complete(err == nil) + return err +} + +func (t *managedTx) Rollback() error { + err := t.tx.Rollback() + t.complete(false) + return err +} + +func (t *managedTx) complete(committed bool) { + emitTxComplete(TxEvent{ + Ctx: t.ctx, + ExecStmts: t.conn.execStmts.Load(), + QueryStmts: t.conn.queryStmts.Load(), + Committed: committed, + }) + t.conn.execStmts.Store(0) + t.conn.queryStmts.Store(0) +} diff --git a/tx_test.go b/tx_test.go new file mode 100644 index 0000000..40e3ca4 --- /dev/null +++ b/tx_test.go @@ -0,0 +1,189 @@ +package hotload_test + +import ( + "context" + "testing" + "time" + + hotload "github.com/infobloxopen/hotload/v3" + "github.com/infobloxopen/hotload/v3/internal/testutil" +) + +// collectTxEvents subscribes a hook capturing TxEvents into a channel. +func collectTxEvents(t *testing.T) chan hotload.TxEvent { + t.Helper() + events := make(chan hotload.TxEvent, 100) + hotload.RegisterHooks(hotload.Hooks{ + OnTxComplete: func(ev hotload.TxEvent) { + select { + case events <- ev: + default: + } + }, + }) + return events +} + +func awaitTxEvent(t *testing.T, events chan hotload.TxEvent) hotload.TxEvent { + t.Helper() + select { + case ev := <-events: + return ev + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for TxEvent") + return hotload.TxEvent{} + } +} + +// TestTxStatementCounters: exec and query statements run on a conn are +// reported when its transaction completes, along with the labels carried by +// the transaction's context. +func TestTxStatementCounters(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + labels := map[string]string{"grpc_service": "svc", "grpc_method": "m"} + ctx := hotload.ContextWithExecLabels(context.Background(), labels) + + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE y"); err != nil { + t.Fatal(err) + } + var dsn string + if err := tx.QueryRow("SELECT dsn").Scan(&dsn); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.ExecStmts != 2 || ev.QueryStmts != 1 { + t.Errorf("counters = exec %d query %d, want exec 2 query 1", ev.ExecStmts, ev.QueryStmts) + } + if !ev.Committed { + t.Error("Committed = false, want true") + } + got := hotload.GetExecLabelsFromContext(ev.Ctx) + if got["grpc_service"] != "svc" || got["grpc_method"] != "m" { + t.Errorf("labels from event ctx = %v, want %v", got, labels) + } +} + +// TestTxRollbackEvent: rollbacks report Committed=false and reset counters +// for the next transaction. +func TestTxRollbackEvent(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + ctx := context.Background() + tx, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("UPDATE x"); err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.Committed { + t.Error("Committed = true, want false") + } + if ev.ExecStmts != 1 { + t.Errorf("ExecStmts = %d, want 1", ev.ExecStmts) + } + + // Counters must reset between transactions on the same conn. + tx2, err := fx.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := tx2.Commit(); err != nil { + t.Fatal(err) + } + ev2 := awaitTxEvent(t, events) + if ev2.ExecStmts != 0 || ev2.QueryStmts != 0 { + t.Errorf("second tx counters = exec %d query %d, want 0/0 (reset)", ev2.ExecStmts, ev2.QueryStmts) + } +} + +// TestTxCountsPreparedStatements: statements executed through prepared +// statements count toward the transaction counters — a blind spot in v1, +// which did not wrap driver.Stmt. +func TestTxCountsPreparedStatements(t *testing.T) { + testutil.NoLeaks(t) + fx := newFixture(t, fxCfg{}) + events := collectTxEvents(t) + + tx, err := fx.db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + stmt, err := tx.Prepare("UPDATE x") + if err != nil { + t.Fatal(err) + } + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + if _, err := stmt.Exec(); err != nil { + t.Fatal(err) + } + if err := stmt.Close(); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + ev := awaitTxEvent(t, events) + if ev.ExecStmts != 2 { + t.Errorf("ExecStmts = %d, want 2 (prepared statement execs must count)", ev.ExecStmts) + } +} + +// TestConnAndWatchEvents: conn open/close and watch hooks fire with the +// group name and redacted DSNs. +func TestConnAndWatchEvents(t *testing.T) { + testutil.NoLeaks(t) + + fx := newFixture(t, fxCfg{}) + opens := make(chan hotload.ConnEvent, 10) + closes := make(chan hotload.ConnEvent, 10) + hotload.RegisterHooks(hotload.Hooks{ + OnConnOpen: func(ev hotload.ConnEvent) { opens <- ev }, + OnConnClose: func(ev hotload.ConnEvent) { closes <- ev }, + }) + + fx.queryDSN() + select { + case ev := <-opens: + if ev.GroupName != fx.dsn { + t.Errorf("open event group = %q, want %q", ev.GroupName, fx.dsn) + } + case <-time.After(2 * time.Second): + t.Fatal("no conn open event") + } + + fx.pushAndWait("dsn-2") + fx.queryDSN() + select { + case ev := <-closes: + if ev.GroupName != fx.dsn { + t.Errorf("close event group = %q, want %q", ev.GroupName, fx.dsn) + } + case <-time.After(2 * time.Second): + t.Fatal("no conn close event after swap") + } +}