Skip to content

Generalize external database modules #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Areson merged 2 commits into from
Aug 7, 2026
Merged

Generalize external database modules #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1902b8
Generalize external database modules
Areson Aug 7, 2026
0ba9c3b
Interpolate typed database config values
Areson Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions blip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ const (
EVENT
)

// Metrics are metrics collected for one plan level, from one MySQL instance.
// Metrics are metrics collected for one plan level, from one monitor.
type Metrics struct {
Begin time.Time // when collection started
End time.Time // when collection completed
MonitorId string // ID of monitor (MySQL)
MonitorId string // ID of monitor
Plan string // plan name
Level string // level name
Interval uint // interval number
Expand Down Expand Up @@ -99,7 +99,7 @@ type SinkFactoryArgs struct {
Tags map[string]string // config.monitor.tags
}

// DbCredentials are MySQL credentials parsed or loaded for a connection.
// DbCredentials are database credentials parsed or loaded for a connection.
type DbCredentials struct {
Username string
Password string
Expand Down
3 changes: 2 additions & 1 deletion collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ type CollectorFactoryWithDBProvider interface {
//
// The domain argument allows one factory to serve domains with different
// compatibility, such as MySQL collectors and database-neutral cloud metrics.
// Implementations must return at least one database type.
// Implementations must return at least one database type. Return
// DatabaseTypeAny by itself for a database-neutral domain.
type CollectorFactoryDatabaseTypes interface {
CollectorFactory
DatabaseTypes(domain string) []DatabaseType
Expand Down
144 changes: 71 additions & 73 deletions config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,8 @@ type ConfigMonitor struct {
// Empty values retain Blip's historical MySQL behavior.
DatabaseType DatabaseType `yaml:"database-type,omitempty"`

// ConfigMySQL:
// Shared connection identity and credentials. Socket and MyCnf are specific
// to Blip's built-in MySQL connection factory.
Socket string `yaml:"socket,omitempty"`
Hostname string `yaml:"hostname,omitempty"`
MyCnf string `yaml:"mycnf,omitempty"`
Expand All @@ -422,9 +423,12 @@ type ConfigMonitor struct {
Heartbeat ConfigHeartbeat `yaml:"heartbeat,omitempty"`
Plans ConfigPlans `yaml:"plans,omitempty"`
Plan string `yaml:"plan,omitempty"`
Postgres ConfigPostgres `yaml:"postgres,omitempty"`
Sinks ConfigSinks `yaml:"sinks,omitempty"`
TLS ConfigTLS `yaml:"tls,omitempty"`
// DatabaseConfig is opaque configuration owned by the external module
// selected by DatabaseType. MySQL continues to use the historical monitor
// fields above and rejects this section.
DatabaseConfig ConfigDatabase `yaml:"database-config,omitempty"`
Sinks ConfigSinks `yaml:"sinks,omitempty"`
TLS ConfigTLS `yaml:"tls,omitempty"`

Meta map[string]string `yaml:"meta,omitempty"`
}
Expand All @@ -434,13 +438,6 @@ const (
DEFAULT_MONITOR_TIMEOUT_CONNECT = "10s"
)

type DatabaseType string

const (
DatabaseTypeMySQL DatabaseType = "mysql"
DatabaseTypePostgres DatabaseType = "postgres"
)

func DefaultConfigMonitor() ConfigMonitor {
return ConfigMonitor{
Username: DEFAULT_MONITOR_USERNAME,
Expand All @@ -459,45 +456,55 @@ func DefaultConfigMonitor() ConfigMonitor {
}

// EffectiveDatabaseType returns the configured database type. An omitted
// value retains Blip's historical MySQL behavior. Environment interpolation is
// resolved here because monitor defaults are applied before the normal
// interpolation pass.
// value retains Blip's historical MySQL behavior. Direct environment-variable
// interpolation is resolved before defaults are selected; database type is a
// structural discriminator and does not support monitor-field interpolation.
func (c ConfigMonitor) EffectiveDatabaseType() DatabaseType {
databaseType := DatabaseType(interpolateEnv(string(c.DatabaseType)))
databaseType := interpolateEnv(string(c.DatabaseType))
if databaseType == "" {
return DatabaseTypeMySQL
}
return databaseType
return DatabaseType(databaseType)
}

func (c ConfigMonitor) Validate() error {
switch c.EffectiveDatabaseType() {
case DatabaseTypeMySQL:
if c.Postgres.Set() {
return fmt.Errorf("config.monitor.postgres requires database-type %q", DatabaseTypePostgres)
}
case DatabaseTypePostgres:
if c.Socket != "" {
return fmt.Errorf("config.monitor.socket is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.MyCnf != "" {
return fmt.Errorf("config.monitor.mycnf is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Heartbeat.set() {
return fmt.Errorf("config.monitor.heartbeat is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Plans.Change.set() {
return fmt.Errorf("config.monitor.plans.change is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Plans.Table != "" {
return fmt.Errorf("config.monitor.plans.table is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Exporter.Mode != "" && c.Exporter.Plan == "" {
return fmt.Errorf("config.monitor.exporter.plan is required for database-type %q", DatabaseTypePostgres)
databaseType := c.EffectiveDatabaseType()
if databaseType == DatabaseTypeMySQL {
if len(c.DatabaseConfig) > 0 {
return fmt.Errorf("config.monitor.database-config requires an external database type")
}
return c.Postgres.Validate()
default:
return fmt.Errorf("config.monitor.database-type: invalid database type %q", c.DatabaseType)
return nil
}
if databaseType == DatabaseTypeAny {
return fmt.Errorf("config.monitor.database-type: %q is reserved for database-neutral collectors", databaseType)
}
if !ValidDatabaseType(databaseType) {
return fmt.Errorf("config.monitor.database-type: invalid database type %q", databaseType)
}
if c.Socket != "" {
return fmt.Errorf("config.monitor.socket is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.MyCnf != "" {
return fmt.Errorf("config.monitor.mycnf is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Heartbeat.set() {
return fmt.Errorf("config.monitor.heartbeat is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Plans.Change.set() {
return fmt.Errorf("config.monitor.plans.change is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Plans.Table != "" {
return fmt.Errorf("config.monitor.plans.table is only supported for database-type %q", DatabaseTypeMySQL)
}
if c.Exporter.set() {
return fmt.Errorf("config.monitor.exporter is only supported for database-type %q", DatabaseTypeMySQL)
}
module, ok := registeredDatabaseModule(databaseType)
if !ok {
return fmt.Errorf("config.monitor.database-type: database module %q is not registered", databaseType)
}
if err := module.ValidateConfig(c); err != nil {
return fmt.Errorf("config.monitor.database-config for %q: %w", databaseType, err)
}
return nil
}
Expand All @@ -509,6 +516,9 @@ func (c ConfigMonitor) Redacted() ConfigMonitor {

func (c ConfigMonitor) redacted(seen map[*ConfigMonitor]*ConfigMonitor) ConfigMonitor {
c.Password = redactPassword(c.Password)
// External module configuration is opaque to Blip, so redact the entire
// section rather than guessing which module-owned fields might be secrets.
c.DatabaseConfig = nil
c.Sinks = c.Sinks.redacted()
c.Plans = c.Plans.redacted(seen)
return c
Expand Down Expand Up @@ -558,20 +568,16 @@ func (c *ConfigMonitor) ApplyDefaults(b Config) {
c.Sinks = ConfigSinks{}
}
c.AWS.ApplyDefaults(b)
c.Exporter.applyDefaults(b, databaseType == DatabaseTypeMySQL)
c.HA.ApplyDefaults(b)
// Heartbeat writes and plan state changes use MySQL-specific SQL. Do not
// inherit their global defaults into PostgreSQL monitors; explicit monitor
// values remain intact so Validate can report the unsupported configuration.
// Exporter emulation, heartbeat writes, and plan state changes are
// MySQL-specific. Do not inherit their global defaults into external database
// monitors; explicit monitor values remain intact so Validate can report the
// unsupported configuration.
if databaseType == DatabaseTypeMySQL {
c.Exporter.ApplyDefaults(b)
c.Heartbeat.ApplyDefaults(b)
}
c.Plans.applyDefaults(b, databaseType == DatabaseTypeMySQL)
if databaseType == DatabaseTypePostgres {
postgresDefaults := DefaultConfigPostgres()
postgresDefaults.ConnectTimeout = c.TimeoutConnect
c.Postgres.ApplyDefaults(postgresDefaults)
}
c.Sinks.ApplyDefaults(b)
c.TLS.ApplyDefaults(b)
}
Expand All @@ -598,7 +604,7 @@ func (c *ConfigMonitor) InterpolateEnvVars() {
c.Heartbeat.InterpolateEnvVars()
c.Plans.InterpolateEnvVars()
c.Plan = interpolateEnv(c.Plan)
c.Postgres.InterpolateEnvVars()
c.DatabaseConfig.interpolate(interpolateEnv)
c.Sinks.InterpolateEnvVars()
c.TLS.InterpolateEnvVars()
}
Expand All @@ -624,7 +630,7 @@ func (c *ConfigMonitor) InterpolateMonitor() {
c.Heartbeat.InterpolateMonitor(c)
c.Plans.InterpolateMonitor(c)
c.Plan = c.interpolateMon(c.Plan)
c.Postgres.InterpolateMonitor(c)
c.DatabaseConfig.interpolate(c.interpolateMon)
c.Sinks.InterpolateMonitor(c)
c.TLS.InterpolateMonitor(c)
}
Expand Down Expand Up @@ -676,20 +682,6 @@ func (c *ConfigMonitor) fieldValue(f string) string {
return c.PasswordFile
case "timeout-connect":
return c.TimeoutConnect
case "postgres.database":
return c.Postgres.Database
case "postgres.application-name":
return c.Postgres.ApplicationName
case "postgres.ssl-mode":
return c.Postgres.SSLMode
case "postgres.connect-timeout":
return c.Postgres.ConnectTimeout
case "postgres.statement-timeout":
return c.Postgres.StatementTimeout
case "postgres.lock-timeout":
return c.Postgres.LockTimeout
case "postgres.dial-address":
return c.Postgres.DialAddress
default:
return ""
}
Expand Down Expand Up @@ -753,6 +745,10 @@ type ConfigExporter struct {
Plan string `yaml:"plan,omitempty"`
}

func (c ConfigExporter) set() bool {
return c.Mode != "" || c.Plan != "" || len(c.Flags) > 0
}

func DefaultConfigExporter() ConfigExporter {
return ConfigExporter{}
}
Expand All @@ -768,10 +764,6 @@ func (c ConfigExporter) Validate() error {
}

func (c *ConfigExporter) ApplyDefaults(b Config) {
c.applyDefaults(b, true)
}

func (c *ConfigExporter) applyDefaults(b Config, useDefaultPlan bool) {
if c.Mode == "" && b.Exporter.Mode != "" {
c.Mode = b.Exporter.Mode
}
Expand All @@ -782,7 +774,7 @@ func (c *ConfigExporter) applyDefaults(b Config, useDefaultPlan bool) {
if c.Plan == "" && b.Exporter.Plan != "" {
c.Plan = b.Exporter.Plan
}
if c.Plan == "" && useDefaultPlan {
if c.Plan == "" {
c.Plan = DEFAULT_EXPORTER_PLAN
}
if len(b.Exporter.Flags) > 0 {
Expand Down Expand Up @@ -992,9 +984,15 @@ func DefaultConfigPlans() ConfigPlans {
}

func (c ConfigPlans) Validate() error {
if c.Table != "" && c.Monitor != nil && c.Monitor.EffectiveDatabaseType() != DatabaseTypeMySQL {
if c.Table == "" || c.Monitor == nil {
return nil
}
if c.Monitor.EffectiveDatabaseType() != DatabaseTypeMySQL {
return fmt.Errorf("config.plans.monitor.database-type: config.plans.table is only supported for database-type %q", DatabaseTypeMySQL)
}
if err := c.Monitor.Validate(); err != nil {
return fmt.Errorf("config.plans.monitor: %w", err)
}
return nil
}

Expand Down
Loading