Skip to content

[PRODCRE-1483] obs stack fixed location #2331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Tofel merged 2 commits into from
Dec 17, 2025
Merged

[PRODCRE-1483] obs stack fixed location #2331

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1cbd72
extract observability files to fixed directory, don't remove them on …
Tofel Dec 15, 2025
197e2a0
fix env var name
Tofel Dec 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 60 additions & 24 deletions framework/observability.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,25 @@ const (
LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1"
LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=5s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-15m&to=now"
LocalPyroScopeURL = "http://localhost:4040/?query=process_cpu%3Acpu%3Ananoseconds%3Acpu%3Ananoseconds%7Bservice_name%3D%22chainlink-node%22%7D&from=now-15m"

CTFCacheDir = ".local/share/ctf"
)

// extractAllFiles goes through the embedded directory and extracts all files to the current directory
// getObservabilityDir returns the fixed directory where observability files are extracted
func getObservabilityDir() (string, error) {
homeDir, err := os.UserHomeDir()
if err != nil {
return "", fmt.Errorf("failed to get home directory: %w", err)
}
return filepath.Join(homeDir, CTFCacheDir), nil
Comment thread
Tofel marked this conversation as resolved.
}

// extractAllFiles goes through the embedded directory and extracts all files to the fixed observability directory
func extractAllFiles(embeddedDir string) error {
// Get current working directory where CLI is running
currentDir, err := os.Getwd()
// Get fixed observability directory
obsDir, err := getObservabilityDir()
if err != nil {
return fmt.Errorf("failed to get current directory: %w", err)
return err
}

// Walk through the embedded files
Expand All @@ -47,7 +58,7 @@ func extractAllFiles(embeddedDir string) error {
}

// Read file content from embedded file system
content, err := EmbeddedObservabilityFiles.ReadFile(path)
content, err := fs.ReadFile(EmbeddedObservabilityFiles, path)
if err != nil {
return fmt.Errorf("failed to read file %s: %w", path, err)
}
Expand All @@ -57,7 +68,7 @@ func extractAllFiles(embeddedDir string) error {
if err != nil {
return fmt.Errorf("failed to determine relative path for %s: %w", path, err)
}
targetPath := filepath.Join(currentDir, relativePath)
targetPath := filepath.Join(obsDir, relativePath)

// Create target directories if necessary
targetDir := filepath.Dir(targetPath)
Expand All @@ -83,8 +94,13 @@ func BlockScoutUp(url, chainID string) error {
if err := extractAllFiles("observability"); err != nil {
return err
}
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
blockscoutDir := filepath.Join(obsDir, "blockscout")
os.Setenv("BLOCKSCOUT_RPC_URL", url)
os.Setenv("BLOCKSCOUT_CHAID_ID", chainID)
os.Setenv("BLOCKSCOUT_CHAIN_ID", chainID)
// old migrations for v15 is still applied somehow, cleaning up DB helps
if err := RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
Expand All @@ -93,13 +109,13 @@ func BlockScoutUp(url, chainID string) error {
rm -rf redis-data && \
rm -rf stats-db-data && \
rm -rf dets
`, filepath.Join("blockscout", "services"))); err != nil {
`, filepath.Join(blockscoutDir, "services"))); err != nil {
return err
}
err := RunCommand("bash", "-c", fmt.Sprintf(`
err = RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose up -d
`, "blockscout"))
`, blockscoutDir))
if err != nil {
return err
}
Expand All @@ -110,15 +126,16 @@ func BlockScoutUp(url, chainID string) error {

func BlockScoutDown(url string) error {
L.Info().Msg("Removing local Blockscout stack")
os.Setenv("BLOCKSCOUT_RPC_URL", url)
err := RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose down -v
`, "blockscout"))
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
return RunCommand("bash", "-c", "rm -rf blockscout/")
blockscoutDir := filepath.Join(obsDir, "blockscout")
os.Setenv("BLOCKSCOUT_RPC_URL", url)
return RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose down -v
`, blockscoutDir))
}

// ObservabilityUpOnlyLoki slim stack with only Loki to verify specific logs of CL nodes or services in tests
Expand All @@ -127,14 +144,19 @@ func ObservabilityUpOnlyLoki() error {
if err := extractAllFiles("observability"); err != nil {
return err
}
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
composeDir := filepath.Join(obsDir, "compose")
_ = DefaultNetwork(nil)
if err := NewPromtail(); err != nil {
return err
}
err := RunCommand("bash", "-c", fmt.Sprintf(`
err = RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose up -d loki grafana
`, "compose"))
`, composeDir))
if err != nil {
return err
}
Expand All @@ -149,14 +171,19 @@ func ObservabilityUp() error {
if err := extractAllFiles("observability"); err != nil {
return err
}
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
composeDir := filepath.Join(obsDir, "compose")
_ = DefaultNetwork(nil)
if err := NewPromtail(); err != nil {
return err
}
err := RunCommand("bash", "-c", fmt.Sprintf(`
err = RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose up -d otel-collector prometheus loki grafana
`, "compose"))
`, composeDir))
if err != nil {
return err
}
Expand All @@ -174,14 +201,19 @@ func ObservabilityUpFull() error {
if err := extractAllFiles("observability"); err != nil {
return err
}
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
composeDir := filepath.Join(obsDir, "compose")
_ = DefaultNetwork(nil)
if err := NewPromtail(); err != nil {
return err
}
err := RunCommand("bash", "-c", fmt.Sprintf(`
err = RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose up -d
`, "compose"))
`, composeDir))
if err != nil {
return err
}
Expand All @@ -197,10 +229,14 @@ func ObservabilityUpFull() error {

func ObservabilityDown() error {
L.Info().Msg("Removing local observability stack")
obsDir, err := getObservabilityDir()
if err != nil {
return err
}
composeDir := filepath.Join(obsDir, "compose")
_ = RunCommand("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose down -v && docker rm -f promtail
`, "compose"))
_ = RunCommand("bash", "-c", "rm -rf compose/")
`, composeDir))
return nil
}
Loading