examples: add changefeed_health collection + integration tests

.gitignore

@@ -18,3 +18,4 @@ tmp/
 certs
 visus
 visus-*
+heap_profiler/

README.md

@@ -47,6 +47,7 @@ The sidecar needs `SELECT ON TABLES` privileges on the `_visus` database to read
 To run many of the sample collections available in the examples directory,
 the 'VIEWACTIVITY' option should be granted to the user.
 The `./visus init` command will provision a `visus` user with the minimal privileges to run the sidecar. Defining new collection may require additional privileges, depending on what data the SQL query associated to the collection has to access.
+Each file in the examples directory states, in a comment at the top, the privileges its query needs beyond the ones `visus init` grants.
 
 ## Example
 

docs/deployment.md

@@ -67,7 +67,10 @@ cluster-scope coordination.
 
 If a collection's query reads tables beyond `crdb_internal` /
 `_visus`, the `visus` role needs SELECT on those tables as well. Grant
-them separately.
+them separately. A query reading `system` tables needs
+`GRANT SYSTEM VIEWSYSTEMTABLE` instead.
+Each file in `examples/` names the privileges its query needs on top of
+what `visus init` grants.
 
 ## Defining collections
 

examples/blocking_statements.yaml

@@ -14,6 +14,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: none beyond what "visus init" grants (VIEWACTIVITY).
+
 name: blocking_statements
 enabled: true
 scope: cluster

examples/changefeed_health.yaml

@@ -0,0 +1,58 @@
+# Copyright 2024 The Cockroach Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Privileges: beyond what "visus init" grants, [SHOW CHANGEFEED JOBS] needs
+#   GRANT SYSTEM VIEWJOB TO visus;
+# otherwise it only returns jobs the visus user owns.
+#
+# lag_seconds is -1 for a changefeed that has no checkpoint yet
+# (high_water_timestamp is NULL). Canceled and failed changefeeds stay in
+# [SHOW CHANGEFEED JOBS] for 12h with a lag_seconds that keeps growing, so
+# alerts should filter on status="running".
+#
+# target_tables/target_database come from full_table_names, always
+# fully-qualified as "database.schema.table"; target_database only reflects
+# the first table, so it's wrong for changefeeds spanning multiple
+# databases. database_name isn't used: it's always NULL today, reserved
+# for CREATE CHANGEFEED FOR DATABASE, which isn't implemented yet (CRDB
+# issue 154053). full_table_names also re-resolves descriptor IDs live, so
+# a since-dropped table/database can show up as NULL or as bracketed IDs
+# (e.g. "[123].[456].foo") instead of its original name.
+#
+name: changefeed_health
+enabled: false
+scope: cluster
+frequency: 30
+maxresults: 200
+labels: [job_id, status, sink_type, target_tables, target_database]
+metrics:
+  - name : lag_seconds
+    kind : gauge
+    help : seconds between the changefeed's high-water mark and now; -1 before the first checkpoint.
+query:
+  SELECT
+      job_id::STRING AS job_id,
+      status,
+      coalesce(regexp_extract(description, $$INTO '([a-zA-Z0-9+-]+):$$), '') AS sink_type,
+      coalesce(array_to_string(full_table_names, ', '), '') AS target_tables,
+      coalesce(split_part(full_table_names[1], '.', 1), '') AS target_database,
+      CASE WHEN high_water_timestamp IS NULL THEN -1
+           ELSE extract(epoch FROM now() - hlc_to_timestamp(high_water_timestamp))
+      END::FLOAT AS lag_seconds
+  FROM
+      [SHOW CHANGEFEED JOBS]
+  LIMIT
+    $1;

examples/cluster_sqlactivity.yaml

@@ -14,6 +14,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, reading system.statement_statistics needs
+#   GRANT SYSTEM VIEWSYSTEMTABLE TO visus;
+
 # Note: This query is meant to be executed at a cluster level, 
 # and very low frequency. 
 # If visus is deployed in a self hosted cluster as a sidecar for each node, 
@@ -65,7 +68,7 @@ query:
     SELECT
       encode(fingerprint_id, 'hex') as statement,
       app_name as application,
-      metadata->'db' as database,
+      metadata->'db' as db_name,
       sum((statistics->'execution_statistics'->'cnt')::float8) AS exec_count,
       max((statistics->'execution_statistics'->'maxDiskUsage'->'mean')::float8) AS max_disk,
       sum((statistics->'execution_statistics'->'networkBytes'->'mean')::float8  * (statistics->'execution_statistics'->'cnt')::float8 )  AS net_bytes,
@@ -84,10 +87,10 @@ query:
       AND aggregated_ts = (select * from max)
       AND aggregation_interval = '1h'  
     GROUP BY
-      statement, application, database  
+      statement, application, db_name
   )
-  select 
-    statement, application, database,
+  select
+    statement, application, db_name as database,
     exec_count,
     net_bytes / exec_count as net_bytes,
     max_disk,

examples/contended_tables.yaml

@@ -14,6 +14,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, the visus user needs SELECT on
+# the tables it reports on. CockroachDB v24.3 hides rows in crdb_internal for
+# tables the user cannot read; later versions do not filter.
+
 name: contended_tables
 enabled: true
 scope: cluster

examples/index_usage.yaml

@@ -14,6 +14,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, the visus user needs SELECT on
+# the tables it reports on. CockroachDB v24.3 hides rows in crdb_internal for
+# tables the user cannot read; later versions do not filter.
+
 
 name: index
 enabled: true

examples/inflight.yaml

@@ -14,6 +14,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: none beyond what "visus init" grants (VIEWACTIVITY).
+
 name: inflight
 enabled: true
 scope: node

examples/intents.yaml

@@ -14,6 +14,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, the visus user needs SELECT on
+# the tables it reports on. CockroachDB v24.3 hides rows in crdb_internal for
+# tables the user cannot read; later versions do not filter.
+
+# crdb_internal.ranges_no_leases lost its table_id column in v23.1, so the
+# table id is parsed out of start_pretty instead, matching table_mvcc.yaml.
 name: intent
 enabled: true
 scope: cluster
@@ -24,16 +30,23 @@ metrics:
   - name : count
     kind : counter
     help : number of intents.
-query: 
+query:
   SELECT table_name, intent_count as count
   FROM (
-          SELECT name as table_name,
+          SELECT t.name as table_name,
                  sum(
-                    (crdb_internal.range_stats(start_key)->'intent_count')::INT8
+                    (crdb_internal.range_stats(r.start_key)->'intent_count')::INT8
                  ) AS intent_count
             FROM crdb_internal.ranges_no_leases AS r
-            JOIN crdb_internal.tables AS t ON r.table_id = t.table_id
-        GROUP BY name
+            JOIN crdb_internal.tables AS t ON (
+                CASE
+                    WHEN strpos(substring(r.start_pretty, 8), '/') > 0
+                    THEN substring(r.start_pretty, 8, strpos(substring(r.start_pretty, 8), '/') - 1)
+                    ELSE substring(r.start_pretty, 8)
+                    END
+                )::INT8 = t.table_id
+           WHERE r.start_pretty::STRING LIKE '/Table/%'
+        GROUP BY t.name
        ) AS inline
  WHERE intent_count > 0
  LIMIT $1;

examples/protected_ts.yaml

@@ -14,6 +14,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, reading system.protected_ts_records needs
+#   GRANT SYSTEM VIEWSYSTEMTABLE TO visus;
+
 
 name: protected_ts
 enabled: false

examples/sqlactivity.yaml

@@ -14,6 +14,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: none beyond what "visus init" grants (VIEWACTIVITY).
+
 name: sqlactivity
 enabled: true
 scope: node

examples/sqlefficiency.yaml

@@ -14,7 +14,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: none beyond what "visus init" grants (VIEWACTIVITY).
 
+# The "explicitlio" type (logical I/O from statements run in an explicit
+# transaction) was dropped: CockroachDB v26.3 removed
+# crdb_internal.node_statement_statistics.implicit_txn, and no replacement
+# is exposed (see cockroachdb/cockroach#141862).
 name: sqlefficiency
 enabled: true
 scope: node
@@ -31,7 +36,6 @@ query:
       AS (
         SELECT
           statement_id,
-          IF(implicit_txn = 'false', 1, 0) AS explicittxn,
           IF(full_scan, 1, 0) AS fullscan,
           IF(sample_plan::STRING LIKE '%index join%', 1, 0) AS ijoinstmt,
           rows_avg::INT8 AS numrows,
@@ -49,13 +53,11 @@ query:
           statement_id,
           sum(fullscan * execcnt) AS fullcnt,
           sum(ijoinstmt * execcnt) AS ijoincnt,
-          sum(explicittxn * execcnt) AS explicitcnt,
           sum(
             (
               IF(
                 fullscan = 0
-                AND ijoinstmt = 0
-                AND explicittxn = 0,
+                AND ijoinstmt = 0,
                 1,
                 0
               )
@@ -76,8 +78,6 @@ query:
   UNION
   SELECT 'ijoinlio' as type,sum(lioperstmt * (IF(ijoincnt > 0, 1, 0))) AS rows FROM sql_distinct_cnt
   UNION
-  SELECT 'explicitlio' as type,sum(lioperstmt * (IF(explicitcnt > 0, 1, 0))) AS rows FROM sql_distinct_cnt
-  UNION
   SELECT 'healtylio' as type,sum(lioperstmt * (IF(healthycnt > 0, 1, 0))) AS rows FROM sql_distinct_cnt
   LIMIT 
     $1  

examples/table_mvcc.yaml

@@ -14,6 +14,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, reading system.namespace needs
+#   GRANT SYSTEM VIEWSYSTEMTABLE TO visus;
+
 # This could be long running and expensive query. 
 # It is a cluster wide query that will be executed only
 # by one visus process.

examples/table_mvcc_node.yaml

@@ -14,6 +14,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, reading system.namespace needs
+#   GRANT SYSTEM VIEWSYSTEMTABLE TO visus;
+
 # This query samples local ranges in a node.
 # It chooses ranges that have a replica that shows up as the first replica
 # in the crdb_internal.ranges_no_leases.

examples/tables.yaml

@@ -14,6 +14,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: beyond what "visus init" grants, the visus user needs SELECT on
+# the tables it reports on. CockroachDB v24.3 hides rows in crdb_internal for
+# tables the user cannot read; later versions do not filter.
+
 name: tables
 enabled: true
 scope: cluster

examples/tables_rows.yaml

@@ -14,6 +14,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+# Privileges: none beyond what "visus init" grants (VIEWACTIVITY).
+
 name: tables_rows
 enabled: true
 scope: cluster