diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ca79ca5..ad69632 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,8 @@ updates: directory: / schedule: interval: weekly + + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a3ea8e2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + fmt: + name: rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - run: cargo fmt --all --check + + clippy: + name: clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # sqllineage-python links against libpython at build time. + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + + - run: cargo clippy --workspace --all-targets --all-features -- -D warnings + + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - run: cargo test --workspace --all-features + + python: + name: Python smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - uses: PyO3/maturin-action@v1 + with: + maturin-version: "v1.9.4" + args: >- + --out dist + --manifest-path sqllineage-python/Cargo.toml + + # Mirrors the smoke test in release.yml so a broken binding fails on the + # pull request instead of on the release tag. + - name: Smoke test + run: | + pip install dist/*.whl + python -c " + import sqllineage + results = sqllineage.analyze('SELECT a FROM t') + assert len(results) == 1 + assert len(results[0].tables.inputs) > 0 + print('OK:', results[0]) + " diff --git a/README.md b/README.md index ae940eb..67ca057 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sqllineage +[![CI](https://github.com/funcpp/sqllineage/actions/workflows/ci.yml/badge.svg)](https://github.com/funcpp/sqllineage/actions/workflows/ci.yml) + Extract table-level and column-level data lineage from SQL statements. `sqllineage` parses SQL (via [sqlparser](https://crates.io/crates/sqlparser)) and produces a structured lineage result showing which tables are read/written and which source columns each output column derives from. diff --git a/sqllineage/src/build/expr.rs b/sqllineage/src/build/expr.rs index 2b2cb3e..34e4d4b 100644 --- a/sqllineage/src/build/expr.rs +++ b/sqllineage/src/build/expr.rs @@ -24,7 +24,10 @@ impl LineageBuilder { vec![node] } - Expr::Value(_) | Expr::TypedString { .. } | Expr::Wildcard(..) | Expr::QualifiedWildcard(..) => vec![], + Expr::Value(_) + | Expr::TypedString { .. } + | Expr::Wildcard(..) + | Expr::QualifiedWildcard(..) => vec![], Expr::Cast { expr, .. } | Expr::Nested(expr) @@ -49,7 +52,12 @@ impl LineageBuilder { Expr::Extract { expr, .. } => self.collect_ancestors(expr), - Expr::Trim { expr, trim_what, trim_characters, .. } => { + Expr::Trim { + expr, + trim_what, + trim_characters, + .. + } => { let mut v = self.collect_ancestors(expr); if let Some(what) = trim_what { v.extend(self.collect_ancestors(what)); @@ -62,7 +70,12 @@ impl LineageBuilder { v } - Expr::Substring { expr, substring_from, substring_for, .. } => { + Expr::Substring { + expr, + substring_from, + substring_for, + .. + } => { let mut v = self.collect_ancestors(expr); if let Some(from) = substring_from { v.extend(self.collect_ancestors(from)); @@ -73,7 +86,13 @@ impl LineageBuilder { v } - Expr::Overlay { expr, overlay_what, overlay_from, overlay_for, .. } => { + Expr::Overlay { + expr, + overlay_what, + overlay_from, + overlay_for, + .. + } => { let mut v = self.collect_ancestors(expr); v.extend(self.collect_ancestors(overlay_what)); v.extend(self.collect_ancestors(overlay_from)); @@ -89,17 +108,36 @@ impl LineageBuilder { v } - Expr::AtTimeZone { timestamp, time_zone } => { + Expr::AtTimeZone { + timestamp, + time_zone, + } => { let mut v = self.collect_ancestors(timestamp); v.extend(self.collect_ancestors(time_zone)); v } Expr::BinaryOp { left, right, .. } - | Expr::Like { expr: left, pattern: right, .. } - | Expr::ILike { expr: left, pattern: right, .. } - | Expr::SimilarTo { expr: left, pattern: right, .. } - | Expr::RLike { expr: left, pattern: right, .. } + | Expr::Like { + expr: left, + pattern: right, + .. + } + | Expr::ILike { + expr: left, + pattern: right, + .. + } + | Expr::SimilarTo { + expr: left, + pattern: right, + .. + } + | Expr::RLike { + expr: left, + pattern: right, + .. + } | Expr::IsDistinctFrom(left, right) | Expr::IsNotDistinctFrom(left, right) => { let mut v = self.collect_ancestors(left); @@ -113,7 +151,9 @@ impl LineageBuilder { v } - Expr::InUnnest { expr, array_expr, .. } => { + Expr::InUnnest { + expr, array_expr, .. + } => { let mut v = self.collect_ancestors(expr); v.extend(self.collect_ancestors(array_expr)); v @@ -192,7 +232,12 @@ impl LineageBuilder { ancestors } - Expr::Case { operand, conditions, else_result, .. } => { + Expr::Case { + operand, + conditions, + else_result, + .. + } => { let mut v = Vec::new(); if let Some(op) = operand { v.extend(self.collect_ancestors(op)); @@ -229,7 +274,9 @@ impl LineageBuilder { vec![] } - Expr::Between { expr, low, high, .. } => { + Expr::Between { + expr, low, high, .. + } => { let mut v = self.collect_ancestors(expr); v.extend(self.collect_ancestors(low)); v.extend(self.collect_ancestors(high)); @@ -251,7 +298,6 @@ impl LineageBuilder { | Expr::Interval(_) | Expr::Lambda(_) | Expr::MatchAgainst { .. } => vec![], - } } } diff --git a/sqllineage/src/build/statement.rs b/sqllineage/src/build/statement.rs index 684f052..e33de12 100644 --- a/sqllineage/src/build/statement.rs +++ b/sqllineage/src/build/statement.rs @@ -280,7 +280,7 @@ impl LineageBuilder { | Statement::UNCache { .. } | Statement::UNLISTEN { .. } | Statement::Unload { .. } - | Statement::UnlockTables { .. } + | Statement::UnlockTables | Statement::Use(_) | Statement::Vacuum { .. } | Statement::WaitFor { .. } diff --git a/sqllineage/src/resolve/mod.rs b/sqllineage/src/resolve/mod.rs index c544d0e..7a596c3 100644 --- a/sqllineage/src/resolve/mod.rs +++ b/sqllineage/src/resolve/mod.rs @@ -68,13 +68,21 @@ pub(crate) fn resolve( if has_back { mappings.push(ColumnMapping { - target: ColumnRef { table: output_table.clone(), column: name.clone() }, - sources: vec![ColumnOrigin::Recursive { base_sources: sources }], + target: ColumnRef { + table: output_table.clone(), + column: name.clone(), + }, + sources: vec![ColumnOrigin::Recursive { + base_sources: sources, + }], transform, }); } else { mappings.push(ColumnMapping { - target: ColumnRef { table: output_table.clone(), column: name.clone() }, + target: ColumnRef { + table: output_table.clone(), + column: name.clone(), + }, sources, transform, }); @@ -105,7 +113,12 @@ pub(crate) fn resolve( _ => None, }) .collect(); - mappings.sort_by_key(|m| name_order.get(&m.target.column).copied().unwrap_or(usize::MAX)); + mappings.sort_by_key(|m| { + name_order + .get(&m.target.column) + .copied() + .unwrap_or(usize::MAX) + }); if let Some(cat) = catalog { catalog::apply_catalog(&mut mappings, cat); @@ -156,7 +169,15 @@ fn expand_star( if let Some(t) = table { let binding = graph.scopes.lookup(scope, &t.table).cloned(); if let Some(Binding::Cte(s) | Binding::DerivedTable(s)) = binding { - expand_scope_columns(s, graph, resolved, incoming, output_table, mappings, visited_scopes); + expand_scope_columns( + s, + graph, + resolved, + incoming, + output_table, + mappings, + visited_scopes, + ); } else { mappings.push(wildcard_mapping(output_table, t.clone())); } @@ -165,12 +186,28 @@ fn expand_star( match binding { Binding::Table(tref) => mappings.push(wildcard_mapping(output_table, tref)), Binding::Cte(s) | Binding::DerivedTable(s) => { - expand_scope_columns(s, graph, resolved, incoming, output_table, mappings, visited_scopes); + expand_scope_columns( + s, + graph, + resolved, + incoming, + output_table, + mappings, + visited_scopes, + ); } } } for &child in graph.scopes.anonymous_derived(scope) { - expand_scope_columns(child, graph, resolved, incoming, output_table, mappings, visited_scopes); + expand_scope_columns( + child, + graph, + resolved, + incoming, + output_table, + mappings, + visited_scopes, + ); } } } @@ -190,7 +227,16 @@ fn expand_scope_columns( } for col in graph.scopes.output_columns(scope_id) { if let RawNode::Star { table, scope } = &graph.nodes[col.node_id] { - expand_star(table.as_ref(), *scope, graph, resolved, incoming, output_table, mappings, visited_scopes); + expand_star( + table.as_ref(), + *scope, + graph, + resolved, + incoming, + output_table, + mappings, + visited_scopes, + ); } else { let mut visited = HashSet::new(); let (sources, edge_kinds, _) = @@ -374,7 +420,14 @@ fn resolve_unqualified( incoming: &[Vec], visited: &mut HashSet, ) -> Option { - resolve_from_bindings(name, &effective_bindings(scope, graph), graph, resolved, incoming, visited) + resolve_from_bindings( + name, + &effective_bindings(scope, graph), + graph, + resolved, + incoming, + visited, + ) } fn resolve_from_bindings( @@ -406,7 +459,12 @@ fn resolve_from_bindings( for (_, binding) in bindings { match binding { Binding::Cte(s) | Binding::DerivedTable(s) => { - if graph.scopes.output_columns(*s).iter().any(|c| c.name == name) { + if graph + .scopes + .output_columns(*s) + .iter() + .any(|c| c.name == name) + { return resolve_through_scope(name, *s, graph, resolved, incoming, visited); } }