From 6334c29ec14cd53cf8ef0e7968738a0437a4c5ef Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:39:09 -0500 Subject: [PATCH 1/2] Q2 FY26: Apply automated update. --- .buildkite/pipeline.yml | 10 +-- .../maintainer_pull_request_template.md | 55 ++++++++------ .github/workflows/auto-release.yml | 13 ++-- .github/workflows/generate-docs.yml | 13 ++++ .gitignore | 73 ++++++++++++++++++- CHANGELOG.md | 19 +++++ LICENSE | 2 +- README.md | 8 +- integration_tests/ci/sample.profiles.yml | 4 - integration_tests/dbt_project.yml | 7 +- integration_tests/requirements.txt | 3 +- 11 files changed, 159 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/generate-docs.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 552f698..9387b23 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -18,7 +18,7 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -35,7 +35,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -47,7 +47,7 @@ steps: key: "run_dbt_redshift" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -62,7 +62,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 3220674..0899dc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -1,29 +1,38 @@ -## PR Overview -**This PR will address the following Issue/Feature:** + +- `dbt run --full-refresh && dbt test` +- `dbt run` && `dbt test` (if incremental models are present) +- The related issue is linked, tagged, and appropriately assigned +- Documentation and version updates are included, if applicable +- `docs` have been regenerated (unless there are no code or YAML changes) +- BuildKite integration tests are passing +--> -**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:** - +## PR Overview +**Package version introduced in this PR:** +- -## PR Checklist -### Basic Validation -Please acknowledge that you have successfully performed the following commands locally: -- [ ] dbt run –full-refresh && dbt test -- [ ] dbt run (if incremental models are present) && dbt test +**This PR addresses the following Issue/Feature(s):** + +- -Before marking this PR as "ready for review" the following have been applied: -- [ ] The appropriate issue has been linked, tagged, and properly assigned -- [ ] All necessary documentation and version upgrades have been applied -- [ ] docs were regenerated (unless this PR does not include any code or yml updates) -- [ ] BuildKite integration tests are passing -- [ ] Detailed validation steps have been provided below +**Summary of changes:** + +- -### Detailed Validation -Please share any and all of your validation steps: - +### Submission Checklist +- [ ] Alignment meeting with the reviewer (if needed) + - [ ] Timeline and validation requirements discussed +- [ ] Provide validation details: + - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests) + - [ ] **Testing Instructions:** Confirm the change addresses the issue(s) + - [ ] **Focus Areas:** Complex logic or queries that need extra attention +- [ ] Merge any relevant open PRs into this PR -### If you had to summarize this PR in an emoji, which would it be? - -:dancer: \ No newline at end of file +### Changelog + +- [ ] Draft changelog for PR +- [ ] Final changelog for release review diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 8ed5853..76d6944 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,11 +3,12 @@ on: pull_request: types: - closed - branches: - - main + - labeled jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file + release: + if: | + (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + github.event.label.name == 'pre-release' + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + secrets: inherit diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..ed7b5a7 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,13 @@ +name: 'generate dbt docs' +on: + pull_request: + types: + - labeled + +jobs: + generate-docs: + if: github.event.label.name == 'docs:ready' + uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main + secrets: inherit + with: + schema_var_name: pardot_schema \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2710741..06f4e77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,73 @@ - -target/ +# dbt +**/package-lock.yml +package-lock.yml +.dbt/ dbt_modules/ +dbt_packages/ logs/ +profiles.yml +target/ +*.log + +# IDE files +.idea/ +.vscode/ +*~ +*.swp +*.swo + +# Jupyter Notebook +.ipynb_checkpoints + +# OS generated files +**/.DS_Store +.DS_Store +.Spotlight-V100 +.Trashes +._* +Thumbs.db +ehthumbs.db + +# Python +*.egg +*.egg-info/ +*.py[cod] +*.so +*$py.class +.Python +__pycache__/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.env +.installed.cfg +lib/ +lib64/ +MANIFEST +parts/ +sdist/ +var/ +wheels/ + +# Secrets and credentials +.env.* +.secrets +credentials.json +service-account.json + +# Temporary files +.cache/ +*.temp +*.tmp -dbt_packages/ \ No newline at end of file +# Virtual environments +.conda/ +.env +.venv +ENV/ +env/ +env.bak/ +venv/ +venv.bak/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a381ea..27e730e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +[PR #20](https://github.com/fivetran/dbt_pardot_source/pull/20) includes the following updates: + +### Under the Hood - July 2025 Updates + +- Updated conditions in `.github/workflows/auto-release.yml`. +- Added `.github/workflows/generate-docs.yml`. +- Added `+docs: show: False` to `integration_tests/dbt_project.yml`. +- Migrated `flags` (e.g., `send_anonymous_usage_stats`, `use_colors`) from `sample.profiles.yml` to `integration_tests/dbt_project.yml`. +- Updated `maintainer_pull_request_template.md` with improved checklist. +- Refreshed README tag block: + - Standardized Quickstart-compatible badge set + - Left-aligned and positioned below the H1 title. +- Updated Python image version to `3.10.13` in `pipeline.yml`. +- Added `CI_DATABRICKS_DBT_CATALOG` to: + - `.buildkite/hooks/pre-command` (as an export) + - `pipeline.yml` (under the `environment` block, after `CI_DATABRICKS_DBT_TOKEN`) +- Added `certifi==2025.1.31` to `requirements.txt` (if missing). +- Updated `.gitignore` to exclude additional DBT, Python, and system artifacts. + # dbt_pardot_source version.version ## Documentation diff --git a/LICENSE b/LICENSE index 261eeb9..f9554a9 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright © 2025 Fivetran Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 85d8c3c..7a7e6d9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ href="https://github.com/fivetran/dbt_pardot_source/blob/main/LICENSE"> - + @@ -67,7 +67,7 @@ vars: #### Change the source table references If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_pardot_source/blob/main/dbt_project.yml) variable declarations to see the expected names. - + ```yml vars: pardot__identifier: your_table_name @@ -77,14 +77,14 @@ vars: ### (Optional) Step 5: Orchestrate your models with Fivetran Transformations for dbt Core™
Expand to view details
- + Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core™ setup guides](https://fivetran.com/docs/transformations/dbt#setupguide).
## Does this package have dependencies? This dbt package is dependent on the following dbt packages. These dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. - + ```yml packages: - package: fivetran/fivetran_utils diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 3724f3c..3fb8c9d 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,10 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True - integration_tests: target: redshift outputs: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 7c0bab9..a56cfe1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -14,8 +14,13 @@ vars: pardot_opportunity_identifier: "pardot_opportunity_data" pardot_opportunity_prospect_identifier: "pardot_opportunity_prospect_data" seeds: + +docs: + show: False +quote_columns: "{{ true if target.type == 'redshift' else false }}" dispatch: - macro_namespace: dbt_utils - search_order: ['spark_utils', 'dbt_utils'] \ No newline at end of file + search_order: ['spark_utils', 'dbt_utils'] +flags: + send_anonymous_usage_stats: False + use_colors: True \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..c9f6024 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 +certifi==2025.1.31 From 766bec46814efbe4893c0cdb6196897f6d5485a4 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:59:51 -0500 Subject: [PATCH 2/2] Q2 FY26: Update auto-release workflow only. --- .github/workflows/auto-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 76d6944..e0a7100 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -10,5 +10,5 @@ jobs: if: | (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || github.event.label.name == 'pre-release' - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main secrets: inherit