feature/freshness-updates (#144)

* feature/freshness-updates

* pr ref fix

Author: fivetran-joemarkiewicz

.buildkite/pipeline.yml

@@ -3,7 +3,7 @@ steps:
     key: "run-dbt-postgres"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -18,7 +18,7 @@ steps:
     key: "run_dbt_snowflake"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -35,7 +35,7 @@ steps:
     key: "run_dbt_bigquery"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -47,7 +47,7 @@ steps:
     key: "run_dbt_redshift"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -62,7 +62,7 @@ steps:
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -77,7 +77,7 @@ steps:
     key: "run_dbt_databricks_sql"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"

.buildkite/scripts/run_models.sh

@@ -18,6 +18,7 @@ cd integration_tests
 dbt deps
 if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db" --full-refresh
+dbt source freshness --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db" || echo "...Only verifying freshness runs..."
 dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db"
 dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db" --full-refresh
 dbt test --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db"
@@ -26,6 +27,7 @@ dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_8, hubspot_marketing_enabled
 dbt test --vars '{hubspot_schema: hubspot_sqlw_tests_8}' --target "$db"
 else
 dbt seed --target "$db" --full-refresh
+dbt source freshness --target "$db" || echo "...Only verifying freshness runs..."
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"            
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db"

.gitignore

@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 .DS_Store
-dbt_packages/
+dbt_packages/
+env/

CHANGELOG.md

@@ -1,3 +1,30 @@
+# dbt_hubspot_source v0.20.0
+
+[PR #144](https://github.com/fivetran/dbt_hubspot_source/pull/144) includes the following updates:
+
+## Breaking Change for dbt Core < 1.9.6
+
+> *Note: This is not relevant to Fivetran Quickstart users.*
+
+Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core. This will resolve the following deprecation warning that users running dbt >= 1.9.6 may have received:
+
+```
+[WARNING]: Deprecated functionality
+Found `freshness` as a top-level property of `hubspot` in file
+`models/src_hubspot.yml`. The `freshness` top-level property should be moved
+into the `config` of `hubspot`.
+```
+
+**IMPORTANT:** Users running dbt Core < 1.9.6 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests.
+
+If you are using dbt Core < 1.9.6 and want to continue running HubSpot freshness tests, please elect **one** of the following options:
+  1. (Recommended) Upgrade to dbt Core >= 1.9.6
+  2. Do not upgrade your installed version of the `hubspot_source` package. Pin your dependency on v0.19.0 in your `packages.yml` file.
+  3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `hubspot` source and apply freshness via the previous release top-level property route. This will require you to copy and paste the entirety of the previous release `src_hubspot.yml` file and add an `overrides: hubspot_source` property.
+
+## Under the Hood
+- Updates to ensure integration tests use latest version of dbt.
+
 # dbt_hubspot_source v0.19.0
 [PR #142](https://github.com/fivetran/dbt_hubspot_source/pull/142) includes the following updates:
 

README.md

@@ -18,7 +18,7 @@
 <!--section="hubspot_source_model"-->
 - Materializes [HubSpot staging tables](https://fivetran.github.io/dbt_hubspot_source/#!/overview/hubspot_source/models/?g_v=1) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/hubspot#schemainformation). These staging tables clean, test, and prepare your HubSpot data from [Fivetran's connector](https://fivetran.com/docs/applications/hubspot) for analysis by doing the following:
   - Name columns for consistency across all packages and for easier analysis
-  - Adds freshness tests to source data
+  - Adds freshness tests to source data. dbt Core >= 1.9.6 is required to run freshness tests out of the box.
   - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values.
 - Generates a comprehensive data dictionary of your HubSpot data through the [dbt docs site](https://fivetran.github.io/dbt_hubspot_source/).
 - These tables are designed to work simultaneously with our [HubSpot transformation package](https://github.com/fivetran/dbt_hubspot).

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.19.0'
+version: '0.20.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -2,10 +2,6 @@
 # HEY! This file is used in the dbt package integrations tests with Buildkite.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
-config:
-    send_anonymous_usage_stats: False
-    use_colors: True
-
 integration_tests:
   target: redshift
   outputs:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source_integration_tests'
-version: '0.19.0'
+version: '0.20.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -250,4 +250,7 @@ seeds:
 
 dispatch:
   - macro_namespace: dbt_utils
-    search_order: ['spark_utils', 'dbt_utils']
+    search_order: ['spark_utils', 'dbt_utils']
+
+flags:
+  send_anonymous_usage_stats: False

integration_tests/requirements.txt

@@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+certifi==2025.1.31

models/src_hubspot.yml

@@ -150,12 +150,12 @@ sources:
 
       - name: contact
         identifier: "{{ var('hubspot_contact_identifier', 'contact')}}"
-        freshness:
-          warn_after: {count: 84, period: hour}
-          error_after: {count: 168, period: hour}
         description: Each record represents a contact in Hubspot.
         config:
           enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_enabled', true) }}"
+          freshness:
+            warn_after: {count: 84, period: hour}
+            error_after: {count: 168, period: hour}
         columns:
           - name: _fivetran_deleted
             description: '{{ doc("_fivetran_deleted") }}'
@@ -301,12 +301,12 @@ sources:
 
       - name: contact_property_history
         identifier: "{{ var('hubspot_contact_property_history_identifier', 'contact_property_history')}}"
-        freshness:
-          warn_after: {count: 84, period: hour}
-          error_after: {count: 168, period: hour}
         description: Each record represents a change to contact record in Hubspot.
         config:
           enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_property_enabled', true) and var('hubspot_contact_property_history_enabled', true) }}"
+          freshness:
+            warn_after: {count: 84, period: hour}
+            error_after: {count: 168, period: hour}
         columns:
           - name: _fivetran_synced
             description: '{{ doc("_fivetran_synced") }}'
@@ -515,12 +515,12 @@ sources:
 
       - name: email_event
         identifier: "{{ var('hubspot_email_event_identifier', 'email_event')}}"
-        freshness:
-          warn_after: {count: 84, period: hour}
-          error_after: {count: 168, period: hour}
         description: Each record represents an email event in Hubspot.
         config:
           enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) }}"
+          freshness:
+            warn_after: {count: 84, period: hour}
+            error_after: {count: 168, period: hour}
         columns:
           - name: _fivetran_synced
             description: '{{ doc("_fivetran_synced") }}'