Ele 5122 cli stop using deprecated tests (#2022)

* add the former deprecated tests project from the package to the CLI as the e2e project

* e2e project models: remove stage ifs

* remove irrelevant macros

* test-warehouse: change CI to use a static dbt project rather than the deprecated package tests

* fixes

* bugfix

* add deps

* test-warehouse: fix packages for e2e project

* forgot to update packages

* generate data before seeding

* full refresh seeds

Author: haritamar

.github/workflows/test-warehouse.yml

@@ -26,16 +26,6 @@ on:
         type: string
         required: false
         description: dbt's version to test with
-      should-run-tests:
-        type: boolean
-        required: false
-        default: true
-        description: Whether to run E2E tests
-      clear-tests:
-        type: boolean
-        required: false
-        default: true
-        description: Whether to clean test environment
       generate-data:
         type: boolean
         required: false
@@ -47,10 +37,6 @@ on:
       warehouse-type:
         type: string
         required: true
-      should-run-tests:
-        type: boolean
-        required: false
-        default: true
       elementary-ref:
         type: string
         required: false
@@ -60,19 +46,16 @@ on:
       dbt-version:
         type: string
         required: false
-      clear-tests:
-        type: boolean
-        required: false
-        default: true
       generate-data:
         type: boolean
         required: false
         default: false
 
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-  DBT_PKG_INTEG_TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests/deprecated_tests
-  ELMENTARY_INTERNAL_DBT_PKG_DIR: ${{ github.workspace }}/elementary/elementary/monitor/dbt_project
+  ELEMENTARY_DBT_PACKAGE_PATH: ${{ github.workspace }}/dbt-data-reliability
+  CLI_INTERNAL_DBT_PKG_DIR: ${{ github.workspace }}/elementary/elementary/monitor/dbt_project
+  E2E_DBT_PROJECT_DIR: ${{ github.workspace }}/elementary/tests/e2e_dbt_project
 
 jobs:
   # PRs from forks require approval, specifically with the "pull_request_target" event as it contains repo secrets.
@@ -117,12 +100,12 @@ jobs:
 
       - name: Start Postgres
         if: inputs.warehouse-type == 'postgres'
-        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         run: docker compose up -d postgres
 
       # - name: Start Clickhouse
       #   if: inputs.warehouse-type == 'clickhouse'
-      #   working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+      #   working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
       #   run: docker compose up -d clickhouse
 
       - name: Setup Python
@@ -170,12 +153,46 @@ jobs:
           rm -rf "$DBT_PKGS_PATH/elementary"
           ln -vs "$GITHUB_WORKSPACE/dbt-data-reliability" "$DBT_PKGS_PATH/elementary"
 
-      - name: Run dbt package integration tests
-        if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
-        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+      - name: Run deps for E2E dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        env:
+          ELEMENTARY_DBT_PACKAGE_PATH: ${{ env.ELEMENTARY_DBT_PACKAGE_PATH }}
         run: |
           dbt deps
-          python run_e2e_tests.py -t "${{ inputs.warehouse-type }}" -g "${{ inputs.warehouse-type == 'postgres' || inputs.generate-data }}" --clear-tests "${{ inputs.clear-tests }}"
+
+      - name: Seed e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        if: inputs.warehouse-type == 'postgres' || inputs.generate-data
+        run: |
+          python generate_data.py
+          dbt seed -f --target "${{ inputs.warehouse-type }}"
+
+      - name: Run e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        run: |
+          dbt run --target "${{ inputs.warehouse-type }}" || true
+
+          # Validate run_results.json: only error_model should be non-success
+          jq -e '
+            [.results[] | select(.status != "success") | .unique_id]
+            | length == 1 and .[0] == "model.elementary_integration_tests.error_model"
+          ' target/run_results.json > /dev/null
+          jq_exit=$?
+
+          if [ $jq_exit -eq 0 ]; then
+            echo "✅ Validation passed: only error_model failed."
+          else
+            echo "❌ Validation failed. Unexpected failures:"
+            jq '[.results[] | select(.status != "success") | .unique_id] | join(", ")' target/run_results.json
+          fi
+
+          exit $jq_exit
+
+      - name: Test e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        continue-on-error: true
+        run: |
+          dbt test --target "${{ inputs.warehouse-type }}"
 
       - name: Run help
         run: edr --help
@@ -187,12 +204,12 @@ jobs:
           edr monitor
           -t "${{ inputs.warehouse-type }}"
           --group-by table
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
           --slack-webhook "$SLACK_WEBHOOK"
 
       - name: Validate alerts statuses were updated
-        working-directory: ${{ env.ELMENTARY_INTERNAL_DBT_PKG_DIR }}
+        working-directory: ${{ env.CLI_INTERNAL_DBT_PKG_DIR }}
         run: |
           dbt deps
           dbt run-operation validate_alert_statuses_are_updated -t "${{ inputs.warehouse-type }}"
@@ -201,7 +218,7 @@ jobs:
         run: >
           edr monitor report
           -t "${{ inputs.warehouse-type }}"
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
 
       - name: Set report artifact name
@@ -230,7 +247,7 @@ jobs:
         run: >
           edr monitor send-report
           -t "${{ inputs.warehouse-type }}"
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
           --slack-file-name "report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
           --slack-token "$SLACK_TOKEN"
@@ -259,5 +276,4 @@ jobs:
           path: elementary/edr_target/edr.log
 
       - name: Run Python package e2e tests
-        if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
         run: pytest -vv tests/e2e --warehouse-type ${{ inputs.warehouse-type }}

.gitignore

@@ -96,4 +96,4 @@ venv/
 
 # elementary outputs
 edr_target/
-tests/tests_with_db/dbt_project/dbt_packages/
+**/dbt_packages/

tests/e2e_dbt_project/README.md

@@ -0,0 +1,2 @@
+This dbt project is intended for our CLI e2e.
+The "edr report" and "edr monitor" commands in `test-warehouse.yml` will run on the data generated by this project.

tests/e2e_dbt_project/data/training/any_type_column_anomalies_training.csv

@@ -0,0 +1,5 @@
+group_a,group_b,group_c,group_d
+Poland,Netherlands,Spain,Ukraine
+Greece,Denmark,Italy,Sweden
+Russia,Germany,Ireland,France
+Czech Republic,Portugal,Croatia,England

tests/e2e_dbt_project/data/training/backfill_days_column_anomalies_training.csv

@@ -0,0 +1,180 @@
+Player,Team,Goals,shots_on_target,Passes,Shots_without_scoring,Crosses,Tackles,Interceptions,Goals_conceded,Saves_made,Offsides,Yellow_cards
+Darijo Srna,Croatia,,,,,16,13,,,,,
+Gordon Schildenfeld,Croatia,,,,,,,,,,,
+Ivan Perisic,Croatia,,,,,,,,,,,
+Ivan Rakitic,Croatia,,,,,,,,,,,
+Ivan Strinic,Croatia,,,,,14,,,,,,
+Luka Modric,Croatia,,,,,,,,,,,
+Mario Mandzukic,Croatia,3,4,,,,,,,,,
+Nikica Jelavic,Croatia,,,,,,,,,,,
+Stipe Pletikosa,Croatia,,,,,,,,3,13,,
+Vedran Corluka,Croatia,,,,,,,,,,,
+David Limbersky,Czech Republic,,,,,,,,,,,2
+Jaroslav Plasil,Czech Republic,,,,,,,,,,,
+Michal Kadlec,Czech Republic,,,,,,,,,,,
+Milan Baros,Czech Republic,,,,,,,,,,3,
+Petr Cech,Czech Republic,,,,,,,,6,9,,
+Petr Jiracek,Czech Republic,2,,,,,,,,,,
+Theodor Gebre Selassie,Czech Republic,,,,,,,,,,,
+Tomás Hübschman,Czech Republic,,,,,,,12,,,,
+Tomas Pekhart,Czech Republic,,,,,,,,,,,
+Tomas Rosicky,Czech Republic,,,,,,,,,,,
+Tomas Sivok,Czech Republic,,,,,,,,,,,
+Vaclav Pilar,Czech Republic,2,4,,,,,,,,,
+Daniel Agger,Denmark,,,,,,,21,,,,
+Dennis Rommedahl,Denmark,,,,,,,,,,3,
+Jakob Poulsen,Denmark,,,,,,,,,,,
+Michael Krohn-Dehli,Denmark,2,,,,,,,,,,
+Nicklas Bendtner,Denmark,2,,,,,,,,,,
+Niki Zimling,Denmark,,,,,,,,,,,
+Simon Kjaer,Denmark,,,,,,,,,,,
+Stephan Andersen,Denmark,,,,,,,,5,10,,
+Tobias Mikkelsen,Denmark,,,,,,,,,,,
+William Kvist,Denmark,,,,,,,12,,,,
+Ashley Young,England,,,,,,,,,,,
+Danny Welbeck,England,,,,,,,,,,,
+Glen Johnson,England,,,,,,,,,,,
+James Milner,England,,,,,17,,,,,,
+Joe Hart,England,,,,,,,,3,22,,
+John Terry,England,,,,,,,14,,,,
+Joleon Lescott,England,,,,,,,,,,,
+Scott Parker,England,,,,,,13,,,,,
+Steven Gerrard,England,,,,,,18,,,,,
+Theo Walcott,England,,,,,,,,,,,
+Wayne Rooney,England,,,,,,,,,,,
+Alou Diarra,France,,,,,,15,,,,,
+Franck Ribéry,France,,,,6,14,,,,,,
+Hugo Lloris,France,,,,,,,,5,6,,
+Jérémy Menez,France,,4,,,,,,,,,2
+Karim Benzema,France,,6,,19,,,,,,3,
+Mathieu Debuchy,France,,,,,,,,,,,
+Olivier Giroud,France,,,,,,,,,,,
+Patrice Evra,France,,,,,,,,,,,
+Philippe Mexes,France,,,,,,,,,,,2
+Samir Nasri,France,,,,,,,,,,,
+Yann M'Vila,France,,,,,,,,,,,
+Yohan Cabaye,France,,,,,,,,,,,
+Andre Schürrle,Germany,,,,7,,,,,,,
+Bastian Schweinsteiger,Germany,,,415,,,13,,,,,
+Holger Badstuber,Germany,,,,,,,,,,,
+Jerome Boateng,Germany,,,,,17,,,,,,
+Manuel Neuer,Germany,,,,,,,,6,10,,
+Marco Reus,Germany,,4,,,,,,,,,
+Mario Gomez,Germany,3,,,,,,,,,3,
+Mats Hummels,Germany,,,,,,18,19,,,,
+Mesut Özil,Germany,,5,319,,17,,,,,,
+Miroslav Klose,Germany,,,,,,,,,,5,
+Philipp Lahm,Germany,,,316,,,,,,,,
+Sami Khedira,Germany,,,310,,,,,,,,
+Thomas Müller,Germany,,,,,14,,,,,,
+Toni Kroos,Germany,,,,,,,,,,,
+Dimitris Salpingidis,Greece,2,,,,,,,,,5,
+Georgios Samaras,Greece,,,,,,,,,,,
+Georgios Tzavellas,Greece,,,,,,,,,,,
+Giorgios Karagounis,Greece,,,,,,,,,,,2
+Jose Holebas,Greece,,,,,,,,,,,2
+Konstantinos Chalkias,Greece,,,,,,,,3,,,
+Konstantinos Katsouranis,Greece,,,,,,,12,,,,
+Kyriakos Papadopoulos,Greece,,,,,,,,,,,
+Michalis Sifakis,Greece,,,,,,,,4,11,,
+Sokratis Papastathopoulos,Greece,,,,,,,,,,,
+Theofanis Gekas,Greece,,,,,,,,,,3,
+Vassilas Torosidis,Greece,,,,,,,12,,,,
+Alessandro Diamanti,Italy,,,,8,,,,,,,
+Andrea Pirlo,Italy,,,423,,,,13,,,,
+Antonio Cassano,Italy,,7,,,14,,,,,3,
+Antonio Di Natale,Italy,,,,,,,,,,6,
+Christian Maggio,Italy,,,,,,,,,,,2
+Claudio Marchisio,Italy,,4,,11,,14,,,,,
+Daniele De Rossi,Italy,,,345,9,,,22,,,,
+Gianluigi Buffon,Italy,,,,,,,,7,20,,
+Giorgio Chiellini,Italy,,,,,,,16,,,,
+Leonardo Bonucci,Italy,,,,,,,15,,,,
+Mario Balotelli,Italy,3,10,,,,,,,,,
+Riccardo Montolivo,Italy,,,,7,,,13,,,,
+Thiago Motta,Italy,,,,,,,,,,,2
+Arjen Robben,Netherlands,,,,11,,,,,,,
+Ibrahim Afellay,Netherlands,,,,,,,,,,,
+Jetro Willems,Netherlands,,,,,,,,,,,2
+Johnny Heitinga,Netherlands,,,,,,,,,,,
+Joris Mathijsen,Netherlands,,,,,,,,,,,
+Maarten Stekelenburg,Netherlands,,,,,,,,5,12,,
+Mark van Bommel,Netherlands,,,,,,,,,,,
+Nigel de Jong,Netherlands,,,,,,,,,,,
+Rafael van der Vaart,Netherlands,,,,,,,,,,,
+Robin van Persie,Netherlands,,5,,,,,,,,,
+Ron Vlaar,Netherlands,,,,,,,,,,,
+Wesley Sneijder,Netherlands,,,,8,17,,,,,,
+Damien Perquis,Poland,,,,,,,,,,,
+Eugen Polanski,Poland,,,,,,16,,,,,2
+Jakub Blaszczykowski,Poland,,,,,,,,,,,
+Ludovic Obraniak,Poland,,,,,19,,,,,,
+Marcin Wasilewski,Poland,,,,,,,,,,,
+Przemyslaw Tyton,Poland,,,,,,,,,6,,
+Robert Lewandowski,Poland,,,,,,,,,,,
+Wojciech Szczesny,Poland,,,,,,,,,,,
+Bruno Alves,Portugal,,,,,,,,,,,
+Cristiano Ronaldo,Portugal,3,9,,,,,,,,,
+Fábio Coentrão,Portugal,,,,,18,14,,,,,2
+Helder Postiga,Portugal,,,,,,,,,,3,
+Hugo Almeida,Portugal,,,,,,,,,,5,
+João Moutinho,Portugal,,,,,,,15,,,,
+João Pereira,Portugal,,,,,,17,,,,,2
+Miguel Veloso,Portugal,,,,,,,,,,,2
+Nani,Portugal,,,,9,19,,,,,,
+Nélson Oliveira,Portugal,,,,,,,,,,,
+Pepe,Portugal,,,,,,,16,,,,
+Raul Meireles,Portugal,,,,,,,11,,,,
+Rolando,Portugal,,,,,,,,,,,
+Rui Patricio,Portugal,,,,,,,,4,10,,
+Silvestre Varela,Portugal,,,,,,,,,,,
+Damien Duff,Republic of Ireland,,,,,16,,,,,,
+Glenn Whelan,Republic of Ireland,,,,,,,,,,,
+Jonathan Walters,Republic of Ireland,,,,,,,,,,,
+Keiren Westwood,Republic of Ireland,,,,,,,,2,,,
+Keith Andrews,Republic of Ireland,,,,12,,,,,,,
+Richard Dunne,Republic of Ireland,,,,,,,,,,,
+Robbie Keane,Republic of Ireland,,,,,,,,,,7,
+Sean St Ledger,Republic of Ireland,,,,,,,,,,,2
+Shay Given,Republic of Ireland,,,,,,,,9,17,,
+Alan Dzagoev,Russia,3,,,,,,,,,,
+Alexander Kerzhakov,Russia,,,,14,,,,,,,
+Alexei Berezoutski,Russia,,,,,,,,,,,
+Andrey Arshavin,Russia,,,,,15,,,,,,
+Igor Denisov,Russia,,,303,,,,,,,,
+Konstantin Zyryanov,Russia,,,,,,,,,,,
+Roman Pavlyuchenko,Russia,,,,,,,,,,,
+Roman Shirokov,Russia,,,,,,,,,,,
+Sergei Ignashevitch,Russia,,,,,,,,,,,
+Vyacheslav Malafeev,Russia,,,,,,,,3,10,,
+Yuri Zhirkov,Russia,,,,,,,,,,,
+Álvaro Arbeloa,Spain,,,319,,,20,,,,3,
+Andrés Iniesta,Spain,,9,446,20,,,,,,,
+David Silva,Spain,2,8,283,,,,,,,3,
+Fernando Torres,Spain,3,5,,,,,,,,5,
+Francesc Fábregas,Spain,2,,,,,,,,,,
+Gerard Piqué,Spain,,,312,,,19,,,,,
+Iker Casillas,Spain,,,,,,,,,15,,
+Jesús Navas,Spain,,,,,16,,,,,,
+Jordi Alba,Spain,,,410,,,,,,,,
+Pedro,Spain,,,,,,,,,,,
+Sergio Busquets,Spain,,,455,,,16,18,,,,
+Sergio Ramos,Spain,,,394,7,,16,,,,,2
+Xabi Alonso,Spain,2,,585,,,16,,,,,
+Xavi,Spain,,,592,12,13,,,,,,
+Anders Svensson,Sweden,,,,,,,,,,,2
+Andreas Isaksson,Sweden,,,,,,,,5,8,,
+Christian Wilhelmsson,Sweden,,,,,,,,,,,
+Kim Källström,Sweden,,,,,,,,,,,
+Ola Toivonen,Sweden,,,,,,,,,,3,
+Olof Mellberg,Sweden,,,,,,,,,,,
+Samuel Holmén,Sweden,,,,,,,,,,,
+Sebastian Larsson,Sweden,,4,,,,,,,,,
+Zlatan Ibrahimovic,Sweden,2,7,,,,,,,,,
+Anatoliy Tymoshchuk,Ukraine,,,,,,14,,,,,2
+Andriy Pyatov,Ukraine,,,,,,,,4,13,,
+Andriy Shevchenko,Ukraine,2,,,,,,,,,,
+Andriy Yarmolenko,Ukraine,,,,,,,,,,,
+Yevgen Selin,Ukraine,,,,,,,,,,,
+Yevhen Khacheridi,Ukraine,,,,,,,,,,,
+Yevhen Konoplyanka,Ukraine,,,,13,,,,,,,

tests/e2e_dbt_project/data/training/dimension_anomalies_training.csv

@@ -0,0 +1,17 @@
+Team,Goals,Shots_on_target,Shots_off_target,Total_shots,Hit_Woodwork,Penalty_goals,Penalties_not_scored,Headed_goals,Passes,Passes_completed,Touches,Crosses,Dribbles,Corners_Taken,Tackles,Clearances,Interceptions,Clearances_off_line,Clean_Sheets,Blocks,Goals_conceded,Saves_made,Fouls_Won,Fouls_Conceded,Offsides,Yellow_Cards,Red_Cards,Subs_on,Subs_off,Players_Used
+Croatia,4,13,12,32,0,0,0,2,1076,828,1706,60,42,14,49,83,56,,0,10,3,13,41,62,2,9,0,9,9,16
+Czech Republic,4,13,18,39,0,0,0,0,1565,1223,2358,46,68,21,62,98,37,2,1,10,6,9,53,73,8,7,0,11,11,19
+Denmark,4,10,10,27,1,0,0,3,1298,1082,1873,43,32,16,40,61,59,0,1,10,5,10,25,38,8,4,0,7,7,15
+England,5,11,18,40,0,0,0,3,1488,1200,2440,58,60,16,86,106,72,1,2,29,3,22,43,45,6,5,0,11,11,16
+France,3,22,24,65,1,0,0,0,2066,1803,2909,55,76,28,71,76,58,0,1,7,5,6,36,51,5,6,0,11,11,19
+Germany,10,32,32,80,2,1,0,2,2774,2427,3761,101,60,35,91,73,69,0,1,11,6,10,63,49,12,4,0,15,15,17
+Greece,5,8,18,32,1,1,1,0,1187,911,2016,52,53,10,65,123,87,0,1,23,7,13,67,48,12,9,1,12,12,20
+Italy,6,34,45,110,2,0,0,2,3016,2531,4363,75,75,30,98,137,136,1,2,18,7,20,101,89,16,16,0,18,18,19
+Netherlands,2,12,36,60,2,0,0,0,1556,1381,2163,50,49,22,34,41,41,0,0,9,5,12,35,30,3,5,0,7,7,15
+Poland,2,15,23,48,0,0,0,1,1059,852,1724,55,39,14,67,87,62,0,0,8,3,6,48,56,3,7,1,7,7,17
+Portugal,6,22,42,82,6,0,0,2,1891,1461,2958,91,64,41,78,92,86,0,2,11,4,10,73,90,10,12,0,14,14,16
+Republic of Ireland,1,7,12,28,0,0,0,1,851,606,1433,43,18,8,45,78,43,1,0,23,9,17,43,51,11,6,1,10,10,17
+Russia,5,9,31,59,2,0,0,1,1602,1345,2278,40,40,21,65,74,58,0,0,8,3,10,34,43,4,6,0,7,7,16
+Spain,12,42,33,100,0,1,0,2,4317,3820,5585,69,106,44,122,102,79,0,5,8,1,15,102,83,19,11,0,17,17,18
+Sweden,5,17,19,39,3,0,0,1,1192,965,1806,44,29,7,56,54,45,0,1,12,5,8,35,51,7,7,0,9,9,18
+Ukraine,2,7,26,38,0,0,0,2,1276,1043,1894,33,26,18,65,97,29,0,0,4,4,13,48,31,4,5,0,9,9,18