diff --git a/Makefile b/Makefile index c889f4e..3b29234 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,15 @@ deploy: ## Deploy the entire stack make dbt @echo "Deployment completed successfully. Run 'make app' to view the app." +debug: ## Clean up everything and redeploy to fix bugs + @echo "Running debug cleanup and redeploy..." + snow sql -c localstack -q "USE DATABASE FACTORY_PIPELINE_DEMO; USE SCHEMA PUBLIC; DROP FILE FORMAT IF EXISTS csv_format;" + snow sql -c localstack -q "DROP PIPE IF EXISTS FACTORY_PIPELINE_DEMO.PUBLIC.SENSOR_DATA_PIPE;" + snow sql -c localstack -q "DROP TABLE IF EXISTS FACTORY_PIPELINE_DEMO.PUBLIC_RAW.SENSOR_DATA;" + awslocal s3 rb s3://factory-sensor-data-local --force + make deploy + @echo "Debug cleanup and redeploy completed successfully." + test: ## Run tests @echo "Running tests..." bash -c "source env/bin/activate && pytest tests/" @@ -90,4 +99,4 @@ ready: ## Make sure the LocalStack container is up logs: ## Save the logs in a separate file @localstack logs > logs.txt -.PHONY: install seed aws upload pipeline dbt app deploy test start stop ready logs +.PHONY: install seed aws upload pipeline dbt app deploy test start stop ready logs debug diff --git a/models/marts/schema.yml b/models/marts/schema.yml index 612a5e4..c9255e7 100644 --- a/models/marts/schema.yml +++ b/models/marts/schema.yml @@ -2,6 +2,8 @@ version: 2 models: - name: processed_machine_health + tests: + - table_not_empty description: "Final processed machine health metrics" columns: - name: machine_id diff --git a/models/staging/schema.yml b/models/staging/schema.yml index 2bb4431..2577898 100644 --- a/models/staging/schema.yml +++ b/models/staging/schema.yml @@ -2,6 +2,8 @@ version: 2 models: - name: sensor_readings_view + tests: + - table_not_empty description: "Cleaned and validated sensor readings" columns: - name: machine_id diff --git a/setup/01_setup_snowflake.sql b/setup/01_setup_snowflake.sql index 56c6cbc..ac71ae1 100644 --- a/setup/01_setup_snowflake.sql +++ b/setup/01_setup_snowflake.sql @@ -17,6 +17,11 @@ CREATE OR REPLACE TABLE RAW_SENSOR_DATA ( -- Create a file format for CSV files CREATE OR REPLACE FILE FORMAT csv_format TYPE = CSV + -- 🐛 DEMO BUG: Incorrect CSV delimiter configuration + -- The actual CSV file uses commas, not semicolons. + -- This mismatch causes data parsing to fail in dbt test "table_not_empty" + -- Snowpipe will fail to parse CSV data correctly and Streamlit app shows no data + -- FIELD_DELIMITER = ';' -- 🚨 BUG: Causes "RAW_SENSOR_DATA" table to remain empty FIELD_DELIMITER = ',' SKIP_HEADER = 1 NULL_IF = ('NULL', 'null') diff --git a/tests/generic/table_not_empty.sql b/tests/generic/table_not_empty.sql new file mode 100644 index 0000000..fa9dff4 --- /dev/null +++ b/tests/generic/table_not_empty.sql @@ -0,0 +1,4 @@ +{% test table_not_empty(model) %} +select 1 as id +where not exists (select 1 from {{ model }} limit 1) +{% endtest %}