Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ deploy: ## Deploy the entire stack
make dbt
@echo "Deployment completed successfully. Run 'make app' to view the app."

debug: ## Clean up everything and redeploy to fix bugs
@echo "Running debug cleanup and redeploy..."
snow sql -c localstack -q "USE DATABASE FACTORY_PIPELINE_DEMO; USE SCHEMA PUBLIC; DROP FILE FORMAT IF EXISTS csv_format;"
snow sql -c localstack -q "DROP PIPE IF EXISTS FACTORY_PIPELINE_DEMO.PUBLIC.SENSOR_DATA_PIPE;"
snow sql -c localstack -q "DROP TABLE IF EXISTS FACTORY_PIPELINE_DEMO.PUBLIC_RAW.SENSOR_DATA;"
awslocal s3 rb s3://factory-sensor-data-local --force
make deploy
@echo "Debug cleanup and redeploy completed successfully."

test: ## Run tests
@echo "Running tests..."
bash -c "source env/bin/activate && pytest tests/"
Expand All @@ -90,4 +99,4 @@ ready: ## Make sure the LocalStack container is up
logs: ## Save the logs in a separate file
@localstack logs > logs.txt

.PHONY: install seed aws upload pipeline dbt app deploy test start stop ready logs
.PHONY: install seed aws upload pipeline dbt app deploy test start stop ready logs debug
2 changes: 2 additions & 0 deletions models/marts/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ version: 2

models:
- name: processed_machine_health
tests:
- table_not_empty
description: "Final processed machine health metrics"
columns:
- name: machine_id
Expand Down
2 changes: 2 additions & 0 deletions models/staging/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ version: 2

models:
- name: sensor_readings_view
tests:
- table_not_empty
description: "Cleaned and validated sensor readings"
columns:
- name: machine_id
Expand Down
5 changes: 5 additions & 0 deletions setup/01_setup_snowflake.sql
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ CREATE OR REPLACE TABLE RAW_SENSOR_DATA (
-- Create a file format for CSV files
CREATE OR REPLACE FILE FORMAT csv_format
TYPE = CSV
-- 🐛 DEMO BUG: Incorrect CSV delimiter configuration
-- The actual CSV file uses commas, not semicolons.
-- This mismatch causes data parsing to fail in dbt test "table_not_empty"
-- Snowpipe will fail to parse CSV data correctly and Streamlit app shows no data
-- FIELD_DELIMITER = ';' -- 🚨 BUG: Causes "RAW_SENSOR_DATA" table to remain empty
FIELD_DELIMITER = ','
SKIP_HEADER = 1
NULL_IF = ('NULL', 'null')
Expand Down
4 changes: 4 additions & 0 deletions tests/generic/table_not_empty.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{% test table_not_empty(model) %}
select 1 as id
where not exists (select 1 from {{ model }} limit 1)
{% endtest %}