Initial commit

Author: sfc-gh-ghernandez

README.md

@@ -0,0 +1,39 @@
+## Advanced Data Engineering with Snowflake
+
+#### How to use this repo throughout the course:
+
+To successfully follow along with the instructor during the course, you'll need to make use of the code in this repo. To follow along, you can either:
+
+* Keep the URL to this repo handy, so that you can easily find and use any code referenced by the instructor during the course
+
+* Clone the repo to your local computing environment (required)
+
+> **Note:** There are a couple of exercises that make use of Snowflake's command line interface, Snowflake CLI. To successfully follow along during those exercises, you'll need to have the repo cloned to your local computing environment, so that the Snowflake CLI can make use of files and code within this repo.
+
+#### How to clone the repo to your local computing environment:
+
+1. Fork the repo to create a copy associated with your GitHub Account: https://github.com/Snowflake-Labs/advanced-data-engineering-snowflake/fork
+
+2. Clone your fork:
+
+```bash
+git clone https://github.com/<your-GitHub-user-name>/modern-data-engineering-snowflake.git
+```
+
+Where `<your-GitHub-user-name>` is replaced by your GitHub user name. This workflow is covered in the course.
+
+You can then open the repo in your preferred code editor. Throughout the course, the instructor will use Visual Studio Code as the code editor.
+
+#### How to navigate this repo
+
+All of the code that you need to successfully complete the course is within this repo. Each folder in this repo corresponds to a module in the online course.
+
+* **module-1** – Corresponds to "Module 1: DevOps with Snowflake" in the course.
+
+* **module-2** – Corresponds to "Module 2: Observability with Snowflake" in the course.
+
+The course instructor will also be sure to reference the exact folder and name of the file to use throughout the course, so that you can follow along.
+
+#### Reporting issues or errata
+
+If you encounter technical issues with this code as you complete the course (i.e. typos, missing code, broken links, etc.), please report those issues in the course through Coursera. Ensure the issue contains sufficient detail so that it can be properly addressed.

cleanup.sql

@@ -0,0 +1,19 @@
+USE ROLE accountadmin;
+USE WAREHOUSE compute_wh;
+
+USE DATABASE tasty_bytes;
+DROP VIEW tasty_bytes.harmonized.daily_weather_v;
+
+DROP VIEW tasty_bytes.harmonized.windspeed_hamburg;
+
+DROP FUNCTION tasty_bytes.analytics.fahrenheit_to_celsius();
+
+DROP FUNCTION tasty_bytes.analytics.inch_to_millimeter();
+
+DROP VIEW tasty_bytes.harmonized.weather_hamburg;
+
+DROP VIEW tasty_bytes.analytics.daily_city_metrics_v;
+
+DROP STREAM tasty_bytes.raw_pos.order_header_stream ON TABLE tasty_bytes.raw_pos.order_header;
+
+DROP PROCEDURE tasty_bytes.raw_pos.process_order_headers_stream();

config.toml

@@ -0,0 +1,9 @@
+# DO NOT EDIT THIS FILE - DO NOT EDIT THIS FILE - DO NOT EDIT THIS FILE 
+
+default_connection_name = "advanced_data_engineering_snowflake"
+
+[connections]
+[connections.advanced_data_engineering_snowflake]
+account=""
+user=""
+password=""

module-1/api_integration.sql

@@ -0,0 +1,25 @@
+USE ROLE accountadmin;
+CREATE DATABASE course_repo;
+USE SCHEMA public;
+
+-- Create credentials
+CREATE OR REPLACE SECRET course_repo.public.github_pat
+  TYPE = password
+  USERNAME = ''
+  PASSWORD = '';
+
+-- Create the API integration
+CREATE OR REPLACE API INTEGRATION
+  API_PROVIDER = git_https_api
+  API_ALLOWED_PREFIXES = ('') -- URL to your GitHub profile
+  ALLOWED_AUTHENTICATION_SECRETS = ()
+  ENABLED = TRUE;
+
+-- Create the git repository object
+CREATE OR REPLACE GIT REPOSITORY course_repo.public.advanced_data_engineering_snowflake
+  API_INTEGRATION =  -- Name of the API integration defined above
+  ORIGIN = '' -- Insert URL of forked repo
+  GIT_CREDENTIALS = ;
+
+-- List the git repositories
+SHOW GIT REPOSITORIES;

module-1/hamburg_weather/main.yml

@@ -0,0 +1,67 @@
+name: Deploy Templatized Data Environment
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - staging
+      - main
+  push:
+    branches:
+      - staging
+      - main
+
+jobs:
+  deploy_data_environment:
+    # Only run when PR is merged or on direct push to environment branches
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
+    runs-on: ubuntu-latest
+
+    env:
+      # Read connection secret
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+
+    steps:
+      # Checkout step is necessary to access repository files
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Determine environment based on target branch
+      - name: Set environment
+        id: set_env
+        run: |
+          TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}"
+          
+          if [[ $TARGET_BRANCH == "staging" ]]; then
+            echo "DEPLOY_ENV=staging" >> $GITHUB_ENV
+            echo "Environment set to staging"
+          elif [[ $TARGET_BRANCH == "main" ]]; then
+            echo "DEPLOY_ENV=prod" >> $GITHUB_ENV
+            echo "Environment set to production"
+          else
+            echo "Unexpected branch: $TARGET_BRANCH"
+            exit 1
+          fi
+
+      # Install Snowflake CLI GitHub Action and point to config file
+      - name: Install SnowflakeCLI
+        uses: snowflakedb/snowflake-cli-action@v1.5
+        with:
+          cli-version: "latest"
+          default-config-file-path: "config.toml"
+
+      # Fetch latest changes from the repository to Snowflake
+      - name: Fetch latest changes to Snowflake
+        run: snow git fetch course_repo.public.advanced_data_engineering_snowflake
+
+      # Deploy templates to the data environment -
+      - name: Deploy templates to data environment
+        run: |
+          # Export TARGET_BRANCH for use in this step
+          echo "Using branch: ${GITHUB_REF_NAME}"
+          snow git execute @advanced_data_engineering_snowflake/branches/FIX-ME/module-1/hamburg_weather/pipeline/data/ \
+          -D "env='${{  }}'" \
+          --database=COURSE_REPO \
+          --schema=PUBLIC

module-1/hamburg_weather/notebooks/debugging.ipynb

@@ -0,0 +1,50 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3775908f-ca36-4846-8f38-5adca39217f2",
+   "metadata": {
+    "language": "sql",
+    "name": "cell1"
+   },
+   "outputs": [],
+   "source": [
+    "-- Set context\n",
+    "USE ROLE accountadmin;\n",
+    "USE DATABASE staging_tasty_bytes;\n",
+    "USE SCHEMA raw_pos;\n",
+    "USE WAREHOUSE compute_wh;"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dfd78d46-46f3-40a7-be43-4ab79cf72a9f",
+   "metadata": {
+    "language": "sql",
+    "name": "cell2"
+   },
+   "outputs": [],
+   "source": [
+    "SHOW TABLES IN staging_tasty_bytes.raw_pos;"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Streamlit Notebook",
+   "name": "streamlit"
+  },
+  "lastEditStatus": {
+   "authorEmail": "gilberto.hernandez@snowflake.com",
+   "authorId": "6002244633419",
+   "authorName": "GILBERTO",
+   "lastEditTime": 1741667434593,
+   "notebookId": "sd4b3qffhw3yzmzwpcr2",
+   "sessionId": "2ad2d56d-f367-4eba-abad-28f4dc5b5f95"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}