Fix notebook format issues

Author: github-actions[bot]

Workloads-Specific/DataWarehouse/Medallion_Architecture/src/0_notebook_bronze_to_silver.ipynb

@@ -1 +1,368 @@
-{"cells":[{"cell_type":"markdown","source":["# PySpark Code to Move Data from Bronze to Silver"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"ab7f0b7c-b0cc-44ec-9948-61d68f4b0b13"},{"cell_type":"markdown","source":["This is an example on how to work with the medallion architecture. From Bronze to Silver"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"44f47922-4e3b-45cc-81a6-c5de97634f73"},{"cell_type":"markdown","source":["## Working with 2020orders information"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4e1afae8-b2ef-4e4c-9ac9-485139f19e9c"},{"cell_type":"code","source":["from pyspark.sql.types import *\n","import pyspark.sql.functions \n","from pyspark.sql import *"],"outputs":[],"execution_count":null,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"febb6c3e-6841-42c1-a633-0da056b7f69c"},{"cell_type":"code","source":["# Read the data from the bronze layer:\n","df_raw_2020orders = spark.read.format(\"delta\").load(\"abfss://[email protected]/raw_Bronze.Lakehouse/Tables/2020orders\")\n","\n","df_raw_2020orders.head(2)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":4,"statement_ids":[4],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:11:30.8930212Z","session_start_time":null,"execution_start_time":"2024-10-25T19:14:18.2840528Z","execution_finish_time":"2024-10-25T19:14:24.7554027Z","parent_msg_id":"4d60fa03-18dd-4bbc-adab-cb12ff2ba6cf"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 4, Finished, Available, Finished)"},"metadata":{}},{"output_type":"execute_result","execution_count":11,"data":{"text/plain":"[Row(ID='SO45376', Count=1, Date='1/5/2020', Name='Edgar Mehta', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992),\n Row(ID='SO45381', Count=1, Date='1/6/2020', Name='Jordan Long', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992)]"},"metadata":{}}],"execution_count":2,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"6357dc47-1bb1-4391-8f38-b5d5a2abf5b2"},{"cell_type":"code","source":["# Clean the data (e.g., filter out rows with null values in the 'age' column):\n","df_cleaned = df_raw_2020orders.filter(df_raw_2020orders[\"Date\"].isNotNull())\n","print(df_cleaned)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":10,"statement_ids":[10],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:21:35.4162864Z","session_start_time":null,"execution_start_time":"2024-10-25T19:21:35.9099818Z","execution_finish_time":"2024-10-25T19:21:36.2079156Z","parent_msg_id":"d65f6fd9-d9ab-4498-ab5d-0710bab459be"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 10, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["DataFrame[ID: string, Count: int, Date: string, Name: string, Style: string, price: double, tax: double]\n"]}],"execution_count":8,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"5f5c8125-cbf5-4e00-9d8f-0c437f25b37f"},{"cell_type":"code","source":["# Save the cleaned data to the \"cleansed_Silver\" table in the Silver lakehouse:\n","df_cleaned.write.format(\"delta\").mode(\"overwrite\").save(\"abfss://[email protected]/cleansed_test_Silver.Lakehouse/Tables/2020orders_silver\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":12,"statement_ids":[12],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:23:52.3238132Z","session_start_time":null,"execution_start_time":"2024-10-25T19:23:52.7414203Z","execution_finish_time":"2024-10-25T19:24:09.4412514Z","parent_msg_id":"8c92d669-7856-4961-a9d0-c38d54833ee4"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 12, Finished, Available, Finished)"},"metadata":{}}],"execution_count":10,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"c83d4e46-2b49-490f-aadb-87a350c85e89"},{"cell_type":"markdown","source":["## Working with products information"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"f830afb3-2b02-4076-800a-85ca9fc33fea"},{"cell_type":"code","source":["# Read data from the Bronze layer\n","bronze_df = spark.read.format(\"delta\").load(\"abfss://[email protected]/raw_Bronze.Lakehouse/Tables/products\")\n","# Perform transformations (if any)\n","silver_df = bronze_df  # Assuming no transformations for simplicity\n","# Write data to the Silver layer\n","silver_df.write.mode(\"overwrite\").format(\"delta\").save(\"abfss://[email protected]/cleansed_test_Silver.Lakehouse/Tables/products_silver\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":15,"statement_ids":[15],"state":"finished","livy_statement_state":"available","session_id":"ecb846c9-e988-4906-95bc-af67b3aacd94","normalized_state":"finished","queued_time":"2024-10-25T19:27:20.1106109Z","session_start_time":null,"execution_start_time":"2024-10-25T19:27:20.5334249Z","execution_finish_time":"2024-10-25T19:27:25.4936309Z","parent_msg_id":"bf665ff4-43d5-4b02-90a6-6c28640576c3"},"text/plain":"StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 15, Finished, Available, Finished)"},"metadata":{}}],"execution_count":13,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"7f72ac98-4ece-4a8a-a5c5-5e1fc7273382"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"[email protected]"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{"default_lakehouse":"251cd515-16a3-4555-a3d2-dfd12adb2335","default_lakehouse_name":"raw_Bronze","default_lakehouse_workspace_id":"597e0afc-c8db-4f4d-8464-d13570f5b075"}}},"nbformat":4,"nbformat_minor":5}
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ab7f0b7c-b0cc-44ec-9948-61d68f4b0b13",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "source": [
+    "# PySpark Code to Move Data from Bronze to Silver"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "44f47922-4e3b-45cc-81a6-c5de97634f73",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "source": [
+    "This is an example on how to work with the medallion architecture. From Bronze to Silver"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e1afae8-b2ef-4e4c-9ac9-485139f19e9c",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "source": [
+    "## Working with 2020orders information"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "febb6c3e-6841-42c1-a633-0da056b7f69c",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from pyspark.sql.types import *\n",
+    "import pyspark.sql.functions \n",
+    "from pyspark.sql import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6357dc47-1bb1-4391-8f38-b5d5a2abf5b2",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.livy.statement-meta+json": {
+       "execution_finish_time": "2024-10-25T19:14:24.7554027Z",
+       "execution_start_time": "2024-10-25T19:14:18.2840528Z",
+       "livy_statement_state": "available",
+       "normalized_state": "finished",
+       "parent_msg_id": "4d60fa03-18dd-4bbc-adab-cb12ff2ba6cf",
+       "queued_time": "2024-10-25T19:11:30.8930212Z",
+       "session_id": "ecb846c9-e988-4906-95bc-af67b3aacd94",
+       "session_start_time": null,
+       "spark_pool": null,
+       "state": "finished",
+       "statement_id": 4,
+       "statement_ids": [
+        4
+       ]
+      },
+      "text/plain": [
+       "StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 4, Finished, Available, Finished)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[Row(ID='SO45376', Count=1, Date='1/5/2020', Name='Edgar Mehta', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992),\n",
+       " Row(ID='SO45381', Count=1, Date='1/6/2020', Name='Jordan Long', Style='Mountain-100 Silver, 38', price=3399.99, tax=271.9992)]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Read the data from the bronze layer:\n",
+    "df_raw_2020orders = spark.read.format(\"delta\").load(\"abfss://[email protected]/raw_Bronze.Lakehouse/Tables/2020orders\")\n",
+    "\n",
+    "df_raw_2020orders.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "5f5c8125-cbf5-4e00-9d8f-0c437f25b37f",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.livy.statement-meta+json": {
+       "execution_finish_time": "2024-10-25T19:21:36.2079156Z",
+       "execution_start_time": "2024-10-25T19:21:35.9099818Z",
+       "livy_statement_state": "available",
+       "normalized_state": "finished",
+       "parent_msg_id": "d65f6fd9-d9ab-4498-ab5d-0710bab459be",
+       "queued_time": "2024-10-25T19:21:35.4162864Z",
+       "session_id": "ecb846c9-e988-4906-95bc-af67b3aacd94",
+       "session_start_time": null,
+       "spark_pool": null,
+       "state": "finished",
+       "statement_id": 10,
+       "statement_ids": [
+        10
+       ]
+      },
+      "text/plain": [
+       "StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 10, Finished, Available, Finished)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DataFrame[ID: string, Count: int, Date: string, Name: string, Style: string, price: double, tax: double]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Clean the data (e.g., filter out rows with null values in the 'age' column):\n",
+    "df_cleaned = df_raw_2020orders.filter(df_raw_2020orders[\"Date\"].isNotNull())\n",
+    "print(df_cleaned)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c83d4e46-2b49-490f-aadb-87a350c85e89",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.livy.statement-meta+json": {
+       "execution_finish_time": "2024-10-25T19:24:09.4412514Z",
+       "execution_start_time": "2024-10-25T19:23:52.7414203Z",
+       "livy_statement_state": "available",
+       "normalized_state": "finished",
+       "parent_msg_id": "8c92d669-7856-4961-a9d0-c38d54833ee4",
+       "queued_time": "2024-10-25T19:23:52.3238132Z",
+       "session_id": "ecb846c9-e988-4906-95bc-af67b3aacd94",
+       "session_start_time": null,
+       "spark_pool": null,
+       "state": "finished",
+       "statement_id": 12,
+       "statement_ids": [
+        12
+       ]
+      },
+      "text/plain": [
+       "StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 12, Finished, Available, Finished)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Save the cleaned data to the \"cleansed_Silver\" table in the Silver lakehouse:\n",
+    "df_cleaned.write.format(\"delta\").mode(\"overwrite\").save(\"abfss://[email protected]/cleansed_test_Silver.Lakehouse/Tables/2020orders_silver\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f830afb3-2b02-4076-800a-85ca9fc33fea",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "source": [
+    "## Working with products information"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "7f72ac98-4ece-4a8a-a5c5-5e1fc7273382",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.livy.statement-meta+json": {
+       "execution_finish_time": "2024-10-25T19:27:25.4936309Z",
+       "execution_start_time": "2024-10-25T19:27:20.5334249Z",
+       "livy_statement_state": "available",
+       "normalized_state": "finished",
+       "parent_msg_id": "bf665ff4-43d5-4b02-90a6-6c28640576c3",
+       "queued_time": "2024-10-25T19:27:20.1106109Z",
+       "session_id": "ecb846c9-e988-4906-95bc-af67b3aacd94",
+       "session_start_time": null,
+       "spark_pool": null,
+       "state": "finished",
+       "statement_id": 15,
+       "statement_ids": [
+        15
+       ]
+      },
+      "text/plain": [
+       "StatementMeta(, ecb846c9-e988-4906-95bc-af67b3aacd94, 15, Finished, Available, Finished)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Read data from the Bronze layer\n",
+    "bronze_df = spark.read.format(\"delta\").load(\"abfss://[email protected]/raw_Bronze.Lakehouse/Tables/products\")\n",
+    "# Perform transformations (if any)\n",
+    "silver_df = bronze_df  # Assuming no transformations for simplicity\n",
+    "# Write data to the Silver layer\n",
+    "silver_df.write.mode(\"overwrite\").format(\"delta\").save(\"abfss://[email protected]/cleansed_test_Silver.Lakehouse/Tables/products_silver\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "dependencies": {
+   "lakehouse": {
+    "default_lakehouse": "251cd515-16a3-4555-a3d2-dfd12adb2335",
+    "default_lakehouse_name": "raw_Bronze",
+    "default_lakehouse_workspace_id": "597e0afc-c8db-4f4d-8464-d13570f5b075"
+   }
+  },
+  "kernel_info": {
+   "name": "synapse_pyspark"
+  },
+  "kernelspec": {
+   "display_name": "Synapse PySpark",
+   "language": "Python",
+   "name": "synapse_pyspark"
+  },
+  "language_info": {
+   "name": "python"
+  },
+  "microsoft": {
+   "language": "python",
+   "language_group": "synapse_pyspark",
+   "ms_spell_check": {
+    "ms_spell_check_language": "en"
+   }
+  },
+  "nteract": {
+   "version": "[email protected]"
+  },
+  "spark_compute": {
+   "compute_id": "/trident/default",
+   "session_options": {
+    "conf": {
+     "spark.synapse.nbs.session.timeout": "1200000"
+    }
+   }
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version": "1.0"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}