-
+
[](https://visitcount.itsvg.in)
@@ -25,15 +25,22 @@ This repository is organized into several key sections to help you find and cont
```
machine-learning-repos/
-├── docs/ # Documentation and guidelines
-├── projects/ # ML projects by category
-│ ├── classification/ # Classification projects
-│ ├── detection/ # Detection models
-│ ├── generative-ai/ # LLMs and generative AI
-│ ├── nlp/ # Natural Language Processing
-│ ├── computer-vision/ # Computer Vision & OpenCV
-│ └── prediction/ # Prediction models
-└── tutorials/ # Educational content
+├── .github/
+├── docs/
+├── implementations/
+├── projects/
+│ ├── algorithms/
+│ ├── classification/
+│ ├── computer-vision/
+│ ├── data-analysis/
+│ ├── detection/
+│ ├── generative-ai/
+│ ├── nlp/
+│ ├── prediction/
+│ ├── recommendation/
+│ └── time-series/
+├── website/
+└── README.md
```
See [Organization Guide](docs/ORGANIZATION.md) for detailed structure.
@@ -1146,7 +1153,7 @@ To truly excel in Machine Learning, consistent practice is essential—engage in
![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Phishing Website Classification Project"
+ ],
+ "metadata": {
+ "id": "19JPa653fAOK"
+ }
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## 1. Objective\n",
+ "The primary objective of this project is to develop and evaluate various machine learning models capable of classifying websites as either legitimate or phishing based on a given dataset. We will explore different algorithms to identify the most effective approach"
+ ],
+ "metadata": {
+ "id": "iSIEyorEGA0M"
+ }
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## 2. Loading Data:"
+ ],
+ "metadata": {
+ "id": "a8ybwvI7F25L"
+ }
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "f08751c8"
+ },
+ "source": [
+ "#importing basic packages\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import seaborn as sns\n",
+ "import matplotlib.pyplot as plt\n",
+ "import os"
+ ],
+ "execution_count": 1,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ebf87b0d",
+ "outputId": "07f88927-d727-4722-a8e6-b58f98be2dae",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 0
+ }
+ },
+ "source": [
+ "from google.colab import drive\n",
+ "\n",
+ "# Local file path\n",
+ "file_path = '/content/Phishing Data - Phishing Data.csv'\n",
+ "\n",
+ "# If the file is not in Colab, mount Google Drive\n",
+ "if not os.path.exists(file_path):\n",
+ " print(\"Local file not found, mounting Google Drive...\")\n",
+ " drive.mount('/content/drive')\n",
+ " file_path = '/content/drive/MyDrive/Phishing Data - Phishing Data.csv'\n",
+ "\n",
+ "# Load dataset\n",
+ "df = pd.read_csv(file_path)\n",
+ "\n",
+ "print(\"Dataset loaded successfully!\")\n",
+ "print(\"Shape:\", df.shape)\n",
+ "display(df.head())"
+ ],
+ "execution_count": 2,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Local file not found, mounting Google Drive...\n",
+ "Mounted at /content/drive\n",
+ "Dataset loaded successfully!\n",
+ "Shape: (2456, 31)\n"
+ ]
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ " having_IP_Address URL_Length Shortining_Service having_At_Symbol \\\n",
+ "0 1 1 0 0 \n",
+ "1 0 1 0 0 \n",
+ "2 0 0 0 0 \n",
+ "3 0 0 0 0 \n",
+ "4 0 0 1 0 \n",
+ "\n",
+ " double_slash_redirecting Prefix_Suffix having_Sub_Domain SSLfinal_State \\\n",
+ "0 1 -1 -1 -1 \n",
+ "1 0 -1 0 1 \n",
+ "2 0 -1 -1 -1 \n",
+ "3 0 -1 -1 -1 \n",
+ "4 0 -1 1 1 \n",
+ "\n",
+ " Domain_registeration_length Favicon ... popUpWidnow Iframe \\\n",
+ "0 0 0 ... 0 0 \n",
+ "1 0 0 ... 0 0 \n",
+ "2 0 0 ... 0 0 \n",
+ "3 1 0 ... 0 0 \n",
+ "4 0 0 ... 1 0 \n",
+ "\n",
+ " age_of_domain DNSRecord web_traffic Page_Rank Google_Index \\\n",
+ "0 -1 1 -1 -1 0 \n",
+ "1 -1 1 0 -1 0 \n",
+ "2 0 1 1 -1 0 \n",
+ "3 -1 1 1 -1 0 \n",
+ "4 -1 1 0 -1 0 \n",
+ "\n",
+ " Links_pointing_to_page Statistical_report Result \n",
+ "0 1 1 1 \n",
+ "1 1 0 1 \n",
+ "2 0 1 1 \n",
+ "3 -1 0 1 \n",
+ "4 1 0 0 \n",
+ "\n",
+ "[5 rows x 31 columns]"
+ ],
+ "text/html": [
+ "\n",
+ " | \n", + " | having_IP_Address | \n", + "URL_Length | \n", + "Shortining_Service | \n", + "having_At_Symbol | \n", + "double_slash_redirecting | \n", + "Prefix_Suffix | \n", + "having_Sub_Domain | \n", + "SSLfinal_State | \n", + "Domain_registeration_length | \n", + "Favicon | \n", + "... | \n", + "popUpWidnow | \n", + "Iframe | \n", + "age_of_domain | \n", + "DNSRecord | \n", + "web_traffic | \n", + "Page_Rank | \n", + "Google_Index | \n", + "Links_pointing_to_page | \n", + "Statistical_report | \n", + "Result | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "1 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "1 | \n", + "-1 | \n", + "-1 | \n", + "-1 | \n", + "0 | \n", + "0 | \n", + "... | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "1 | \n", + "-1 | \n", + "-1 | \n", + "0 | \n", + "1 | \n", + "1 | \n", + "1 | \n", + "
| 1 | \n", + "0 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "0 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "... | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "-1 | \n", + "0 | \n", + "1 | \n", + "0 | \n", + "1 | \n", + "
| 2 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "-1 | \n", + "-1 | \n", + "0 | \n", + "0 | \n", + "... | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "1 | \n", + "1 | \n", + "-1 | \n", + "0 | \n", + "0 | \n", + "1 | \n", + "1 | \n", + "
| 3 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "-1 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "... | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "1 | \n", + "1 | \n", + "-1 | \n", + "0 | \n", + "-1 | \n", + "0 | \n", + "1 | \n", + "
| 4 | \n", + "0 | \n", + "0 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "-1 | \n", + "1 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "... | \n", + "1 | \n", + "0 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "-1 | \n", + "0 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "
5 rows × 31 columns
\n", + "DecisionTreeClassifier(max_depth=5)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
DecisionTreeClassifier(max_depth=5)
RandomForestClassifier(max_depth=5)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
RandomForestClassifier(max_depth=5)
MLPClassifier(alpha=0.001, hidden_layer_sizes=[100, 100, 100])In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
MLPClassifier(alpha=0.001, hidden_layer_sizes=[100, 100, 100])
XGBClassifier(base_score=None, booster=None, callbacks=None,\n", + " colsample_bylevel=None, colsample_bynode=None,\n", + " colsample_bytree=None, device=None, early_stopping_rounds=None,\n", + " enable_categorical=False, eval_metric=None, feature_types=None,\n", + " feature_weights=None, gamma=None, grow_policy=None,\n", + " importance_type=None, interaction_constraints=None,\n", + " learning_rate=0.3, max_bin=None, max_cat_threshold=None,\n", + " max_cat_to_onehot=None, max_delta_step=None, max_depth=7,\n", + " max_leaves=None, min_child_weight=None, missing=nan,\n", + " monotone_constraints=None, multi_strategy=None, n_estimators=None,\n", + " n_jobs=None, num_parallel_tree=None, ...)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
XGBClassifier(base_score=None, booster=None, callbacks=None,\n", + " colsample_bylevel=None, colsample_bynode=None,\n", + " colsample_bytree=None, device=None, early_stopping_rounds=None,\n", + " enable_categorical=False, eval_metric=None, feature_types=None,\n", + " feature_weights=None, gamma=None, grow_policy=None,\n", + " importance_type=None, interaction_constraints=None,\n", + " learning_rate=0.3, max_bin=None, max_cat_threshold=None,\n", + " max_cat_to_onehot=None, max_delta_step=None, max_depth=7,\n", + " max_leaves=None, min_child_weight=None, missing=nan,\n", + " monotone_constraints=None, multi_strategy=None, n_estimators=None,\n", + " n_jobs=None, num_parallel_tree=None, ...)
SVC(kernel='linear', random_state=12)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
SVC(kernel='linear', random_state=12)
SVC(random_state=12)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
SVC(random_state=12)
| \n", + " | ML Model | \n", + "Train Accuracy | \n", + "Test Accuracy | \n", + "
|---|---|---|---|
| 0 | \n", + "Decision Tree | \n", + "0.947 | \n", + "0.943 | \n", + "
| 1 | \n", + "Random Forest | \n", + "0.952 | \n", + "0.957 | \n", + "
| 2 | \n", + "Multilayer Perceptrons | \n", + "0.996 | \n", + "0.974 | \n", + "
| 3 | \n", + "XGBoost | \n", + "0.995 | \n", + "0.970 | \n", + "
| 4 | \n", + "SVM | \n", + "0.967 | \n", + "0.955 | \n", + "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Reproducing the dataset\n",
+ "objective : the objective is to simulate the behaviour of high-frequency time-series data.\n",
+ "\n",
+ "(which is data collected at regular intervals, often at high frequencies (like every 5ms).)"
+ ],
+ "metadata": {
+ "id": "_NPSp7r3zyCF"
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "QvlejI03ypPH"
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Create metadata lookup (ItemId -> ItemName, Comment, Unit)\n",
+ "meta = pd.DataFrame({\n",
+ " 'ItemId': [35827, 35828],\n",
+ " 'ItemName': ['L_R1SDW_SPDREF', 'L_R1SDD_SPDREF'],\n",
+ " 'Comment': ['R1 Screw Down WS Speed Reference (mps)', 'R1 Screw Down DS Speed Reference (mps)'],\n",
+ " 'Unit': ['mps', 'mps']\n",
+ "}).set_index('ItemName')\n",
+ "meta.head()"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 143
+ },
+ "id": "tKfgrDI-z8_E",
+ "outputId": "88c25f7a-8b69-409a-a2d7-720719236e0a"
+ },
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ " ItemId Comment Unit\n",
+ "ItemName \n",
+ "L_R1SDW_SPDREF 35827 R1 Screw Down WS Speed Reference (mps) mps\n",
+ "L_R1SDD_SPDREF 35828 R1 Screw Down DS Speed Reference (mps) mps"
+ ],
+ "text/html": [
+ "\n",
+ " | \n", + " | ItemId | \n", + "Comment | \n", + "Unit | \n", + "
|---|---|---|---|
| ItemName | \n", + "\n", + " | \n", + " | \n", + " |
| L_R1SDW_SPDREF | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| L_R1SDD_SPDREF | \n", + "35828 | \n", + "R1 Screw Down DS Speed Reference (mps) | \n", + "mps | \n", + "
| \n", + " | timestamp | \n", + "L_R1SDW_SPDREF | \n", + "L_R1SDD_SPDREF | \n", + "
|---|---|---|---|
| 0 | \n", + "2021-05-08 21:30:39.000 | \n", + "16.464405 | \n", + "32.460739 | \n", + "
| 1 | \n", + "2021-05-08 21:30:39.005 | \n", + "21.455681 | \n", + "19.043359 | \n", + "
| 2 | \n", + "2021-05-08 21:30:39.010 | \n", + "18.082901 | \n", + "20.926240 | \n", + "
| 3 | \n", + "2021-05-08 21:30:39.015 | \n", + "16.346495 | \n", + "10.020823 | \n", + "
| 4 | \n", + "2021-05-08 21:30:39.020 | \n", + "12.709644 | \n", + "24.201721 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 1995 | \n", + "2021-05-08 21:30:48.975 | \n", + "23.738447 | \n", + "13.985575 | \n", + "
| 1996 | \n", + "2021-05-08 21:30:48.980 | \n", + "21.635943 | \n", + "17.394945 | \n", + "
| 1997 | \n", + "2021-05-08 21:30:48.985 | \n", + "14.403234 | \n", + "33.200118 | \n", + "
| 1998 | \n", + "2021-05-08 21:30:48.990 | \n", + "19.315921 | \n", + "37.312247 | \n", + "
| 1999 | \n", + "2021-05-08 21:30:48.995 | \n", + "15.053194 | \n", + "12.333537 | \n", + "
2000 rows × 3 columns
\n", + "| \n", + " | timestamp | \n", + "tag_name | \n", + "tag_value | \n", + "
|---|---|---|---|
| 0 | \n", + "2021-05-08 21:30:39.000 | \n", + "L_R1SDW_SPDREF | \n", + "16.464405 | \n", + "
| 1 | \n", + "2021-05-08 21:30:39.005 | \n", + "L_R1SDW_SPDREF | \n", + "21.455681 | \n", + "
| 2 | \n", + "2021-05-08 21:30:39.010 | \n", + "L_R1SDW_SPDREF | \n", + "18.082901 | \n", + "
| 3 | \n", + "2021-05-08 21:30:39.015 | \n", + "L_R1SDW_SPDREF | \n", + "16.346495 | \n", + "
| 4 | \n", + "2021-05-08 21:30:39.020 | \n", + "L_R1SDW_SPDREF | \n", + "12.709644 | \n", + "
| 5 | \n", + "2021-05-08 21:30:39.025 | \n", + "L_R1SDW_SPDREF | \n", + "19.376823 | \n", + "
| 6 | \n", + "2021-05-08 21:30:39.030 | \n", + "L_R1SDW_SPDREF | \n", + "13.127616 | \n", + "
| 7 | \n", + "2021-05-08 21:30:39.035 | \n", + "L_R1SDW_SPDREF | \n", + "26.753190 | \n", + "
| 8 | \n", + "2021-05-08 21:30:39.040 | \n", + "L_R1SDW_SPDREF | \n", + "28.909883 | \n", + "
| 9 | \n", + "2021-05-08 21:30:39.045 | \n", + "L_R1SDW_SPDREF | \n", + "11.503246 | \n", + "
| \n", + " | timestamp | \n", + "tag_name | \n", + "tag_value | \n", + "tag__id | \n", + "tag__desc | \n", + "tag__unit | \n", + "
|---|---|---|---|---|---|---|
| 0 | \n", + "2021-05-08 21:30:39.000 | \n", + "L_R1SDW_SPDREF | \n", + "16.464405 | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| 1 | \n", + "2021-05-08 21:30:39.005 | \n", + "L_R1SDW_SPDREF | \n", + "21.455681 | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| 2 | \n", + "2021-05-08 21:30:39.010 | \n", + "L_R1SDW_SPDREF | \n", + "18.082901 | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| 3 | \n", + "2021-05-08 21:30:39.015 | \n", + "L_R1SDW_SPDREF | \n", + "16.346495 | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| 4 | \n", + "2021-05-08 21:30:39.020 | \n", + "L_R1SDW_SPDREF | \n", + "12.709644 | \n", + "35827 | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "
| \n", + " | time_bucket | \n", + "tag__id | \n", + "tag_name | \n", + "tag__desc | \n", + "tag__unit | \n", + "tag__value | \n", + "
|---|---|---|---|---|---|---|
| 0 | \n", + "2021-05-08 21:30:30 | \n", + "35827 | \n", + "L_R1SDW_SPDREF | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "99.123456 | \n", + "
| 1 | \n", + "2021-05-08 21:30:30 | \n", + "35828 | \n", + "L_R1SDD_SPDREF | \n", + "R1 Screw Down DS Speed Reference (mps) | \n", + "mps | \n", + "39.581003 | \n", + "
| 2 | \n", + "2021-05-08 21:30:40 | \n", + "35827 | \n", + "L_R1SDW_SPDREF | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "mps | \n", + "29.994257 | \n", + "
| 3 | \n", + "2021-05-08 21:30:40 | \n", + "35828 | \n", + "L_R1SDD_SPDREF | \n", + "R1 Screw Down DS Speed Reference (mps) | \n", + "mps | \n", + "88.654321 | \n", + "
| \n", + " | event_timestamp | \n", + "tag__id | \n", + "tag_name | \n", + "tag__desc | \n", + "tag__value | \n", + "tag__unit | \n", + "
|---|---|---|---|---|---|---|
| 0 | \n", + "08/05/2021 21:30:30 | \n", + "35827 | \n", + "L_R1SDW_SPDREF | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "99.123456 | \n", + "mps | \n", + "
| 1 | \n", + "08/05/2021 21:30:30 | \n", + "35828 | \n", + "L_R1SDD_SPDREF | \n", + "R1 Screw Down DS Speed Reference (mps) | \n", + "39.581003 | \n", + "mps | \n", + "
| 2 | \n", + "08/05/2021 21:30:40 | \n", + "35827 | \n", + "L_R1SDW_SPDREF | \n", + "R1 Screw Down WS Speed Reference (mps) | \n", + "29.994257 | \n", + "mps | \n", + "
| 3 | \n", + "08/05/2021 21:30:40 | \n", + "35828 | \n", + "L_R1SDD_SPDREF | \n", + "R1 Screw Down DS Speed Reference (mps) | \n", + "88.654321 | \n", + "mps | \n", + "
| \n", + " | tag__value | \n", + "
|---|---|
| tag__id | \n", + "\n", + " |
| 35827 | \n", + "99.123456 | \n", + "
| 35828 | \n", + "88.654321 | \n", + "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "### Timeseries Data Handling\n",
+ "\n",
+ "* **Time-Series Data:** now i can understand the unique nature of time-series data, which is data collected at regular intervals, often at high frequencies (like every 5ms).\n",
+ "* **Data Transformation:** mastered the critical process of transforming data from a wide format (one row, many columns) to a long format (many rows, one column for each type of value). This is often called \"melting\" or \"unpivoting\" in Pandas and is essential for working with sensor data.\n",
+ "* **Aggregation:** learned how to use aggregation functions, eg. MAX, to reduce huge volumes of data into a more manageable and meaningful summary. This is a common practice in Industrial IoT (IIoT) to prevent data overload.\n",
+ "* **Data Pipelining:** By combining these steps—reading the data, melting, mapping metadata, aggregating, and formatting the output built a small but complete data pipeline."
+ ],
+ "metadata": {
+ "id": "QDGMmWr7u1xz"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import os\n",
+ "\n",
+ "def process_iiot_data(file_path: str, output_path: str = \"Sample_Output.csv\"):\n",
+ " \"\"\"\n",
+ " Transforms IIoT sensor data from a wide, high-frequency format to a long,\n",
+ " aggregated time-series.\n",
+ "\n",
+ " Args:\n",
+ " file_path (str): The path to the input CSV file containing metadata and\n",
+ " high-frequency sensor readings.\n",
+ " output_path (str): The file path to save the final aggregated CSV.\n",
+ " \"\"\"\n",
+ " if not os.path.exists(file_path):\n",
+ " print(f\"Error: The file '{file_path}' was not found.\")\n",
+ " return\n",
+ "\n",
+ " ## 1. READ METADATA & DATA\n",
+ " # Dynamically detect the header line to separate metadata from readings\n",
+ " with open(file_path, 'r') as f:\n",
+ " header_line = None\n",
+ " for i, line in enumerate(f):\n",
+ " if 'Date' in line and 'Time' in line:\n",
+ " header_line = i\n",
+ " break\n",
+ "\n",
+ " # Read metadata from the top of the file down to the header line\n",
+ " meta = pd.read_csv(file_path, nrows=header_line)\n",
+ " meta = meta.loc[:, ~meta.columns.str.contains('^Unnamed')]\n",
+ "\n",
+ " # Read high-frequency data from the header line onwards\n",
+ " hf_data = pd.read_csv(file_path, skiprows=header_line)\n",
+ "\n",
+ " ## 2. CREATE MAPPING\n",
+ " # Use the metadata to create lookup dictionaries for each sensor\n",
+ " itemid_map = dict(zip(meta[\"ItemName\"], meta[\"ItemId\"]))\n",
+ " desc_map = dict(zip(meta[\"ItemName\"], meta[\"Comment\"]))\n",
+ " unit_map = dict(zip(meta[\"ItemName\"], meta[\"Unit\"]))\n",
+ "\n",
+ " ## 3. CONVERT TO LONG FORMAT (MELT)\n",
+ " # Combine Date, Time, Milli Sec into a single timestamp\n",
+ " hf_data['timestamp'] = pd.to_datetime(hf_data['Date'] + ' ' + hf_data['Time'], dayfirst=True) + pd.to_timedelta(hf_data['Milli Sec'], unit='ms')\n",
+ "\n",
+ " # Get a list of only the sensor columns\n",
+ " sensor_cols = [col for col in hf_data.columns if col not in ['Date', 'Time', 'Milli Sec', 'timestamp']]\n",
+ "\n",
+ " # Melt the DataFrame from a wide format to a long format\n",
+ " hf_long = hf_data.melt(\n",
+ " id_vars=['timestamp'],\n",
+ " value_vars=sensor_cols,\n",
+ " var_name='tag_name',\n",
+ " value_name='tag_value'\n",
+ " )\n",
+ "\n",
+ " ## 4. MAP METADATA TO THE LONG DATAFRAME\n",
+ " # Attach the metadata to each sensor reading using the mapping dictionaries\n",
+ " hf_long['tag__id'] = hf_long['tag_name'].map(itemid_map)\n",
+ " hf_long['tag__desc'] = hf_long['tag_name'].map(desc_map)\n",
+ " hf_long['tag__unit'] = 'mps'\n",
+ "\n",
+ " ## 5.Create a 10-second time bucket for grouping\n",
+ " hf_long['time_bucket'] = hf_long['timestamp'].dt.floor('10s')\n",
+ "\n",
+ " # Group by the time bucket and sensor metadata, then aggregate using MAX\n",
+ " agg = (\n",
+ " hf_long\n",
+ " .groupby(['time_bucket', 'tag__id', 'tag_name', 'tag__desc', 'tag__unit'], as_index=False)\n",
+ " .agg(tag__value=('tag_value', 'max'))\n",
+ " )\n",
+ "\n",
+ " ## 6.Format the time_bucket to match the required output timestamp format\n",
+ " agg['event_timestamp'] = agg['time_bucket'].dt.strftime('%d/%m/%Y %H:%M:%S')\n",
+ "\n",
+ " out = agg[['event_timestamp', 'tag__id', 'tag_name', 'tag__desc', 'tag__value', 'tag__unit']]\n",
+ "\n",
+ " # Export the final DataFrame\n",
+ " out.to_csv(output_path, index=False)\n",
+ " print(\"Transformation complete. Output saved as {}\".format(output_path))\n",
+ "\n",
+ "\n",
+ "if __name__ == \"__main__\":\n",
+ " input_file_path = \"/content/Sample Input - Sample.csv\"\n",
+ "\n",
+ " # Run the data processing pipeline\n",
+ " process_iiot_data(input_file_path)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "a0XE5TPZX7u2",
+ "outputId": "e0a9593f-8151-496a-e685-f1eddf7cd129"
+ },
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Transformation complete. Output saved as Sample_Output.csv\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [],
+ "metadata": {
+ "id": "UjurqzQFJq3c"
+ },
+ "execution_count": null,
+ "outputs": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/projects/time_series/src/pipeline.py b/projects/time_series/src/pipeline.py
new file mode 100644
index 000000000..bfc348aca
--- /dev/null
+++ b/projects/time_series/src/pipeline.py
@@ -0,0 +1,86 @@
+import pandas as pd
+import numpy as np
+import os
+
+def process_iiot_data(file_path: str, output_path: str = "Sample_Output.csv"):
+ """
+ Transforms IIoT sensor data from a wide, high-frequency format to a long,
+ aggregated time-series.
+
+ Args:
+ file_path (str): The path to the input CSV file containing metadata and
+ high-frequency sensor readings.
+ output_path (str): The file path to save the final aggregated CSV.
+ """
+ if not os.path.exists(file_path):
+ print(f"Error: The file '{file_path}' was not found.")
+ return
+
+ ## 1. READ METADATA & DATA
+ # Dynamically detect the header line to separate metadata from readings
+ with open(file_path, 'r') as f:
+ header_line = None
+ for i, line in enumerate(f):
+ if 'Date' in line and 'Time' in line:
+ header_line = i
+ break
+
+ # Read metadata from the top of the file down to the header line
+ meta = pd.read_csv(file_path, nrows=header_line)
+ meta = meta.loc[:, ~meta.columns.str.contains('^Unnamed')]
+
+ # Read high-frequency data from the header line onwards
+ hf_data = pd.read_csv(file_path, skiprows=header_line)
+
+ ## 2. CREATE MAPPING
+ # Use the metadata to create lookup dictionaries for each sensor
+ itemid_map = dict(zip(meta["ItemName"], meta["ItemId"]))
+ desc_map = dict(zip(meta["ItemName"], meta["Comment"]))
+ unit_map = dict(zip(meta["ItemName"], meta["Unit"]))
+
+ ## 3. CONVERT TO LONG FORMAT (MELT)
+ # Combine Date, Time, Milli Sec into a single timestamp
+ hf_data['timestamp'] = pd.to_datetime(hf_data['Date'] + ' ' + hf_data['Time'], dayfirst=True) + pd.to_timedelta(hf_data['Milli Sec'], unit='ms')
+
+ # Get a list of only the sensor columns
+ sensor_cols = [col for col in hf_data.columns if col not in ['Date', 'Time', 'Milli Sec', 'timestamp']]
+
+ # Melt the DataFrame from a wide format to a long format
+ hf_long = hf_data.melt(
+ id_vars=['timestamp'],
+ value_vars=sensor_cols,
+ var_name='tag_name',
+ value_name='tag_value'
+ )
+
+ ## 4. MAP METADATA TO THE LONG DATAFRAME
+ # Attach the metadata to each sensor reading using the mapping dictionaries
+ hf_long['tag__id'] = hf_long['tag_name'].map(itemid_map)
+ hf_long['tag__desc'] = hf_long['tag_name'].map(desc_map)
+ hf_long['tag__unit'] = 'mps'
+
+ ## 5.Create a 10-second time bucket for grouping
+ hf_long['time_bucket'] = hf_long['timestamp'].dt.floor('10s')
+
+ # Group by the time bucket and sensor metadata, then aggregate using MAX
+ agg = (
+ hf_long
+ .groupby(['time_bucket', 'tag__id', 'tag_name', 'tag__desc', 'tag__unit'], as_index=False)
+ .agg(tag__value=('tag_value', 'max'))
+ )
+
+ ## 6.Format the time_bucket to match the required output timestamp format
+ agg['event_timestamp'] = agg['time_bucket'].dt.strftime('%d/%m/%Y %H:%M:%S')
+
+ out = agg[['event_timestamp', 'tag__id', 'tag_name', 'tag__desc', 'tag__value', 'tag__unit']]
+
+ # Export the final DataFrame
+ out.to_csv(output_path, index=False)
+ print("Transformation complete. Output saved as {}".format(output_path))
+
+
+if __name__ == "__main__":
+ input_file_path = "/content/Sample Input - Sample.csv"
+
+ # Run the data processing pipeline
+ process_iiot_data(input_file_path)
\ No newline at end of file