From 2726e816b2834f55bcb2c293fc53391bc2e8da3a Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Tue, 26 Nov 2024 00:06:57 -0500 Subject: [PATCH 1/7] Add notebook on ConvMixer implementation --- automation/notebooks-table-data.csv | 3 +- notebooks/architectures/ConvMixer.ipynb | 216 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 notebooks/architectures/ConvMixer.ipynb diff --git a/automation/notebooks-table-data.csv b/automation/notebooks-table-data.csv index 506af50..9234d8d 100644 --- a/automation/notebooks-table-data.csv +++ b/automation/notebooks-table-data.csv @@ -13,4 +13,5 @@ MLP Mixer,architectures/mlp-mixer.ipynb,,https://arxiv.org/abs/2105.01601 GloVe Word Embeddings, data_exploration/glove-word-embeddings.ipynb,https://github.com/stanfordnlp/GloVe,https://nlp.stanford.edu/pubs/glove.pdf Vision Transformer (ViT),architectures/vit.ipynb,,https://arxiv.org/pdf/2010.11929 Multi-Head Attention, modules/multihead-self-attention.ipynb,,https://arxiv.org/abs/1706.03762 -ResNet,architectures/resnet.ipynb,,https://arxiv.org/abs/1512.03385 \ No newline at end of file +ResNet,architectures/resnet.ipynb,,https://arxiv.org/abs/1512.03385 +ConvMixer,architectures/ConvMixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file diff --git a/notebooks/architectures/ConvMixer.ipynb b/notebooks/architectures/ConvMixer.ipynb new file mode 100644 index 0000000..297e6da --- /dev/null +++ b/notebooks/architectures/ConvMixer.ipynb @@ -0,0 +1,216 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## ConvMixer" + ], + "metadata": { + "id": "IJveajFZvdXK" + } + }, + { + "cell_type": "code", + "source": [ + "#@title **Install required packages**\n", + "\n", + "%%capture\n", + "! pip install torchinfo" + ], + "metadata": { + "id": "q5AuzFB2tA12" + }, + "execution_count": 27, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "118P89a1osHb" + }, + "outputs": [], + "source": [ + "#@title **Importing libraries**\n", + "import torch # 2.5.1+cu121\n", + "import torch.nn as nn\n", + "import torchinfo #1.8.0" + ] + }, + { + "cell_type": "code", + "source": [ + "# Note: Not all dependencies have the __version__ method.\n", + "print(f\"torch version: {torch.__version__}\")\n", + "print(f\"torchinfo version: {torchinfo.__version__}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QchOSIbro1zf", + "outputId": "94003f5a-622f-4f54-e916-26fbbff86c17" + }, + "execution_count": 37, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "torch version: 2.5.1+cu121\n", + "torchinfo version: 1.8.0\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "**ConvMixer architecture code**\n" + ], + "metadata": { + "id": "hrG5QYB4pRMu" + } + }, + { + "cell_type": "code", + "source": [ + "class Residual(nn.Module):\n", + " def __init__(self, fn):\n", + " super().__init__()\n", + " self.fn = fn\n", + "\n", + " def forward(self, x):\n", + " return self.fn(x) + x\n", + "\n", + "def ConvMixer(dim, depth, kernel_size = 9, patch_size = 7, n_classes = 1000):\n", + " return nn.Sequential(\n", + " nn.Conv2d(3, dim, kernel_size = patch_size, stride = patch_size),\n", + " nn.GELU(),\n", + " nn.BatchNorm2d(dim),\n", + " *[nn.Sequential(\n", + " Residual(nn.Sequential(\n", + " nn.Conv2d(dim, dim, kernel_size, groups=dim, padding=\"same\"),\n", + " nn.GELU(),\n", + " nn.BatchNorm2d(dim)\n", + " )),\n", + " nn.Conv2d(dim,dim, kernel_size = 1),\n", + " nn.GELU(),\n", + " nn.BatchNorm2d(dim)\n", + " )for i in range(depth)],\n", + " nn.AdaptiveAvgPool2d((1,1)),\n", + " nn.Flatten(),\n", + " nn.Linear(dim, n_classes)\n", + " )" + ], + "metadata": { + "id": "r2d2e2P0pdan" + }, + "execution_count": 38, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "model = ConvMixer(2048, 8, kernel_size=9, patch_size=1, n_classes=1000)\n", + "torchinfo.summary(model)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9YM5zfNoslTx", + "outputId": "1b5a1295-d3ac-49c9-d61d-ae986021ad00" + }, + "execution_count": 41, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "=================================================================\n", + "Layer (type:depth-idx) Param #\n", + "=================================================================\n", + "Sequential --\n", + "├─Conv2d: 1-1 8,192\n", + "├─GELU: 1-2 --\n", + "├─BatchNorm2d: 1-3 4,096\n", + "├─Sequential: 1-4 --\n", + "│ └─Residual: 2-1 --\n", + "│ │ └─Sequential: 3-1 172,032\n", + "│ └─Conv2d: 2-2 4,196,352\n", + "│ └─GELU: 2-3 --\n", + "│ └─BatchNorm2d: 2-4 4,096\n", + "├─Sequential: 1-5 --\n", + "│ └─Residual: 2-5 --\n", + "│ │ └─Sequential: 3-2 172,032\n", + "│ └─Conv2d: 2-6 4,196,352\n", + "│ └─GELU: 2-7 --\n", + "│ └─BatchNorm2d: 2-8 4,096\n", + "├─Sequential: 1-6 --\n", + "│ └─Residual: 2-9 --\n", + "│ │ └─Sequential: 3-3 172,032\n", + "│ └─Conv2d: 2-10 4,196,352\n", + "│ └─GELU: 2-11 --\n", + "│ └─BatchNorm2d: 2-12 4,096\n", + "├─Sequential: 1-7 --\n", + "│ └─Residual: 2-13 --\n", + "│ │ └─Sequential: 3-4 172,032\n", + "│ └─Conv2d: 2-14 4,196,352\n", + "│ └─GELU: 2-15 --\n", + "│ └─BatchNorm2d: 2-16 4,096\n", + "├─Sequential: 1-8 --\n", + "│ └─Residual: 2-17 --\n", + "│ │ └─Sequential: 3-5 172,032\n", + "│ └─Conv2d: 2-18 4,196,352\n", + "│ └─GELU: 2-19 --\n", + "│ └─BatchNorm2d: 2-20 4,096\n", + "├─Sequential: 1-9 --\n", + "│ └─Residual: 2-21 --\n", + "│ │ └─Sequential: 3-6 172,032\n", + "│ └─Conv2d: 2-22 4,196,352\n", + "│ └─GELU: 2-23 --\n", + "│ └─BatchNorm2d: 2-24 4,096\n", + "├─Sequential: 1-10 --\n", + "│ └─Residual: 2-25 --\n", + "│ │ └─Sequential: 3-7 172,032\n", + "│ └─Conv2d: 2-26 4,196,352\n", + "│ └─GELU: 2-27 --\n", + "│ └─BatchNorm2d: 2-28 4,096\n", + "├─Sequential: 1-11 --\n", + "│ └─Residual: 2-29 --\n", + "│ │ └─Sequential: 3-8 172,032\n", + "│ └─Conv2d: 2-30 4,196,352\n", + "│ └─GELU: 2-31 --\n", + "│ └─BatchNorm2d: 2-32 4,096\n", + "├─AdaptiveAvgPool2d: 1-12 --\n", + "├─Flatten: 1-13 --\n", + "├─Linear: 1-14 2,049,000\n", + "=================================================================\n", + "Total params: 37,041,128\n", + "Trainable params: 37,041,128\n", + "Non-trainable params: 0\n", + "=================================================================" + ] + }, + "metadata": {}, + "execution_count": 41 + } + ] + } + ] +} \ No newline at end of file From 9bfe579a9ec535cf8d4694c985ef69b8c80909ae Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Tue, 17 Dec 2024 17:37:52 -0500 Subject: [PATCH 2/7] change the name of the notebook and the csv name --- automation/notebooks-table-data.csv | 2 +- notebooks/architectures/ConvMixer.ipynb | 100 ++++++++++++------------ 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/automation/notebooks-table-data.csv b/automation/notebooks-table-data.csv index 9234d8d..e4efca9 100644 --- a/automation/notebooks-table-data.csv +++ b/automation/notebooks-table-data.csv @@ -14,4 +14,4 @@ GloVe Word Embeddings, data_exploration/glove-word-embeddings.ipynb,https://gith Vision Transformer (ViT),architectures/vit.ipynb,,https://arxiv.org/pdf/2010.11929 Multi-Head Attention, modules/multihead-self-attention.ipynb,,https://arxiv.org/abs/1706.03762 ResNet,architectures/resnet.ipynb,,https://arxiv.org/abs/1512.03385 -ConvMixer,architectures/ConvMixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file +convmixer,architectures/ConvMixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file diff --git a/notebooks/architectures/ConvMixer.ipynb b/notebooks/architectures/ConvMixer.ipynb index 297e6da..c8f8680 100644 --- a/notebooks/architectures/ConvMixer.ipynb +++ b/notebooks/architectures/ConvMixer.ipynb @@ -1,41 +1,25 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ - "## ConvMixer" - ], - "metadata": { - "id": "IJveajFZvdXK" - } + "[![deep-learning-notes](https://github.com/semilleroCV/deep-learning-notes/raw/main/assets/banner-notebook.png)](https://github.com/semilleroCV/deep-learning-notes)" + ] }, { "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "q5AuzFB2tA12" + }, + "outputs": [], "source": [ "#@title **Install required packages**\n", "\n", "%%capture\n", "! pip install torchinfo" - ], - "metadata": { - "id": "q5AuzFB2tA12" - }, - "execution_count": 27, - "outputs": [] + ] }, { "cell_type": "code", @@ -53,11 +37,7 @@ }, { "cell_type": "code", - "source": [ - "# Note: Not all dependencies have the __version__ method.\n", - "print(f\"torch version: {torch.__version__}\")\n", - "print(f\"torchinfo version: {torchinfo.__version__}\")" - ], + "execution_count": 37, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -65,29 +45,38 @@ "id": "QchOSIbro1zf", "outputId": "94003f5a-622f-4f54-e916-26fbbff86c17" }, - "execution_count": 37, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch version: 2.5.1+cu121\n", "torchinfo version: 1.8.0\n" ] } + ], + "source": [ + "# Note: Not all dependencies have the __version__ method.\n", + "print(f\"torch version: {torch.__version__}\")\n", + "print(f\"torchinfo version: {torchinfo.__version__}\")" ] }, { "cell_type": "markdown", - "source": [ - "**ConvMixer architecture code**\n" - ], "metadata": { "id": "hrG5QYB4pRMu" - } + }, + "source": [ + "**ConvMixer architecture code**\n" + ] }, { "cell_type": "code", + "execution_count": 38, + "metadata": { + "id": "r2d2e2P0pdan" + }, + "outputs": [], "source": [ "class Residual(nn.Module):\n", " def __init__(self, fn):\n", @@ -116,19 +105,11 @@ " nn.Flatten(),\n", " nn.Linear(dim, n_classes)\n", " )" - ], - "metadata": { - "id": "r2d2e2P0pdan" - }, - "execution_count": 38, - "outputs": [] + ] }, { "cell_type": "code", - "source": [ - "model = ConvMixer(2048, 8, kernel_size=9, patch_size=1, n_classes=1000)\n", - "torchinfo.summary(model)" - ], + "execution_count": 41, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -136,10 +117,8 @@ "id": "9YM5zfNoslTx", "outputId": "1b5a1295-d3ac-49c9-d61d-ae986021ad00" }, - "execution_count": 41, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "=================================================================\n", @@ -207,10 +186,29 @@ "=================================================================" ] }, + "execution_count": 41, "metadata": {}, - "execution_count": 41 + "output_type": "execute_result" } + ], + "source": [ + "model = ConvMixer(2048, 8, kernel_size=9, patch_size=1, n_classes=1000)\n", + "torchinfo.summary(model)" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 2949096a5d8848777121ccb01a2f3fd4d9a1b30a Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Sun, 26 Jan 2025 22:59:22 -0500 Subject: [PATCH 3/7] Change the Csv name to lowercase --- automation/notebooks-table-data.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/notebooks-table-data.csv b/automation/notebooks-table-data.csv index e4efca9..b91c8e6 100644 --- a/automation/notebooks-table-data.csv +++ b/automation/notebooks-table-data.csv @@ -14,4 +14,4 @@ GloVe Word Embeddings, data_exploration/glove-word-embeddings.ipynb,https://gith Vision Transformer (ViT),architectures/vit.ipynb,,https://arxiv.org/pdf/2010.11929 Multi-Head Attention, modules/multihead-self-attention.ipynb,,https://arxiv.org/abs/1706.03762 ResNet,architectures/resnet.ipynb,,https://arxiv.org/abs/1512.03385 -convmixer,architectures/ConvMixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file +ConvMixer,architectures/convmixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file From f2b9bc03b6f46f166a76ab40b9685c2867f08fb2 Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Sun, 2 Feb 2025 18:04:27 -0500 Subject: [PATCH 4/7] Cambio temporal para forzar renombrado --- notebooks/architectures/{ConvMixer.ipynb => temp.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notebooks/architectures/{ConvMixer.ipynb => temp.ipynb} (100%) diff --git a/notebooks/architectures/ConvMixer.ipynb b/notebooks/architectures/temp.ipynb similarity index 100% rename from notebooks/architectures/ConvMixer.ipynb rename to notebooks/architectures/temp.ipynb From a3c14a1557d85028ec206aaa84294a850f7c295d Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Sun, 2 Feb 2025 18:04:50 -0500 Subject: [PATCH 5/7] LoweCase file --- notebooks/architectures/{temp.ipynb => convmixer.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notebooks/architectures/{temp.ipynb => convmixer.ipynb} (100%) diff --git a/notebooks/architectures/temp.ipynb b/notebooks/architectures/convmixer.ipynb similarity index 100% rename from notebooks/architectures/temp.ipynb rename to notebooks/architectures/convmixer.ipynb From 845e03f89f4b5a42834710cc33a14e490bb2b393 Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Sun, 2 Feb 2025 18:05:19 -0500 Subject: [PATCH 6/7] only lower case --- notebooks/architectures/convmixer.ipynb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/notebooks/architectures/convmixer.ipynb b/notebooks/architectures/convmixer.ipynb index c8f8680..cf523c4 100644 --- a/notebooks/architectures/convmixer.ipynb +++ b/notebooks/architectures/convmixer.ipynb @@ -2,14 +2,16 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "IJveajFZvdXK" + }, "source": [ - "[![deep-learning-notes](https://github.com/semilleroCV/deep-learning-notes/raw/main/assets/banner-notebook.png)](https://github.com/semilleroCV/deep-learning-notes)" + "## ConvMixer" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": { "id": "q5AuzFB2tA12" }, @@ -23,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": { "id": "118P89a1osHb" }, @@ -37,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -72,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": { "id": "r2d2e2P0pdan" }, @@ -109,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" From d772905f1abb34b98b1653271ca8979a3174d9d5 Mon Sep 17 00:00:00 2001 From: BrayanQuintero123 Date: Wed, 5 Feb 2025 14:29:30 -0500 Subject: [PATCH 7/7] Csv conflicts resolved --- automation/notebooks-table-data.csv | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/notebooks-table-data.csv b/automation/notebooks-table-data.csv index b91c8e6..70b5166 100644 --- a/automation/notebooks-table-data.csv +++ b/automation/notebooks-table-data.csv @@ -14,4 +14,6 @@ GloVe Word Embeddings, data_exploration/glove-word-embeddings.ipynb,https://gith Vision Transformer (ViT),architectures/vit.ipynb,,https://arxiv.org/pdf/2010.11929 Multi-Head Attention, modules/multihead-self-attention.ipynb,,https://arxiv.org/abs/1706.03762 ResNet,architectures/resnet.ipynb,,https://arxiv.org/abs/1512.03385 -ConvMixer,architectures/convmixer.ipynb,,https://arxiv.org/pdf/2201.09792 \ No newline at end of file +ConvMixer,architectures/convmixer.ipynb,,https://arxiv.org/pdf/2201.09792 +Convolutional Block Attention, modules/convolutional-block-attention.ipynb,,https://arxiv.org/abs/1807.06521 +Transformer, architectures/transformer.ipynb,,https://arxiv.org/abs/1706.03762 \ No newline at end of file