Initial commit

Author: Niccolo Dal Santo

.gitattributes

@@ -0,0 +1,28 @@
+*.fig binary
+*.mat binary
+*.mdl binary diff merge=mlAutoMerge
+*.mdlp binary
+*.mexa64 binary
+*.mexw64 binary
+*.mexmaci64 binary
+*.mlapp binary
+*.mldatx binary
+*.mlproj binary
+*.mlx binary
+*.p binary
+*.sfx binary
+*.sldd binary
+*.slreqx binary merge=mlAutoMerge
+*.slmx binary merge=mlAutoMerge
+*.sltx binary
+*.slxc binary
+*.slx binary merge=mlAutoMerge
+*.slxp binary
+
+## Other common binary file types
+*.docx binary
+*.exe binary
+*.jpg binary
+*.pdf binary
+*.png binary
+*.xlsx binary

.gitignore

@@ -0,0 +1 @@
+# List of untracked files to ignore

ConvMixer.prj

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<MATLABProject xmlns="http://www.mathworks.com/MATLABProjectFile" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0"/>

README.md

@@ -0,0 +1,31 @@
+# ConvMixer -- Patches are all you need?
+
+This demo shows how to implement and train a ConvMixer architecture for image classification with MATLAB®, as described in the paper "Patches are all you need?" https://openreview.net/forum?id=TVHS5Y4dNvM
+
+The ConvMixer architecture employs a Patch Embedding representation of the input followed by repeated fully-convolutional blocks.
+
+![ConvMixer Architecture](images/convMixer.png)
+
+## How to get started
+
+Start the project ConvMixer.prj to add to the path the relevant functions. There are examples in the `convmixer/examples` folder to get you started with training a ConvMixer for the digits dataset and the CIFAR-10 dataset [1].
+
+The latter employs the ADAM algorithm with fixed weight decay regularization, as described in [2].
+
+Training a ConvMixer for the CIFAR-10 architecture can be demanding in terms of computational resources: in the same `convmixer/examples` folder you can find a pretrained network. This model was trained on the CIFAR-10, available at https://www.cs.toronto.edu/~kriz/cifar-10-matlab.tar.gz
+
+The source code for building the architecture is in the `convmixer/convmixer` directory.
+
+## Requirements
+
+- MATLAB® R2021b or later 
+- Deep Learning Toolbox™
+
+## License
+
+The license is available in the license file within this repository.
+
+Copyright 2021 The MathWorks, Inc.
+
+[1] Krizhevsky, Alex. "Learning multiple layers of features from tiny images." (2009). https://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf
+[2] Loshchilov, Ilya, and Frank Hutter. "Fixing weight decay regularization in ADAM." (2018). https://openreview.net/forum?id=rk6qdGgCZ

SECURITY.md

@@ -0,0 +1,6 @@
+# Reporting Security Vulnerabilities 
+
+If you believe you have discovered a security vulnerability, please report it to 
+[[email protected]](mailto:[email protected]). Please see 
+[MathWorks Vulnerability Disclosure Policy for Security Researchers](https://www.mathworks.com/company/aboutus/policies_statements/vulnerability-disclosure-policy.html) 
+for additional information.

convmixer/convMixerLayers.m

@@ -0,0 +1,101 @@
+function lgraph = convMixerLayers(opts)
+% convMixerLayers   Build ConvMixer architecture.
+%
+% lgraph = convMixerLayers() returns a LayerGraph object with a ConvMixer
+% architecture with default options as proposed in
+% https://openreview.net/forum?id=TVHS5Y4dNvM.
+%
+% lgraph = convMixerLayers(PARAM1=VAL1,PARAM2=VAL2,...) specifies optional
+% parameter name/value pairs for creating the layer graph:
+% 
+%       'InputSize'           - Size of the input images.
+%
+%       'NumClasses'          - Number of classes the network predicts.
+%
+%       'KernelSize'          - Size of the kernel for the depthwise
+%                               convolution.
+%
+%       'PatchSize'           - Size of the pathes for the patch embedding
+%                               layer.
+%
+%       'Depth'               - Number of repeated fully-convolutional
+%                               blocks.
+%
+%       'HiddenDimension'     - Number of channels output by the patch 
+%                               embedding.
+%
+%       'ConnectOutputLayer'  - Determines whether to append a softmax and
+%                               classification output layers to the
+%                               returned LayerGraph object.
+%
+% Example:
+% 
+%   lgraph = convMixerLayers(InputSize=[28 28 1], Depth=5, NumClasses=10)
+
+% Copyright 2021 The MathWorks, Inc.
+
+arguments
+    opts.InputSize = [227 227 3]
+    opts.NumClasses = 1000
+    opts.KernelSize = 9
+    opts.PatchSize = 7
+    opts.Depth = 20
+    opts.HiddenDimension = 1536
+    opts.ConnectOutputLayer logical = false
+end
+
+input_size = opts.InputSize;
+num_classes = opts.NumClasses;
+
+kernel_size = opts.KernelSize;
+patch_size = opts.PatchSize;
+depth = opts.Depth;
+hidden_dim = opts.HiddenDimension;
+connectOutputLayers = opts.ConnectOutputLayer;
+
+% First layer is a "path embedding". Seems to be this:
+patchEmbedding = convolution2dLayer(patch_size, hidden_dim, ...
+    Stride=patch_size, ...
+    Name="patchEmbedding", ...
+    WeightsInitializer="glorot");
+
+% Make Layer Graph
+lgraph = layerGraph();
+
+start = [
+    imageInputLayer(input_size,Normalization="none")
+    patchEmbedding
+    geluLayer(Name="gelu_0")
+    batchNormalizationLayer(Name="batchnorm_0")
+    ];
+lgraph = addLayers(lgraph,start);
+
+for i = 1:depth
+    convMixer = [
+        groupedConvolution2dLayer(kernel_size,1,"channel-wise",Name="depthwiseConv_"+i,Padding="same",WeightsInitializer="glorot")
+        geluLayer(Name="gelu_"+(2*i-1))
+        batchNormalizationLayer(Name="batchnorm_"+(2*i-1))
+        additionLayer(2,Name="addition_"+i)
+        convolution2dLayer([1 1],hidden_dim,Name="pointwiseConv_"+i,WeightsInitializer="glorot")
+        geluLayer(Name="gelu_"+2*i)
+        batchNormalizationLayer(Name="batchnorm_"+2*i)
+        ];
+    lgraph = addLayers(lgraph,convMixer);
+    lgraph = connectLayers(lgraph,"batchnorm_"+2*(i-1),"depthwiseConv_"+i);
+    lgraph = connectLayers(lgraph,"batchnorm_"+2*(i-1),"addition_"+i+"/in2");
+end
+
+gapFc = [
+    globalAveragePooling2dLayer(Name="GAP")
+    fullyConnectedLayer(num_classes)
+    ];
+lgraph = addLayers(lgraph,gapFc);
+lgraph = connectLayers(lgraph,"batchnorm_"+2*depth,"GAP");
+
+if connectOutputLayers
+    lgraph = addLayers(lgraph, softmaxLayer('Name','softmax'));
+    lgraph = addLayers(lgraph, classificationLayer('Name','classification'));
+    lgraph = connectLayers(lgraph,'fc','softmax');
+    lgraph = connectLayers(lgraph,'softmax','classification');
+end
+end

convmixer/geluLayer.m

@@ -0,0 +1,46 @@
+classdef geluLayer < nnet.layer.Layer
+% geluLayer   GELU layer.
+%
+% gLayer = geluLayer() returns a geluLayer object.
+%
+% gLayer = geluLayer(PARAM1=VAL1,PARAM2=VAL2,...) specifies optional
+% parameter name/value pairs for creating the layer graph:
+% 
+%       'Mode'           - Size of the input images. Options are 'fast
+%
+%       'Name'           - Name of the layer.
+%
+%    See https://paperswithcode.com/method/gelu for details.
+%
+% Example:
+% 
+%   gLayer = geluLayer()
+
+%    Copyright 2021 The MathWorks, Inc.
+
+    properties(SetAccess='private')
+        Mode
+    end
+
+    methods
+        function obj = geluLayer(opts)
+            arguments
+                opts.Mode string {mustBeMember(opts.Mode,["fast", "exact"])} = "fast";
+                opts.Name string {mustBeText} = "gelu";
+            end
+            obj.Name = opts.Name;
+            obj.Mode = opts.Mode;
+        end
+
+        function y = predict(obj,x)
+            switch obj.Mode
+                case "exact"
+                    y = x/2.*(1+erf(x/sqrt(2)));
+                case "fast"
+                    y = x/2.*(1+tanh(sqrt(2/pi)*(x+0.044715*x.^3)));
+                otherwise
+                    error Unknown
+            end
+        end
+    end
+end

data/downloadCIFARData.m

@@ -0,0 +1,15 @@
+function downloadCIFARData(destination)
+% downloadCIFARData   Download CIFAR-10 dataset.
+
+% Copyright 2021 The MathWorks, Inc.
+
+url = 'https://www.cs.toronto.edu/~kriz/cifar-10-matlab.tar.gz';
+
+unpackedData = fullfile(destination,'cifar-10-batches-mat');
+if ~exist(unpackedData,'dir')
+    fprintf('Downloading CIFAR-10 dataset (175 MB). This can take a while...');
+    untar(url,destination);
+    fprintf('done.\n\n');
+end
+
+end

data/loadCIFARData.m

@@ -0,0 +1,32 @@
+function [XTrain,YTrain,XTest,YTest] = loadCIFARData(location)
+% loadCIFARData   Load CIFAR-10 dataset and split it to training and
+% validation sets.
+
+% Copyright 2021 The MathWorks, Inc.
+
+location = fullfile(location,'cifar-10-batches-mat');
+
+[XTrain1,YTrain1] = loadBatchAsFourDimensionalArray(location,'data_batch_1.mat');
+[XTrain2,YTrain2] = loadBatchAsFourDimensionalArray(location,'data_batch_2.mat');
+[XTrain3,YTrain3] = loadBatchAsFourDimensionalArray(location,'data_batch_3.mat');
+[XTrain4,YTrain4] = loadBatchAsFourDimensionalArray(location,'data_batch_4.mat');
+[XTrain5,YTrain5] = loadBatchAsFourDimensionalArray(location,'data_batch_5.mat');
+XTrain = cat(4,XTrain1,XTrain2,XTrain3,XTrain4,XTrain5);
+YTrain = [YTrain1;YTrain2;YTrain3;YTrain4;YTrain5];
+
+[XTest,YTest] = loadBatchAsFourDimensionalArray(location,'test_batch.mat');
+end
+
+function [XBatch,YBatch] = loadBatchAsFourDimensionalArray(location,batchFileName)
+s = load(fullfile(location,batchFileName));
+XBatch = s.data';
+XBatch = reshape(XBatch,32,32,3,[]);
+XBatch = permute(XBatch,[2 1 3 4]);
+YBatch = convertLabelsToCategorical(location,s.labels);
+end
+
+function categoricalLabels = convertLabelsToCategorical(location,integerLabels)
+s = load(fullfile(location,'batches.meta.mat'));
+categoricalLabels = categorical(integerLabels,0:9,s.label_names);
+end
+