Adressing feedback.

Author: debymf

+bert/+tokenizer/+internal/FullTokenizer.m

@@ -93,7 +93,7 @@
                 % Default case
                 this.Basic = bert.tokenizer.internal.BasicTokenizer('IgnoreCase',nvp.IgnoreCase);
             else
-                assert(isa(nvp.BasicTokenizer,'bert.tokenizer.internal.Tokenizer'),"BasicTokenizer must be a bert.tokenizer.internal.Tokenizer implementation.");
+                mustBeA(nvp.BasicTokenizer,'bert.tokenizer.internal.Tokenizer');
                 this.Basic = nvp.BasicTokenizer;
             end
             this.WordPiece = bert.tokenizer.internal.WordPieceTokenizer(vocab);
@@ -106,10 +106,9 @@
             %   tokens = tokenize(tokenizer,text) tokenizes the input
             %   string text using the FullTokenizer specified by tokenizer.
             basicToks = this.Basic.tokenize(txt);
-            basicToksUnicode = cellfun(@textanalytics.unicode.UTF32,basicToks,UniformOutput=false);
             toks = cell(numel(txt),1);
             for i = 1:numel(txt)
-                theseBasicToks = basicToksUnicode{i};
+                theseBasicToks = textanalytics.unicode.UTF32(basicToks{i});
                 theseSubToks = cell(numel(theseBasicToks),1);
                 for j = 1:numel(theseBasicToks)
                     theseSubToks{j} = this.WordPiece.tokenize(theseBasicToks(j));

+bert/+tokenizer/BERTTokenizer.m

@@ -110,7 +110,7 @@
                 ignoreCase = nvp.IgnoreCase;
                 this.FullTokenizer = bert.tokenizer.internal.FullTokenizer(vocabFile,'IgnoreCase',ignoreCase);
             else
-                assert(isa(nvp.FullTokenizer,'bert.tokenizer.internal.FullTokenizer'),"FullTokenizer must be a bert.tokenizer.internal.FullTokenizer.");
+                mustBeA(nvp.FullTokenizer,'bert.tokenizer.internal.FullTokenizer');
                 this.FullTokenizer = nvp.FullTokenizer;
             end
             this.PaddingCode = this.FullTokenizer.encode(this.PaddingToken);

FineTuneBERTJapanese.m

@@ -135,6 +135,7 @@
 
 %%
 % Initialize training progress plot.
+% In 23a you can use trainingProgressMonitor
 figure
 C = colororder;
 lineLossTrain = animatedline("Color",C(2,:));

bert.m

@@ -23,45 +23,9 @@
 
 switch nvp.Model
     case "japanese-base"
-        zipFilePath = bert.internal.getSupportFilePath("japanese-base", "bert-base-japanese.zip");
-        modelDir = fullfile(fileparts(zipFilePath), "bert-base-japanese");      
-        unzip(zipFilePath, modelDir);
-        % Build the tokenizer
-        btok = bert.tokenizer.internal.TokenizedDocumentTokenizer("Language","ja","TokenizeMethod","mecab",IgnoreCase=false);
-        vocabFile = fullfile(modelDir, "vocab.txt");
-        ftok = bert.tokenizer.internal.FullTokenizer(vocabFile,BasicTokenizer=btok);
-        tok = bert.tokenizer.BERTTokenizer(vocabFile,FullTokenizer=ftok);
-        % Build the model
-        params.Weights = load(fullfile(modelDir, "weights.mat"));
-        params.Weights = dlupdate(@dlarray,params.Weights);
-        params.Hyperparameters = struct(...
-            NumHeads=12,...
-            NumLayers=12,...
-            NumContext=512,...
-            HiddenSize=768);
-        mdl = struct(...
-            Tokenizer=tok,...
-            Parameters=params);
+        mdl = iJapaneseBERTModel("japanese-base", "bert-base-japanese.zip");
     case "japanese-base-wwm"
-        zipFilePath = bert.internal.getSupportFilePath("japanese-base", "bert-base-japanese-whole-word-masking.zip");
-        modelDir = fullfile(fileparts(zipFilePath), "bert-base-japanese-whole-word-masking");      
-        unzip(zipFilePath, modelDir);
-        % Build the tokenizer
-        btok = bert.tokenizer.internal.TokenizedDocumentTokenizer("Language","ja","TokenizeMethod","mecab",IgnoreCase=false);
-        vocabFile = fullfile(modelDir, "vocab.txt");
-        ftok = bert.tokenizer.internal.FullTokenizer(vocabFile,BasicTokenizer=btok);
-        tok = bert.tokenizer.BERTTokenizer(vocabFile,FullTokenizer=ftok);
-        % Build the model
-        params.Weights = load(fullfile(modelDir, "weights.mat"));
-        params.Weights = dlupdate(@dlarray,params.Weights);
-        params.Hyperparameters = struct(...
-            NumHeads=12,...
-            NumLayers=12,...
-            NumContext=512,...
-            HiddenSize=768);
-        mdl = struct(...
-            Tokenizer=tok,...
-            Parameters=params);
+        mdl = iJapaneseBERTModel("japanese-base-wwm", "bert-base-japanese-whole-word-masking.zip");
     otherwise
         % Download the license file
         bert.internal.getSupportFilePath(nvp.Model,"bert.RIGHTS");
@@ -76,4 +40,26 @@
             'Tokenizer',bert.tokenizer.BERTTokenizer(vocabFile,'IgnoreCase',ignoreCase),...
             'Parameters',params);
 end
+end
+
+function mdl = iJapaneseBERTModel(modelName, zipFileName)
+zipFilePath = bert.internal.getSupportFilePath(modelName, zipFileName);
+modelDir = fullfile(fileparts(zipFilePath), replace(zipFileName, ".zip", ""));      
+unzip(zipFilePath, modelDir);
+% Build the tokenizer
+btok = bert.tokenizer.internal.TokenizedDocumentTokenizer("Language","ja","TokenizeMethod","mecab",IgnoreCase=false);
+vocabFile = fullfile(modelDir, "vocab.txt");
+ftok = bert.tokenizer.internal.FullTokenizer(vocabFile,BasicTokenizer=btok);
+tok = bert.tokenizer.BERTTokenizer(vocabFile,FullTokenizer=ftok);
+% Build the model
+params.Weights = load(fullfile(modelDir, "weights.mat"));
+params.Weights = dlupdate(@dlarray,params.Weights);
+params.Hyperparameters = struct(...
+    NumHeads=12,...
+    NumLayers=12,...
+    NumContext=512,...
+    HiddenSize=768);
+mdl = struct(...
+    Tokenizer=tok,...
+    Parameters=params);
 end

test/bert/tokenizer/internal/tBasicTokenizer.m

@@ -17,6 +17,15 @@ function canTokenize(test)
             act_out = tok.tokenize(str);
             test.verifyEqual(act_out,exp_out);           
         end
+
+        function canTokenizeBatch(test)
+            tok = bert.tokenizer.internal.BasicTokenizer();
+            manyStrs = repmat("foo bar baz",1,20);
+            act_out = tokenize(tok, manyStrs);
+            exp_out = arrayfun(@(str) tokenize(tok,str),manyStrs,UniformOutput=false);
+            exp_out = [exp_out{:}];
+            test.verifyEqual(act_out,exp_out);           
+        end
 
         function removesControlCharactersAndWhitespace(test)
             tok = bert.tokenizer.internal.BasicTokenizer();

test/tbert.m

@@ -1,19 +1,28 @@
 classdef(SharedTestFixtures = {
-        DownloadBERTFixture}) tbert < matlab.unittest.TestCase
+        DownloadBERTFixture, DownloadJPBERTFixture}) tbert < matlab.unittest.TestCase
     % tbert   System level tests for bert
 
     % Copyright 2021 The MathWorks, Inc.
 
     properties(TestParameter)
         UncasedVersion = {"base", ...
                    "tiny"}
+        AllModels = {"base","multilingual-cased","medium",...
+            "small","mini","tiny","japanese-base",...
+            "japanese-base-wwm"}
     end
 
     methods(Test)
+        
         function canConstructModelWithDefault(test)
             % Verify the default model can be constructed.
             test.verifyWarningFree(@() bert());
         end
+
+        function canConstructAllModels(test, AllModels)
+            % Verify the all available models can be constructed.
+            test.verifyWarningFree(@() bert('Model', AllModels));
+        end
 
         function canConstructModelWithNVPAndVerifyDefault(test)
             % Verify the default model matches the default model.