Fix tokenizers w/ padding: true and non-null max_length (#1284)

* tokenizers respect padding: true with non-null max_length

This commit changes the behavior of tokenizers to match the
behavior described in the docs and the behavior of the Python
library.

Before this commit, passing
{
  padding: true,
  max_length: 512
}
or
{
  padding: 'max_length',
  max_length: 512
}
would both always pad all outputs to 512 tokens.

After this change,
{
  padding: true,
  max_length: 512
}
will now pad the outputs to match the longest encoding
or max_length, whichever is shorter.

This commit  also adds a test to prevent regressions.

* Revamp tokenizer padding/truncation test suite

* Fix tokenization padding/truncation logic

* nit

---------

Co-authored-by: Joshua Lochner <[email protected]>

Author: dwisdom0

src/tokenizers.js

@@ -2806,22 +2806,29 @@ export class PreTrainedTokenizer extends Callable {
             // For single input, we just wrap in an array, and then unwrap later.
             encodedTokens = [this._encode_plus(text, { text_pair, add_special_tokens, return_token_type_ids })];
         }
-        // At this point, tokens is batched: [batch_size, tokens]
-        // However, array may be jagged. So, we pad to max_length
-
+        // At this point, `encodedTokens` is batched, of shape [batch_size, tokens].
+        // However, array may be jagged. So, we may need pad to max_length.
         if (max_length === null) {
-            if (padding === 'max_length') {
+            max_length = this.model_max_length;
+        } else if (truncation === null) {
+            if (padding === true) {
+                console.warn(
+                    "`max_length` is ignored when `padding: true` and there is no truncation strategy. " +
+                    "To pad to max length, use `padding: 'max_length'`."
+                )
                 max_length = this.model_max_length;
-            } else {
-                // Calculate max length from sequences
-                max_length = max(encodedTokens.map(x => x.input_ids.length))[0];
-            }
-        } else {
-            if (!truncation) {
-                console.warn(`Truncation was not explicitly activated but \`max_length\` is provided a specific value, please use \`truncation=true\` to explicitly truncate examples to max length.`)
+            } else if (padding === false) {
+                console.warn("Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation: true` to explicitly truncate examples to max length.");
+                truncation = true;
             }
         }
 
+        // padding: 'max_length' doesn't require any additional calculation
+        // but padding: true has to calculate max_length from the sequences
+        if (padding === true) {
+            max_length = Math.min(max(encodedTokens.map(x => x.input_ids.length))[0], max_length ?? Infinity);
+        }
+
         // Ensure it is less than model max length
         max_length = Math.min(max_length, this.model_max_length ?? Infinity);