Improve the Type-1 font parsing

Move Type1Font._tokens into a top-level function _tokenize that is a
coroutine. The parsing stage consuming the tokens can instruct the
tokenizer to return a binary token - this is necessary when decrypting
the CharStrings and Subrs arrays, since the preceding context determines
which parts of the data need to be decrypted.

The function now also parses the encrypted portion of the font file.

To support usage as a coroutine, move the whitespace filtering into the
function, since passing the information about binary tokens would not
easily work through a filter.

The function now returns tokens as subclasses of a new _Token class,
which carry the position and value of the token and can have
token-specific helper methods. The position data will be needed when
modifying the file, as the font is transformed or subsetted.

A new helper function _expression can be used to consume tokens that
form a balanced subexpression delimited by [] or {}. This helps fix a
bug in UniqueID removal: if the font includes PostScript code that
checks if the UniqueID is set in the current dictionary, the previous
code broke that code instead of removing the UniqueID definition. Fonts
can include UniqueID in the encrypted portion as well as the cleartext
one, and removal is now done in both portions.

Fix a bug related to font weight: the key is title-cased and not
lower-cased, so font.prop['weight'] should not exist.

Author: jkseppan

LICENSE/LICENSE_COURIERTEN

@@ -0,0 +1,18 @@
+The Courier10PitchBT-Bold.pfb file is a Type-1 version of
+Courier 10 Pitch BT Bold by Bitstream, obtained from
+<https://ctan.org/tex-archive/fonts/courierten>. It is included
+here as test data only, but the following license applies.
+
+
+(c) Copyright 1989-1992, Bitstream Inc., Cambridge, MA.
+
+You are hereby granted permission under all Bitstream propriety rights
+to use, copy, modify, sublicense, sell, and redistribute the 4 Bitstream
+Charter (r) Type 1 outline fonts and the 4 Courier Type 1 outline fonts
+for any purpose and without restriction; provided, that this notice is
+left intact on all copies of such fonts and that Bitstream's trademark
+is acknowledged as shown below on all unmodified copies of the 4 Charter
+Type 1 fonts.
+
+BITSTREAM CHARTER is a registered trademark of Bitstream Inc.
+

doc/api/next_api_changes/behavior/20715-JKS.rst

@@ -0,0 +1,8 @@
+``Type1Font`` objects include more properties
+---------------------------------------------
+
+The `.type1font.Type1Font.prop` dictionary now includes more keys, such
+as ``CharStrings`` and ``Subrs``. The value of the ``Encoding`` key is
+now a dictionary mapping codes to glyph names. The
+`.type1font.Type1Font.transform` method now correctly removes
+``UniqueID`` properties from the font.

lib/matplotlib/tests/Courier10PitchBT-Bold.pfb

@@ -1,6 +1,7 @@
 import matplotlib.type1font as t1f
 import os.path
 import difflib
+import pytest
 
 
 def test_Type1Font():
@@ -13,25 +14,48 @@ def test_Type1Font():
     assert font.parts[0] == rawdata[0x0006:0x10c5]
     assert font.parts[1] == rawdata[0x10cb:0x897f]
     assert font.parts[2] == rawdata[0x8985:0x8ba6]
-    assert font.parts[1:] == slanted.parts[1:]
-    assert font.parts[1:] == condensed.parts[1:]
     assert font.decrypted.startswith(b'dup\n/Private 18 dict dup begin')
     assert font.decrypted.endswith(b'mark currentfile closefile\n')
+    assert slanted.decrypted.startswith(b'dup\n/Private 18 dict dup begin')
+    assert slanted.decrypted.endswith(b'mark currentfile closefile\n')
+    assert b'UniqueID 5000793' in font.parts[0]
+    assert b'UniqueID 5000793' in font.decrypted
+    assert font._pos['UniqueID'] == [(797, 818), (4483, 4504)]
+
+    len0 = len(font.parts[0])
+    for key in font._pos.keys():
+        for pos0, pos1 in font._pos[key]:
+            if pos0 < len0:
+                data = font.parts[0][pos0:pos1]
+            else:
+                data = font.decrypted[pos0-len0:pos1-len0]
+            assert data.startswith(f'/{key}'.encode('ascii'))
+    assert {'FontType', 'FontMatrix', 'PaintType', 'ItalicAngle', 'RD'
+            } < set(font._pos.keys())
+
+    assert b'UniqueID 5000793' not in slanted.parts[0]
+    assert b'UniqueID 5000793' not in slanted.decrypted
+    assert 'UniqueID' not in slanted._pos
+    assert font.prop['Weight'] == 'Medium'
+    assert not font.prop['isFixedPitch']
+    assert font.prop['ItalicAngle'] == 0
+    assert slanted.prop['ItalicAngle'] == -45
+    assert font.prop['Encoding'][5] == 'Pi'
+    assert isinstance(font.prop['CharStrings']['Pi'], bytes)
 
     differ = difflib.Differ()
     diff = list(differ.compare(
         font.parts[0].decode('latin-1').splitlines(),
         slanted.parts[0].decode('latin-1').splitlines()))
     for line in (
          # Removes UniqueID
-         '- FontDirectory/CMR10 known{/CMR10 findfont dup/UniqueID known{dup',
-         '+ FontDirectory/CMR10 known{/CMR10 findfont dup',
+         '- /UniqueID 5000793 def',
          # Changes the font name
          '- /FontName /CMR10 def',
-         '+ /FontName /CMR10_Slant_1000 def',
+         '+ /FontName/CMR10_Slant_1000 def',
          # Alters FontMatrix
          '- /FontMatrix [0.001 0 0 0.001 0 0 ]readonly def',
-         '+ /FontMatrix [0.001 0 0.001 0.001 0 0]readonly def',
+         '+ /FontMatrix [0.001 0 0.001 0.001 0 0] readonly def',
          # Alters ItalicAngle
          '-  /ItalicAngle 0 def',
          '+  /ItalicAngle -45.0 def'):
@@ -42,17 +66,72 @@ def test_Type1Font():
         condensed.parts[0].decode('latin-1').splitlines()))
     for line in (
          # Removes UniqueID
-         '- FontDirectory/CMR10 known{/CMR10 findfont dup/UniqueID known{dup',
-         '+ FontDirectory/CMR10 known{/CMR10 findfont dup',
+         '- /UniqueID 5000793 def',
          # Changes the font name
          '- /FontName /CMR10 def',
-        '+ /FontName /CMR10_Extend_500 def',
+         '+ /FontName/CMR10_Extend_500 def',
          # Alters FontMatrix
          '- /FontMatrix [0.001 0 0 0.001 0 0 ]readonly def',
-         '+ /FontMatrix [0.0005 0 0 0.001 0 0]readonly def'):
+         '+ /FontMatrix [0.0005 0 0 0.001 0 0] readonly def'):
         assert line in diff, 'diff to condensed font must contain %s' % line
 
 
+def test_Type1Font_2():
+    filename = os.path.join(os.path.dirname(__file__),
+                            'Courier10PitchBT-Bold.pfb')
+    font = t1f.Type1Font(filename)
+    assert font.prop['Weight'] == 'Bold'
+    assert font.prop['isFixedPitch']
+    assert font.prop['Encoding'][65] == 'A'  # the font uses StandardEncoding
+    (pos0, pos1), = font._pos['Encoding']
+    assert font.parts[0][pos0:pos1] == b'/Encoding StandardEncoding'
+
+
+def test_tokenize():
+    data = (b'1234/abc false -9.81  Foo <<[0 1 2]<0 1ef a\t>>>\n'
+            b'(string with(nested\t\\) par)ens\\\\)')
+    #         1           2          x    2     xx1
+    # 1 and 2 are matching parens, x means escaped character
+    n, w, num, kw, d = 'name', 'whitespace', 'number', 'keyword', 'delimiter'
+    b, s = 'boolean', 'string'
+    correct = [
+        (num, 1234), (n, 'abc'), (w, ' '), (b, False), (w, ' '), (num, -9.81),
+        (w, '  '), (kw, 'Foo'), (w, ' '), (d, '<<'), (d, '['), (num, 0),
+        (w, ' '), (num, 1), (w, ' '), (num, 2), (d, ']'), (s, b'\x01\xef\xa0'),
+        (d, '>>'), (w, '\n'), (s, 'string with(nested\t) par)ens\\')
+    ]
+    correct_no_ws = [x for x in correct if x[0] != w]
+
+    def convert(tokens):
+        return [(t.kind, t.value()) for t in tokens]
+
+    assert convert(t1f._tokenize(data, False)) == correct
+    assert convert(t1f._tokenize(data, True)) == correct_no_ws
+
+    def bin_after(n):
+        tokens = t1f._tokenize(data, True)
+        result = []
+        for _ in range(n):
+            result.append(next(tokens))
+        result.append(tokens.send(10))
+        return convert(result)
+
+    for n in range(1, len(correct_no_ws)):
+        result = bin_after(n)
+        assert result[:-1] == correct_no_ws[:n]
+        assert result[-1][0] == 'binary'
+        assert isinstance(result[-1][1], bytes)
+
+
+def test_tokenize_errors():
+    with pytest.raises(ValueError):
+        list(t1f._tokenize(b'1234 (this (string) is unterminated\\)', True))
+    with pytest.raises(ValueError):
+        list(t1f._tokenize(b'/Foo<01234', True))
+    with pytest.raises(ValueError):
+        list(t1f._tokenize(b'/Foo<01234abcg>/Bar', True))
+
+
 def test_overprecision():
     # We used to output too many digits in FontMatrix entries and
     # ItalicAngle, which could make Type-1 parsers unhappy.