Handle misencoded license text files graceful. (#884)

---------

Signed-off-by: Michael Schlenker <[email protected]>
Signed-off-by: Jan Kowalleck <[email protected]>
Co-authored-by: Michael Schlenker <[email protected]>
Co-authored-by: Jan Kowalleck <[email protected]>

Author: 3 people

cyclonedx_py/_internal/utils/bytes.py

@@ -0,0 +1,28 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+from sys import getdefaultencoding
+
+from chardet import detect as chardetect
+
+
+def bytes2str(data: bytes, *, errors: str = 'strict') -> str:
+    # see https://docs.python.org/3/library/codecs.html#standard-encodings
+    encoding = (chardetect(data)['encoding'] or getdefaultencoding()).replace(
+        # replace Windows-encoding with code-page
+        'Windows-', 'cp')
+    return data.decode(encoding, errors)

cyclonedx_py/_internal/utils/io.py

@@ -15,20 +15,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
-from sys import getdefaultencoding
 from tempfile import NamedTemporaryFile
 from typing import BinaryIO
 
-from chardet import detect as chardetect
+from .bytes import bytes2str
 
 
 def io2str(io: BinaryIO, *, errors: str = 'strict') -> str:
-    data = io.read()
-    # see https://docs.python.org/3/library/codecs.html#standard-encodings
-    encoding = (chardetect(data)['encoding'] or getdefaultencoding()).replace(
-        # replace Windows-encoding with code-page
-        'Windows-', 'cp')
-    return data.decode(encoding, errors)
+    return bytes2str(io.read(), errors=errors)
 
 
 def io2file(io: BinaryIO, *, errors: str = 'strict') -> str:

cyclonedx_py/_internal/utils/pep639.py

@@ -30,6 +30,7 @@
 from cyclonedx.model import AttachedText, Encoding
 from cyclonedx.model.license import DisjunctiveLicense, LicenseAcknowledgement
 
+from .bytes import bytes2str
 from .mimetypes import guess_type
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -38,6 +39,10 @@
 
     from cyclonedx.model.license import License
 
+# per spec > license files are stored in the `.dist-info/licenses/` subdirectory of the produced wheel.
+# but in practice, other locations are used, too.
+_LICENSE_LOCATIONS = ('licenses', 'license_files', '')
+
 
 def dist2licenses(
     dist: 'Distribution',
@@ -55,12 +60,20 @@ def dist2licenses(
         for mlfile in set(metadata.get_all('License-File', ())):
             # see spec: https://peps.python.org/pep-0639/#add-license-file-field
             # latest spec rev: https://discuss.python.org/t/pep-639-round-3-improving-license-clarity-with-better-package-metadata/53020  # noqa: E501
-
-            # per spec > license files are stored in the `.dist-info/licenses/` subdirectory of the produced wheel.
-            # but in practice, other locations are used, too.
-            content = dist.read_text(join('licenses', mlfile)) \
-                or dist.read_text(join('license_files', mlfile)) \
-                or dist.read_text(mlfile)
+            content = None
+            for mlpath in _LICENSE_LOCATIONS:
+                try:
+                    content = dist.read_text(join(mlpath, mlfile))
+                except UnicodeDecodeError as err:
+                    try:
+                        content = bytes2str(err.object)
+                    except UnicodeDecodeError:
+                        pass
+                    else:
+                        break  # for-loop
+                else:
+                    if content is not None:
+                        break  # for-loop
             if content is None:  # pragma: no cover
                 logger.debug('Error: failed to read license file %r for dist %r',
                              mlfile, metadata['Name'])

tests/_data/infiles/_helpers/local_pckages/with-license-pep639_regression-issue868/.editorconfig

@@ -0,0 +1,11 @@
+# EditorConfig is awesome: https://editorconfig.org
+
+[my_licenses/utf-8*]
+charset = utf-8
+
+[my_licenses/utf-16le*]
+charset = utf-16le
+
+[my_licenses/utf-16be*]
+charset = utf-16be
+

tests/_data/infiles/_helpers/local_pckages/with-license-pep639_regression-issue868/.gitattributes

@@ -0,0 +1,2 @@
+Licenses/*      binary
+Licenses/*.txt  binary diff=txt

tests/_data/infiles/_helpers/local_pckages/with-license-pep639_regression-issue868/README.md

@@ -0,0 +1,6 @@
+# PEP 639 - regression 868
+
+see <https://github.com/CycloneDX/cyclonedx-python/issues/868>
+
+PEP-630 expects license gfiles to be UTF8 encoded text.
+some license files may not be text, some may not be UTF8 encoded, but still be added as license files.

tests/_data/infiles/_helpers/local_pckages/with-license-pep639_regression-issue868/my_licenses/richtext.rtf

@@ -0,0 +1,4 @@
+this file is
+utf-8 encoded
+without BOM
+😃