Ensure policies in policy.json are compliant with PEP600 (#287)

This removes 2 non existing symbols from manylinux1 i686 policy and removes ncurses librairies from manylinux1 whitelist.

Author: mayeut

auditwheel/policy/__init__.py

@@ -1,7 +1,8 @@
 import sys
 import json
 import platform as _platform_module
-from typing import Optional, List
+from collections import defaultdict
+from typing import Dict, List, Optional, Set
 from os.path import join, dirname, abspath
 import logging
 
@@ -23,9 +24,42 @@ def get_arch_name() -> str:
 _ARCH_NAME = get_arch_name()
 
 
+def _validate_pep600_compliance(policies) -> None:
+    symbol_versions: Dict[str, Dict[str, Set[str]]] = {}
+    lib_whitelist: Set[str] = set()
+    for policy in sorted(policies, key=lambda x: x['priority'], reverse=True):
+        if policy['name'] == 'linux':
+            continue
+        if not lib_whitelist.issubset(set(policy['lib_whitelist'])):
+            diff = lib_whitelist - set(policy["lib_whitelist"])
+            raise ValueError(
+                'Invalid "policy.json" file. Missing whitelist libraries in '
+                f'"{policy["name"]}" compared to previous policies: {diff}'
+            )
+        lib_whitelist.update(policy['lib_whitelist'])
+        for arch in policy['symbol_versions'].keys():
+            symbol_versions_arch = symbol_versions.get(arch, defaultdict(set))
+            for prefix in policy['symbol_versions'][arch].keys():
+                policy_symbol_versions = set(
+                    policy['symbol_versions'][arch][prefix])
+                if not symbol_versions_arch[prefix].issubset(
+                        policy_symbol_versions):
+                    diff = symbol_versions_arch[prefix] - \
+                           policy_symbol_versions
+                    raise ValueError(
+                        'Invalid "policy.json" file. Symbol versions missing '
+                        f'in "{policy["name"]}_{arch}" for "{prefix}" '
+                        f'compared to previous policies: {diff}'
+                    )
+                symbol_versions_arch[prefix].update(
+                    policy['symbol_versions'][arch][prefix])
+            symbol_versions[arch] = symbol_versions_arch
+
+
 with open(join(dirname(abspath(__file__)), 'policy.json')) as f:
     _POLICIES = []
     _policies_temp = json.load(f)
+    _validate_pep600_compliance(_policies_temp)
     for _p in _policies_temp:
         if _ARCH_NAME in _p['symbol_versions'].keys() or _p['name'] == 'linux':
             if _p['name'] != 'linux':

auditwheel/policy/policy.json

@@ -9,8 +9,8 @@
      "symbol_versions": {
          "i686": {
            "CXXABI": ["1.3", "1.3.1"],
-           "GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "3.4.4", "4.0.0", "4.2.0"],
-           "GLIBC": ["2.0", "2.1", "2.1.1", "2.1.2", "2.1.3", "2.2", "2.2.1", "2.2.2", "2.2.3", "2.2.4", "2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", "2.5"],
+           "GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "4.0.0", "4.2.0"],
+           "GLIBC": ["2.0", "2.1", "2.1.1", "2.1.2", "2.1.3", "2.2", "2.2.1", "2.2.2", "2.2.3", "2.2.4", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", "2.5"],
            "GLIBCXX": ["3.4", "3.4.1", "3.4.2", "3.4.3", "3.4.4", "3.4.5", "3.4.6", "3.4.7", "3.4.8"]
          },
          "x86_64": {
@@ -21,7 +21,6 @@
          }
      },
      "lib_whitelist": [
-         "libpanelw.so.5", "libncursesw.so.5",
          "libgcc_s.so.1",
          "libstdc++.so.6",
          "libm.so.6", "libdl.so.2", "librt.so.1",

tests/unit/test_policy.py

@@ -3,7 +3,7 @@
 import pytest
 
 from auditwheel.policy import get_arch_name, get_policy_name, \
-    get_priority_by_name, get_replace_platforms
+    get_priority_by_name, get_replace_platforms, _validate_pep600_compliance
 
 
 @patch("auditwheel.policy._platform_module.machine")
@@ -43,6 +43,59 @@ def test_replacement_platform(name, expected):
     assert get_replace_platforms(name) == expected
 
 
+def test_pep600_compliance():
+    _validate_pep600_compliance([{
+        "name": "manylinux1", "priority": 100, "symbol_versions": {
+            "i686": {"CXXABI": ["1.3"]},
+        },
+        "lib_whitelist": ["libgcc_s.so.1"]
+    }, {
+        "name": "manylinux2010", "priority": 90, "symbol_versions": {
+            "i686": {"CXXABI": ["1.3", "1.3.1"]},
+        },
+        "lib_whitelist": ["libgcc_s.so.1", "libstdc++.so.6"],
+    }])
+
+    _validate_pep600_compliance([{
+        "name": "manylinux1", "priority": 100, "symbol_versions": {
+            "i686": {"CXXABI": ["1.3"]},
+            "x86_64": {"CXXABI": ["1.3"]},
+        },
+        "lib_whitelist": ["libgcc_s.so.1"]
+    }, {
+        "name": "manylinux2010", "priority": 90, "symbol_versions": {
+            "i686": {"CXXABI": ["1.3", "1.3.1"]},
+        },
+        "lib_whitelist": ["libgcc_s.so.1", "libstdc++.so.6"],
+    }])
+
+    with pytest.raises(ValueError, match="manylinux2010_i686.*CXXABI.*1.3.2"):
+        _validate_pep600_compliance([{
+            "name": "manylinux1", "priority": 100, "symbol_versions": {
+                "i686": {"CXXABI": ["1.3", "1.3.2"]},
+            },
+            "lib_whitelist": ["libgcc_s.so.1"]
+        }, {
+            "name": "manylinux2010", "priority": 90, "symbol_versions": {
+                "i686": {"CXXABI": ["1.3", "1.3.1"]},
+            },
+            "lib_whitelist": ["libgcc_s.so.1", "libstdc++.so.6"],
+        }])
+
+    with pytest.raises(ValueError, match="manylinux2010.*libstdc\+\+\.so\.6"):
+        _validate_pep600_compliance([{
+            "name": "manylinux1", "priority": 100, "symbol_versions": {
+                "i686": {"CXXABI": ["1.3"]},
+            },
+            "lib_whitelist": ["libgcc_s.so.1", "libstdc++.so.6"]
+        }, {
+            "name": "manylinux2010", "priority": 90, "symbol_versions": {
+                "i686": {"CXXABI": ["1.3", "1.3.1"]},
+            },
+            "lib_whitelist": ["libgcc_s.so.1"],
+        }])
+
+
 class TestPolicyAccess:
 
     def test_get_by_priority(self):