Skip to content

Commit 7b4f77b

Browse files
committed
Remove globals in policy module
The policy module creates several globals on the module level, which makes it very difficult to test code which interacts with the policies. This patch creates a `WheelPolicies` class which encapsulates the policy processing code and exposes functions to query the parsed policies. Unfortunately, a lot of functions directly operate on policies and these all need a new argument to take the new policy object. This change should make testing substantially easier as policies can be easily created on the fly and passed down to functions which require these.
1 parent 1a1ee6c commit 7b4f77b

File tree

11 files changed

+210
-183
lines changed

11 files changed

+210
-183
lines changed

src/auditwheel/main_repair.py

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,15 @@
66

77
from auditwheel.patcher import Patchelf
88

9-
from .policy import (
10-
POLICY_PRIORITY_HIGHEST,
11-
get_policy_by_name,
12-
get_policy_name,
13-
get_priority_by_name,
14-
load_policies,
15-
)
9+
from .policy import WheelPolicies
1610
from .tools import EnvironmentDefault
1711

1812
logger = logging.getLogger(__name__)
1913

2014

2115
def configure_parser(sub_parsers):
22-
policies = load_policies()
16+
wheel_policy = WheelPolicies.load()
17+
policies = wheel_policy.policies
2318
policy_names = [p["name"] for p in policies]
2419
policy_names += [alias for p in policies for alias in p["aliases"]]
2520
epilog = """PLATFORMS:
@@ -32,7 +27,7 @@ def configure_parser(sub_parsers):
3227
if len(p["aliases"]) > 0:
3328
epilog += f" (aliased by {', '.join(p['aliases'])})"
3429
epilog += "\n"
35-
highest_policy = get_policy_name(POLICY_PRIORITY_HIGHEST)
30+
highest_policy = wheel_policy.get_policy_name(wheel_policy.priority_highest)
3631
help = """Vendor in external shared library dependencies of a wheel.
3732
If multiple wheels are specified, an error processing one
3833
wheel will abort processing of subsequent wheels.
@@ -114,6 +109,8 @@ def execute(args, p):
114109
from .repair import repair_wheel
115110
from .wheel_abi import NonPlatformWheel, analyze_wheel_abi
116111

112+
wheel_policy = WheelPolicies.load()
113+
117114
for wheel_file in args.WHEEL_FILE:
118115
if not isfile(wheel_file):
119116
p.error("cannot access %s. No such file" % wheel_file)
@@ -124,23 +121,23 @@ def execute(args, p):
124121
os.makedirs(args.WHEEL_DIR)
125122

126123
try:
127-
wheel_abi = analyze_wheel_abi(wheel_file)
124+
wheel_abi = analyze_wheel_abi(wheel_policy, wheel_file)
128125
except NonPlatformWheel:
129126
logger.info(NonPlatformWheel.LOG_MESSAGE)
130127
return 1
131128

132-
policy = get_policy_by_name(args.PLAT)
129+
policy = wheel_policy.get_policy_by_name(args.PLAT)
133130
reqd_tag = policy["priority"]
134131

135-
if reqd_tag > get_priority_by_name(wheel_abi.sym_tag):
132+
if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.sym_tag):
136133
msg = (
137134
'cannot repair "%s" to "%s" ABI because of the presence '
138135
"of too-recent versioned symbols. You'll need to compile "
139136
"the wheel on an older toolchain." % (wheel_file, args.PLAT)
140137
)
141138
p.error(msg)
142139

143-
if reqd_tag > get_priority_by_name(wheel_abi.ucs_tag):
140+
if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.ucs_tag):
144141
msg = (
145142
'cannot repair "%s" to "%s" ABI because it was compiled '
146143
"against a UCS2 build of Python. You'll need to compile "
@@ -149,7 +146,7 @@ def execute(args, p):
149146
)
150147
p.error(msg)
151148

152-
if reqd_tag > get_priority_by_name(wheel_abi.blacklist_tag):
149+
if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.blacklist_tag):
153150
msg = (
154151
'cannot repair "%s" to "%s" ABI because it depends on '
155152
"black-listed symbols." % (wheel_file, args.PLAT)
@@ -158,7 +155,7 @@ def execute(args, p):
158155

159156
abis = [policy["name"]] + policy["aliases"]
160157
if not args.ONLY_PLAT:
161-
if reqd_tag < get_priority_by_name(wheel_abi.overall_tag):
158+
if reqd_tag < wheel_policy.get_priority_by_name(wheel_abi.overall_tag):
162159
logger.info(
163160
(
164161
"Wheel is eligible for a higher priority tag. "
@@ -168,11 +165,12 @@ def execute(args, p):
168165
args.PLAT,
169166
wheel_abi.overall_tag,
170167
)
171-
higher_policy = get_policy_by_name(wheel_abi.overall_tag)
168+
higher_policy = wheel_policy.get_policy_by_name(wheel_abi.overall_tag)
172169
abis = [higher_policy["name"]] + higher_policy["aliases"] + abis
173170

174171
patcher = Patchelf()
175172
out_wheel = repair_wheel(
173+
wheel_policy,
176174
wheel_file,
177175
abis=abis,
178176
lib_sdir=args.LIB_SDIR,

src/auditwheel/main_show.py

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import logging
44

5+
from auditwheel.policy import WheelPolicies
6+
57
logger = logging.getLogger(__name__)
68

79

@@ -23,22 +25,17 @@ def execute(args, p):
2325
import json
2426
from os.path import basename, isfile
2527

26-
from .policy import (
27-
POLICY_PRIORITY_HIGHEST,
28-
POLICY_PRIORITY_LOWEST,
29-
get_policy_name,
30-
get_priority_by_name,
31-
load_policies,
32-
)
3328
from .wheel_abi import NonPlatformWheel, analyze_wheel_abi
3429

30+
wheel_policy = WheelPolicies.load()
31+
3532
fn = basename(args.WHEEL_FILE)
3633

3734
if not isfile(args.WHEEL_FILE):
3835
p.error("cannot access %s. No such file" % args.WHEEL_FILE)
3936

4037
try:
41-
winfo = analyze_wheel_abi(args.WHEEL_FILE)
38+
winfo = analyze_wheel_abi(wheel_policy, args.WHEEL_FILE)
4239
except NonPlatformWheel:
4340
logger.info(NonPlatformWheel.LOG_MESSAGE)
4441
return 1
@@ -52,7 +49,10 @@ def execute(args, p):
5249
% (fn, winfo.overall_tag)
5350
)
5451

55-
if get_priority_by_name(winfo.pyfpe_tag) < POLICY_PRIORITY_HIGHEST:
52+
if (
53+
wheel_policy.get_priority_by_name(winfo.pyfpe_tag)
54+
< wheel_policy.priority_highest
55+
):
5656
printp(
5757
"This wheel uses the PyFPE_jbuf function, which is not compatible with the"
5858
" manylinux1 tag. (see https://www.python.org/dev/peps/pep-0513/"
@@ -61,7 +61,7 @@ def execute(args, p):
6161
if args.verbose < 1:
6262
return
6363

64-
if get_priority_by_name(winfo.ucs_tag) < POLICY_PRIORITY_HIGHEST:
64+
if wheel_policy.get_priority_by_name(winfo.ucs_tag) < wheel_policy.priority_highest:
6565
printp(
6666
"This wheel is compiled against a narrow unicode (UCS2) "
6767
"version of Python, which is not compatible with the "
@@ -81,7 +81,7 @@ def execute(args, p):
8181
"system-provided shared libraries: %s" % ", ".join(libs_with_versions)
8282
)
8383

84-
if get_priority_by_name(winfo.sym_tag) < POLICY_PRIORITY_HIGHEST:
84+
if wheel_policy.get_priority_by_name(winfo.sym_tag) < wheel_policy.priority_highest:
8585
printp(
8686
(
8787
'This constrains the platform tag to "%s". '
@@ -95,15 +95,17 @@ def execute(args, p):
9595
if args.verbose < 1:
9696
return
9797

98-
libs = winfo.external_refs[get_policy_name(POLICY_PRIORITY_LOWEST)]["libs"]
98+
libs = winfo.external_refs[
99+
wheel_policy.get_policy_name(wheel_policy.priority_lowest)
100+
]["libs"]
99101
if len(libs) == 0:
100102
printp("The wheel requires no external shared libraries! :)")
101103
else:
102104
printp("The following external shared libraries are required " "by the wheel:")
103105
print(json.dumps(dict(sorted(libs.items())), indent=4))
104106

105-
for p in sorted(load_policies(), key=lambda p: p["priority"]):
106-
if p["priority"] > get_priority_by_name(winfo.overall_tag):
107+
for p in sorted(wheel_policy.policies, key=lambda p: p["priority"]):
108+
if p["priority"] > wheel_policy.get_priority_by_name(winfo.overall_tag):
107109
libs = winfo.external_refs[p["name"]]["libs"]
108110
if len(libs):
109111
printp(

src/auditwheel/policy/__init__.py

Lines changed: 88 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,87 @@
1818
# https://docs.python.org/3/library/platform.html#platform.architecture
1919
bits = 8 * (8 if sys.maxsize > 2**32 else 4)
2020

21+
_POLICY_JSON_MAP = {
22+
Libc.GLIBC: _HERE / "manylinux-policy.json",
23+
Libc.MUSL: _HERE / "musllinux-policy.json",
24+
}
25+
26+
27+
class WheelPolicies:
28+
@staticmethod
29+
def load():
30+
libc_variant = get_libc()
31+
policies_path = _POLICY_JSON_MAP[libc_variant]
32+
policy_dict = json.loads(policies_path.read_text())
33+
return WheelPolicies(policy_dict)
34+
35+
def __init__(self, policies: dict) -> None:
36+
self._policies = []
37+
self._musl_policy = _get_musl_policy()
38+
self._arch_name = get_arch_name()
39+
self._libc_variant = get_libc()
40+
41+
_validate_pep600_compliance(policies)
42+
for policy in policies:
43+
if self._musl_policy is not None and policy["name"] not in {
44+
"linux",
45+
self._musl_policy,
46+
}:
47+
continue
48+
if (
49+
self._arch_name in policy["symbol_versions"].keys()
50+
or policy["name"] == "linux"
51+
):
52+
if policy["name"] != "linux":
53+
policy["symbol_versions"] = policy["symbol_versions"][
54+
self._arch_name
55+
]
56+
policy["name"] = policy["name"] + "_" + self._arch_name
57+
policy["aliases"] = [
58+
alias + "_" + self._arch_name for alias in policy["aliases"]
59+
]
60+
policy["lib_whitelist"] = _fixup_musl_libc_soname(
61+
policy["lib_whitelist"]
62+
)
63+
self._policies.append(policy)
64+
65+
if self._libc_variant == Libc.MUSL:
66+
assert len(self._policies) == 2, self._policies
67+
68+
@property
69+
def policies(self):
70+
return self._policies
71+
72+
@property
73+
def priority_highest(self):
74+
return max(p["priority"] for p in self._policies)
75+
76+
@property
77+
def priority_lowest(self):
78+
return min(p["priority"] for p in self._policies)
79+
80+
def get_policy_by_name(self, name: str) -> dict | None:
81+
matches = [
82+
p for p in self._policies if p["name"] == name or name in p["aliases"]
83+
]
84+
if len(matches) == 0:
85+
return None
86+
if len(matches) > 1:
87+
raise RuntimeError("Internal error. Policies should be unique")
88+
return matches[0]
89+
90+
def get_policy_name(self, priority: int) -> str | None:
91+
matches = [p["name"] for p in self._policies if p["priority"] == priority]
92+
if len(matches) == 0:
93+
return None
94+
if len(matches) > 1:
95+
raise RuntimeError("Internal error. priorities should be unique")
96+
return matches[0]
97+
98+
def get_priority_by_name(self, name: str) -> int | None:
99+
policy = self.get_policy_by_name(name)
100+
return None if policy is None else policy["priority"]
101+
21102

22103
def get_arch_name() -> str:
23104
machine = _platform_module.machine()
@@ -65,22 +146,13 @@ def _validate_pep600_compliance(policies) -> None:
65146
symbol_versions[arch] = symbol_versions_arch
66147

67148

68-
_POLICY_JSON_MAP = {
69-
Libc.GLIBC: _HERE / "manylinux-policy.json",
70-
Libc.MUSL: _HERE / "musllinux-policy.json",
71-
}
72-
73-
74149
def _get_musl_policy():
75150
if _LIBC != Libc.MUSL:
76151
return None
77152
musl_version = get_musl_version(find_musl_libc())
78153
return f"musllinux_{musl_version.major}_{musl_version.minor}"
79154

80155

81-
_MUSL_POLICY = _get_musl_policy()
82-
83-
84156
def _fixup_musl_libc_soname(whitelist):
85157
if _LIBC != Libc.MUSL:
86158
return whitelist
@@ -105,60 +177,6 @@ def _fixup_musl_libc_soname(whitelist):
105177
return new_whitelist
106178

107179

108-
with _POLICY_JSON_MAP[_LIBC].open() as f:
109-
_POLICIES = []
110-
_policies_temp = json.load(f)
111-
_validate_pep600_compliance(_policies_temp)
112-
for _p in _policies_temp:
113-
if _MUSL_POLICY is not None and _p["name"] not in {"linux", _MUSL_POLICY}:
114-
continue
115-
if _ARCH_NAME in _p["symbol_versions"].keys() or _p["name"] == "linux":
116-
if _p["name"] != "linux":
117-
_p["symbol_versions"] = _p["symbol_versions"][_ARCH_NAME]
118-
_p["name"] = _p["name"] + "_" + _ARCH_NAME
119-
_p["aliases"] = [alias + "_" + _ARCH_NAME for alias in _p["aliases"]]
120-
_p["lib_whitelist"] = _fixup_musl_libc_soname(_p["lib_whitelist"])
121-
_POLICIES.append(_p)
122-
if _LIBC == Libc.MUSL:
123-
assert len(_POLICIES) == 2, _POLICIES
124-
125-
POLICY_PRIORITY_HIGHEST = max(p["priority"] for p in _POLICIES)
126-
POLICY_PRIORITY_LOWEST = min(p["priority"] for p in _POLICIES)
127-
128-
129-
def load_policies():
130-
return _POLICIES
131-
132-
133-
def _load_policy_schema():
134-
with open(join(dirname(abspath(__file__)), "policy-schema.json")) as f_:
135-
schema = json.load(f_)
136-
return schema
137-
138-
139-
def get_policy_by_name(name: str) -> dict | None:
140-
matches = [p for p in _POLICIES if p["name"] == name or name in p["aliases"]]
141-
if len(matches) == 0:
142-
return None
143-
if len(matches) > 1:
144-
raise RuntimeError("Internal error. Policies should be unique")
145-
return matches[0]
146-
147-
148-
def get_policy_name(priority: int) -> str | None:
149-
matches = [p["name"] for p in _POLICIES if p["priority"] == priority]
150-
if len(matches) == 0:
151-
return None
152-
if len(matches) > 1:
153-
raise RuntimeError("Internal error. priorities should be unique")
154-
return matches[0]
155-
156-
157-
def get_priority_by_name(name: str) -> int | None:
158-
policy = get_policy_by_name(name)
159-
return None if policy is None else policy["priority"]
160-
161-
162180
def get_replace_platforms(name: str) -> list[str]:
163181
"""Extract platform tag replacement rules from policy
164182
@@ -185,10 +203,14 @@ def get_replace_platforms(name: str) -> list[str]:
185203
from .external_references import lddtree_external_references # noqa
186204
from .versioned_symbols import versioned_symbols_policy # noqa
187205

206+
def _load_policy_schema():
207+
with open(join(dirname(abspath(__file__)), "policy-schema.json")) as f_:
208+
schema = json.load(f_)
209+
return schema
210+
211+
188212
__all__ = [
189213
"lddtree_external_references",
190214
"versioned_symbols_policy",
191-
"load_policies",
192-
"POLICY_PRIORITY_HIGHEST",
193-
"POLICY_PRIORITY_LOWEST",
215+
"WheelPolicies",
194216
]

0 commit comments

Comments
 (0)