@@ -29,7 +29,8 @@ behavior.
2929load ("//python/private:normalize_name.bzl" , "normalize_name" )
3030load (":index_sources.bzl" , "index_sources" )
3131load (":parse_requirements_txt.bzl" , "parse_requirements_txt" )
32- load (":whl_target_platforms.bzl" , "select_whls" , "whl_target_platforms" )
32+ load (":requirements_files_by_platform.bzl" , "requirements_files_by_platform" )
33+ load (":whl_target_platforms.bzl" , "select_whls" )
3334
3435# This includes the vendored _translate_cpu and _translate_os from
3536# @platforms//host:extension.bzl at version 0.0.9 so that we don't
@@ -80,73 +81,6 @@ DEFAULT_PLATFORMS = [
8081 "windows_x86_64" ,
8182]
8283
83- def _default_platforms (* , filter ):
84- if not filter :
85- fail ("Must specific a filter string, got: {}" .format (filter ))
86-
87- if filter .startswith ("cp3" ):
88- # TODO @aignas 2024-05-23: properly handle python versions in the filter.
89- # For now we are just dropping it to ensure that we don't fail.
90- _ , _ , filter = filter .partition ("_" )
91-
92- sanitized = filter .replace ("*" , "" ).replace ("_" , "" )
93- if sanitized and not sanitized .isalnum ():
94- fail ("The platform filter can only contain '*', '_' and alphanumerics" )
95-
96- if "*" in filter :
97- prefix = filter .rstrip ("*" )
98- if "*" in prefix :
99- fail ("The filter can only contain '*' at the end of it" )
100-
101- if not prefix :
102- return DEFAULT_PLATFORMS
103-
104- return [p for p in DEFAULT_PLATFORMS if p .startswith (prefix )]
105- else :
106- return [p for p in DEFAULT_PLATFORMS if filter in p ]
107-
108- def _platforms_from_args (extra_pip_args ):
109- platform_values = []
110-
111- for arg in extra_pip_args :
112- if platform_values and platform_values [- 1 ] == "" :
113- platform_values [- 1 ] = arg
114- continue
115-
116- if arg == "--platform" :
117- platform_values .append ("" )
118- continue
119-
120- if not arg .startswith ("--platform" ):
121- continue
122-
123- _ , _ , plat = arg .partition ("=" )
124- if not plat :
125- _ , _ , plat = arg .partition (" " )
126- if plat :
127- platform_values .append (plat )
128- else :
129- platform_values .append ("" )
130-
131- if not platform_values :
132- return []
133-
134- platforms = {
135- p .target_platform : None
136- for arg in platform_values
137- for p in whl_target_platforms (arg )
138- }
139- return list (platforms .keys ())
140-
141- def _platform (platform_string , python_version = None ):
142- if not python_version or platform_string .startswith ("cp3" ):
143- return platform_string
144-
145- _ , _ , tail = python_version .partition ("." )
146- minor , _ , _ = tail .partition ("." )
147-
148- return "cp3{}_{}" .format (minor , platform_string )
149-
15084def parse_requirements (
15185 ctx ,
15286 * ,
@@ -201,105 +135,20 @@ def parse_requirements(
201135
202136 The second element is extra_pip_args should be passed to `whl_library`.
203137 """
204- if not (
205- requirements_lock or
206- requirements_linux or
207- requirements_osx or
208- requirements_windows or
209- requirements_by_platform
210- ):
211- fail_fn (
212- "A 'requirements_lock' attribute must be specified, a platform-specific lockfiles " +
213- "via 'requirements_by_platform' or an os-specific lockfiles must be specified " +
214- "via 'requirements_*' attributes" ,
215- )
216- return None
217-
218- platforms = _platforms_from_args (extra_pip_args )
219- if logger :
220- logger .debug (lambda : "Platforms from pip args: {}" .format (platforms ))
221-
222- if platforms :
223- lock_files = [
224- f
225- for f in [
226- requirements_lock ,
227- requirements_linux ,
228- requirements_osx ,
229- requirements_windows ,
230- ] + list (requirements_by_platform .keys ())
231- if f
232- ]
233-
234- if len (lock_files ) > 1 :
235- # If the --platform argument is used, check that we are using
236- # a single `requirements_lock` file instead of the OS specific ones as that is
237- # the only correct way to use the API.
238- fail_fn ("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute" )
239- return None
240-
241- files_by_platform = [
242- (lock_files [0 ], platforms ),
243- ]
244- if logger :
245- logger .debug (lambda : "Files by platform with the platform set in the args: {}" .format (files_by_platform ))
246- else :
247- files_by_platform = {
248- file : [
249- platform
250- for filter_or_platform in specifier .split ("," )
251- for platform in (_default_platforms (filter = filter_or_platform ) if filter_or_platform .endswith ("*" ) else [filter_or_platform ])
252- ]
253- for file , specifier in requirements_by_platform .items ()
254- }.items ()
255-
256- if logger :
257- logger .debug (lambda : "Files by platform with the platform set in the attrs: {}" .format (files_by_platform ))
258-
259- for f in [
260- # If the users need a greater span of the platforms, they should consider
261- # using the 'requirements_by_platform' attribute.
262- (requirements_linux , _default_platforms (filter = "linux_*" )),
263- (requirements_osx , _default_platforms (filter = "osx_*" )),
264- (requirements_windows , _default_platforms (filter = "windows_*" )),
265- (requirements_lock , None ),
266- ]:
267- if f [0 ]:
268- if logger :
269- logger .debug (lambda : "Adding an extra item to files_by_platform: {}" .format (f ))
270- files_by_platform .append (f )
271-
272- configured_platforms = {}
138+ requirements_by_platform , download_only = requirements_files_by_platform (
139+ requirements_by_platform = requirements_by_platform ,
140+ requirements_osx = requirements_osx ,
141+ requirements_linux = requirements_linux ,
142+ requirements_lock = requirements_lock ,
143+ requirements_windows = requirements_windows ,
144+ extra_pip_args = extra_pip_args ,
145+ python_version = python_version ,
146+ logger = logger ,
147+ )
273148
274149 options = {}
275150 requirements = {}
276- for file , plats in files_by_platform :
277- if plats :
278- plats = [_platform (p , python_version ) for p in plats ]
279- for p in plats :
280- if p in configured_platforms :
281- fail_fn (
282- "Expected the platform '{}' to be map only to a single requirements file, but got multiple: '{}', '{}'" .format (
283- p ,
284- configured_platforms [p ],
285- file ,
286- ),
287- )
288- return None
289-
290- configured_platforms [p ] = file
291- else :
292- plats = [
293- _platform (p , python_version )
294- for p in DEFAULT_PLATFORMS
295- if p not in configured_platforms
296- ]
297- for p in plats :
298- configured_platforms [p ] = file
299-
300- if logger :
301- logger .debug (lambda : "Configured platforms for file {} are {}" .format (file , plats ))
302-
151+ for plat , file in requirements_by_platform .items ():
303152 contents = ctx .read (file )
304153
305154 # Parse the requirements file directly in starlark to get the information
@@ -332,13 +181,10 @@ def parse_requirements(
332181 tokenized_options .append (p )
333182
334183 pip_args = tokenized_options + extra_pip_args
335- logger .debug (lambda : "Using {} for {}" .format (file , plats ))
336- for p in plats :
337- if p in requirements :
338- fail_fn ("Attempting to override a requirements file. Existing value: {}" .format (requirements [p ]))
339- return None
340- requirements [p ] = requirements_dict
341- options [p ] = pip_args
184+ if logger :
185+ logger .debug (lambda : "Using {} for {}" .format (file , plat ))
186+ requirements [plat ] = requirements_dict
187+ options [plat ] = pip_args
342188
343189 requirements_by_platform = {}
344190 reqs_with_env_markers = {}
@@ -362,7 +208,7 @@ def parse_requirements(
362208 requirement_line = requirement_line ,
363209 target_platforms = [],
364210 extra_pip_args = extra_pip_args ,
365- download = len ( platforms ) > 0 ,
211+ download = download_only ,
366212 ),
367213 )
368214 for_req .target_platforms .append (target_platform )
@@ -405,12 +251,14 @@ def parse_requirements(
405251 for p in target_platforms :
406252 requirement_target_platforms [p ] = None
407253
408- is_exposed = len (requirement_target_platforms ) == len (configured_platforms )
254+ # TODO @aignas 2024-07-15: the `is_exposed` logic here is wrong at the
255+ # moment, have to think of a good way to get the value.
256+ is_exposed = len (requirement_target_platforms ) == len (requirements_by_platform )
409257 if not is_exposed and logger :
410258 logger .debug (lambda : "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}" .format (
411259 whl_name ,
412260 sorted (requirement_target_platforms ),
413- sorted (configured_platforms ),
261+ sorted (requirements_by_platform ),
414262 ))
415263
416264 for r in sorted (reqs .values (), key = lambda r : r .requirement_line ):
0 commit comments