Skip to content

Commit 2df131f

Browse files
phoeriousphoerious
committed
Remove deprecated distutils dependency
1 parent 77379e3 commit 2df131f

File tree

2 files changed

+143
-139
lines changed

2 files changed

+143
-139
lines changed

‎fastwarc/setup.py‎

Lines changed: 73 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -14,76 +14,82 @@
1414

1515
import os
1616
import platform
17+
from shutil import copytree
1718
import sys
1819

1920
from Cython.Build import cythonize
2021
from Cython.Distutils.build_ext import new_build_ext as build_ext
21-
import distutils.ccompiler
22-
from distutils.dir_util import copy_tree
2322
from setuptools import Extension, setup
2423

2524
TRACE = bool(int(os.getenv('TRACE', 0)))
2625
DEBUG = bool(int(os.getenv('DEBUG', 0))) or TRACE
2726
ASAN = bool(int(os.getenv('ASAN', 0)))
2827

2928
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
30-
CXX = distutils.ccompiler.get_default_compiler()
31-
32-
# Construct vcpkg lib and include paths
33-
def _vcpkg_path():
34-
osname = platform.system().lower().replace('darwin', 'osx')
35-
arch = platform.machine().lower()
36-
if os.environ.get('_PYTHON_HOST_PLATFORM', '').startswith('macosx-'):
37-
arch = os.environ['_PYTHON_HOST_PLATFORM'].split('-')[-1]
38-
elif osname == 'linux' and arch == 'arm64':
39-
arch = 'aarch64'
40-
arch = arch.replace('x86_64', 'x64').replace('amd64', 'x64')
41-
triplet = f'{arch}-{osname}'
42-
43-
if os.environ.get('RESILIPARSE_VCPKG_PATH'):
44-
return os.path.join(os.environ['RESILIPARSE_VCPKG_PATH'], triplet)
45-
return os.path.join(os.path.dirname(ROOT_DIR), 'vcpkg_installed', triplet)
46-
47-
INCLUDE_PATH = os.path.join(_vcpkg_path(), 'include')
48-
LIBRARY_PATH = os.path.join(_vcpkg_path(), 'lib')
49-
50-
51-
def get_cpp_args():
52-
cpp_args = {}
53-
54-
if TRACE:
55-
cpp_args.update(dict(define_macros=[('CYTHON_TRACE_NOGIL', '1')]))
56-
57-
if CXX == 'unix':
58-
cpp_args.update(dict(
59-
extra_compile_args=['-std=c++17',
60-
f'-O{0 if DEBUG else 3}',
61-
f'-I{INCLUDE_PATH}',
62-
'-Wall',
63-
'-Wno-deprecated-declarations',
64-
'-Wno-unreachable-code',
65-
'-Wno-unused-function'],
66-
extra_link_args=['-std=c++17', f'-L{LIBRARY_PATH}', f'-Wl,-rpath,{LIBRARY_PATH}']
67-
))
68-
if DEBUG:
69-
cpp_args['extra_compile_args'].append('-Werror')
70-
if ASAN:
71-
cpp_args['extra_compile_args'].append('-fsanitize=address')
72-
cpp_args['extra_link_args'].append('-fsanitize=address')
73-
74-
elif CXX == 'msvc':
75-
cpp_args.update(dict(
76-
extra_compile_args=['/std:c++latest',
77-
'/W3',
78-
f'/O{"d" if DEBUG else 2}',
79-
f'/I{INCLUDE_PATH}'],
80-
extra_link_args=[f'/LIBPATH:{LIBRARY_PATH}']
81-
))
82-
if DEBUG:
83-
cpp_args['extra_compile_args'].append('/WX')
84-
cpp_args['extra_link_args'].append('/WX')
85-
86-
return cpp_args
29+
30+
31+
class resiliparse_build_ext(build_ext):
32+
def build_extension(self, ext):
33+
for k, v in self.get_cpp_args().items():
34+
setattr(ext, k, v)
35+
# Zlib is built with different name on MSVC
36+
if self.compiler.compiler_type == 'msvc':
37+
ext.libraries = ['zlib' if l == 'z' else l for l in ext.libraries]
38+
39+
return super().build_extension(ext)
40+
41+
def get_vcpkg_path(self):
42+
osname = platform.system().lower().replace('darwin', 'osx')
43+
arch = platform.machine().lower()
44+
if os.environ.get('_PYTHON_HOST_PLATFORM', '').startswith('macosx-'):
45+
arch = os.environ['_PYTHON_HOST_PLATFORM'].split('-')[-1]
46+
elif osname == 'linux' and arch == 'arm64':
47+
arch = 'aarch64'
48+
arch = arch.replace('x86_64', 'x64').replace('amd64', 'x64')
49+
triplet = f'{arch}-{osname}'
50+
51+
if os.environ.get('RESILIPARSE_VCPKG_PATH'):
52+
return os.path.join(os.environ['RESILIPARSE_VCPKG_PATH'], triplet)
53+
return os.path.join(os.path.dirname(ROOT_DIR), 'vcpkg_installed', triplet)
54+
55+
def get_cpp_args(self):
56+
include_path = os.path.join(self.get_vcpkg_path(), 'include')
57+
library_path = os.path.join(self.get_vcpkg_path(), 'lib')
58+
cpp_args = {}
59+
60+
if TRACE:
61+
cpp_args.update(dict(define_macros=[('CYTHON_TRACE_NOGIL', '1')]))
62+
63+
if self.compiler.compiler_type == 'unix':
64+
cpp_args.update(dict(
65+
extra_compile_args=['-std=c++17',
66+
f'-O{0 if DEBUG else 3}',
67+
f'-I{include_path}',
68+
'-Wall',
69+
'-Wno-deprecated-declarations',
70+
'-Wno-unreachable-code',
71+
'-Wno-unused-function'],
72+
extra_link_args=['-std=c++17', f'-L{library_path}', f'-Wl,-rpath,{library_path}']
73+
))
74+
if DEBUG:
75+
cpp_args['extra_compile_args'].append('-Werror')
76+
if ASAN:
77+
cpp_args['extra_compile_args'].append('-fsanitize=address')
78+
cpp_args['extra_link_args'].append('-fsanitize=address')
79+
80+
elif self.compiler.compiler_type == 'msvc':
81+
cpp_args.update(dict(
82+
extra_compile_args=['/std:c++latest',
83+
'/W3',
84+
f'/O{"d" if DEBUG else 2}',
85+
f'/I{include_path}'],
86+
extra_link_args=[f'/LIBPATH:{library_path}']
87+
))
88+
if DEBUG:
89+
cpp_args['extra_compile_args'].append('/WX')
90+
cpp_args['extra_link_args'].append('/WX')
91+
92+
return cpp_args
8793

8894

8995
def get_cython_args():
@@ -102,28 +108,26 @@ def get_cython_args():
102108

103109

104110
def get_ext_modules():
105-
cpp_args = get_cpp_args()
106-
107111
fastwarc_extensions = [
108-
Extension('fastwarc.warc', sources=['fastwarc/warc.pyx'], **cpp_args),
112+
Extension('fastwarc.warc', sources=['fastwarc/warc.pyx']),
109113
Extension('fastwarc.stream_io', sources=['fastwarc/stream_io.pyx'],
110-
libraries=['zlib' if CXX == 'msvc' else 'z', 'lz4'], **cpp_args),
111-
Extension('fastwarc.tools', sources=['fastwarc/tools.pyx'], **cpp_args)
114+
libraries=['z', 'lz4']),
115+
Extension('fastwarc.tools', sources=['fastwarc/tools.pyx'])
112116
]
113117

114118
return cythonize(fastwarc_extensions, **get_cython_args())
115119

116120

117121
# Copy Resiliparse header files
118122
if os.path.isdir(os.path.join(ROOT_DIR, '..', 'resiliparse', 'resiliparse_inc')):
119-
copy_tree(os.path.join(ROOT_DIR, '..', 'resiliparse', 'resiliparse_inc'),
120-
os.path.join(ROOT_DIR, 'resiliparse_inc'), update=1)
121-
copy_tree(os.path.join(ROOT_DIR, '..', 'resiliparse', 'resiliparse_common'),
122-
os.path.join(ROOT_DIR, 'resiliparse_common'), update=1)
123+
copytree(os.path.join(ROOT_DIR, '..', 'resiliparse', 'resiliparse_inc'),
124+
os.path.join(ROOT_DIR, 'resiliparse_inc'), dirs_exist_ok=True)
125+
copytree(os.path.join(ROOT_DIR, '..', 'resiliparse', 'resiliparse_common'),
126+
os.path.join(ROOT_DIR, 'resiliparse_common'), dirs_exist_ok=True)
123127

124128
setup(
125129
ext_modules=get_ext_modules(),
126-
cmdclass=dict(build_ext=build_ext),
130+
cmdclass=dict(build_ext=resiliparse_build_ext),
127131
exclude_package_data={
128132
'': [] if 'sdist' in sys.argv else ['*.pxd', '*.pxi', '*.pyx', '*.h', '*.cpp']
129133
}

‎resiliparse/setup.py‎

Lines changed: 70 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -20,73 +20,75 @@
2020

2121
from Cython.Build import cythonize
2222
from Cython.Distutils.build_ext import new_build_ext as build_ext
23-
import distutils.ccompiler
2423
from setuptools import Extension, setup
2524

2625
TRACE = bool(int(os.getenv('TRACE', 0)))
2726
DEBUG = bool(int(os.getenv('DEBUG', 0))) or TRACE
2827
ASAN = bool(int(os.getenv('ASAN', 0)))
2928

3029
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
31-
CXX = distutils.ccompiler.get_default_compiler()
32-
33-
# Construct vcpkg lib and include paths
34-
def _vcpkg_path():
35-
osname = platform.system().lower().replace('darwin', 'osx')
36-
arch = platform.machine().lower()
37-
if os.environ.get('_PYTHON_HOST_PLATFORM', '').startswith('macosx-'):
38-
arch = os.environ['_PYTHON_HOST_PLATFORM'].split('-')[-1]
39-
elif osname == 'linux' and arch == 'arm64':
40-
arch = 'aarch64'
41-
arch = arch.replace('x86_64', 'x64').replace('amd64', 'x64')
42-
triplet = f'{arch}-{osname}'
43-
44-
if os.environ.get('RESILIPARSE_VCPKG_PATH'):
45-
return os.path.join(os.environ['RESILIPARSE_VCPKG_PATH'], triplet)
46-
return os.path.join(os.path.dirname(ROOT_DIR), 'vcpkg_installed', triplet)
47-
48-
INCLUDE_PATH = os.path.join(_vcpkg_path(), 'include')
49-
LIBRARY_PATH = os.path.join(_vcpkg_path(), 'lib')
50-
51-
52-
def get_cpp_args():
53-
cpp_args = {}
54-
55-
if TRACE:
56-
cpp_args.update(dict(define_macros=[('CYTHON_TRACE_NOGIL', '1')]))
57-
58-
if CXX == 'unix':
59-
cpp_args.update(dict(
60-
extra_compile_args=['-std=c++17',
61-
f'-O{0 if DEBUG else 3}',
62-
f'-I{INCLUDE_PATH}',
63-
'-Wall',
64-
'-Wno-deprecated-declarations',
65-
'-Wno-unreachable-code',
66-
'-Wno-unused-function'],
67-
extra_link_args=['-std=c++17', f'-L{LIBRARY_PATH}', f'-Wl,-rpath,{LIBRARY_PATH}']
68-
))
69-
if DEBUG:
70-
cpp_args['extra_compile_args'].append('-Werror')
71-
if ASAN:
72-
cpp_args['extra_compile_args'].append('-fsanitize=address')
73-
cpp_args['extra_link_args'].append('-fsanitize=address')
74-
if platform.system() == 'Darwin':
75-
cpp_args['extra_link_args'].append('-headerpad_max_install_names')
76-
77-
elif CXX == 'msvc':
78-
cpp_args.update(dict(
79-
extra_compile_args=['/std:c++latest',
80-
'/W3',
81-
f'/O{"d" if DEBUG else 2}',
82-
f'/I{INCLUDE_PATH}'],
83-
extra_link_args=[f'/LIBPATH:{LIBRARY_PATH}']
84-
))
85-
if DEBUG:
86-
cpp_args['extra_compile_args'].append('/WX')
87-
cpp_args['extra_link_args'].append('/WX')
88-
89-
return cpp_args
30+
31+
32+
class resiliparse_build_ext(build_ext):
33+
def build_extension(self, ext):
34+
for k, v in self.get_cpp_args().items():
35+
setattr(ext, k, v)
36+
return super().build_extension(ext)
37+
38+
def get_vcpkg_path(self):
39+
osname = platform.system().lower().replace('darwin', 'osx')
40+
arch = platform.machine().lower()
41+
if os.environ.get('_PYTHON_HOST_PLATFORM', '').startswith('macosx-'):
42+
arch = os.environ['_PYTHON_HOST_PLATFORM'].split('-')[-1]
43+
elif osname == 'linux' and arch == 'arm64':
44+
arch = 'aarch64'
45+
arch = arch.replace('x86_64', 'x64').replace('amd64', 'x64')
46+
triplet = f'{arch}-{osname}'
47+
48+
if os.environ.get('RESILIPARSE_VCPKG_PATH'):
49+
return os.path.join(os.environ['RESILIPARSE_VCPKG_PATH'], triplet)
50+
return os.path.join(os.path.dirname(ROOT_DIR), 'vcpkg_installed', triplet)
51+
52+
def get_cpp_args(self):
53+
include_path = os.path.join(self.get_vcpkg_path(), 'include')
54+
library_path = os.path.join(self.get_vcpkg_path(), 'lib')
55+
cpp_args = {}
56+
57+
if TRACE:
58+
cpp_args.update(dict(define_macros=[('CYTHON_TRACE_NOGIL', '1')]))
59+
60+
if self.compiler.compiler_type == 'unix':
61+
cpp_args.update(dict(
62+
extra_compile_args=['-std=c++17',
63+
f'-O{0 if DEBUG else 3}',
64+
f'-I{include_path}',
65+
'-Wall',
66+
'-Wno-deprecated-declarations',
67+
'-Wno-unreachable-code',
68+
'-Wno-unused-function'],
69+
extra_link_args=['-std=c++17', f'-L{library_path}', f'-Wl,-rpath,{library_path}']
70+
))
71+
if DEBUG:
72+
cpp_args['extra_compile_args'].append('-Werror')
73+
if ASAN:
74+
cpp_args['extra_compile_args'].append('-fsanitize=address')
75+
cpp_args['extra_link_args'].append('-fsanitize=address')
76+
if platform.system() == 'Darwin':
77+
cpp_args['extra_link_args'].append('-headerpad_max_install_names')
78+
79+
elif self.compiler.compiler_type == 'msvc':
80+
cpp_args.update(dict(
81+
extra_compile_args=['/std:c++latest',
82+
'/W3',
83+
f'/O{"d" if DEBUG else 2}',
84+
f'/I{include_path}'],
85+
extra_link_args=[f'/LIBPATH:{library_path}']
86+
))
87+
if DEBUG:
88+
cpp_args['extra_compile_args'].append('/WX')
89+
cpp_args['extra_link_args'].append('/WX')
90+
91+
return cpp_args
9092

9193

9294
def get_cython_args():
@@ -105,27 +107,25 @@ def get_cython_args():
105107

106108

107109
def get_ext_modules():
108-
cpp_args = get_cpp_args()
109-
110110
resiliparse_extensions = [
111111
Extension('resiliparse.itertools',
112-
sources=[f'resiliparse/itertools.pyx'], **cpp_args),
112+
sources=[f'resiliparse/itertools.pyx']),
113113
Extension('resiliparse.extract.html2text',
114-
sources=[f'resiliparse/extract/html2text.pyx'], libraries=['lexbor', 're2'], **cpp_args),
114+
sources=[f'resiliparse/extract/html2text.pyx'], libraries=['lexbor', 're2']),
115115
Extension('resiliparse.parse.encoding',
116-
sources=[f'resiliparse/parse/encoding.pyx'], libraries=['uchardet', 'lexbor'], **cpp_args),
116+
sources=[f'resiliparse/parse/encoding.pyx'], libraries=['uchardet', 'lexbor']),
117117
Extension('resiliparse.parse.html',
118-
sources=[f'resiliparse/parse/html.pyx'], libraries=['lexbor'], **cpp_args),
118+
sources=[f'resiliparse/parse/html.pyx'], libraries=['lexbor']),
119119
Extension('resiliparse.parse.http',
120-
sources=[f'resiliparse/parse/http.pyx'], **cpp_args),
120+
sources=[f'resiliparse/parse/http.pyx']),
121121
Extension('resiliparse.parse.lang',
122-
sources=[f'resiliparse/parse/lang.pyx'], **cpp_args),
122+
sources=[f'resiliparse/parse/lang.pyx']),
123123
]
124124
if os.name == 'posix':
125125
# Process Guards are unsupported on Windows
126126
resiliparse_extensions.append(
127127
Extension('resiliparse.process_guard', sources=[f'resiliparse/process_guard.pyx'],
128-
libraries=['pthread'], **cpp_args)
128+
libraries=['pthread'])
129129
)
130130

131131
return cythonize(resiliparse_extensions, **get_cython_args())
@@ -138,7 +138,7 @@ def get_ext_modules():
138138

139139
setup(
140140
ext_modules=get_ext_modules(),
141-
cmdclass=dict(build_ext=build_ext),
141+
cmdclass=dict(build_ext=resiliparse_build_ext),
142142
exclude_package_data={
143143
'': [] if 'sdist' in sys.argv else ['*.pxd', '*.pxi', '*.pyx', '*.h', '*.cpp']
144144
}

0 commit comments

Comments
 (0)