Skip to content

Commit 86aca15

Browse files
chinyeunglichinyeungli
committed
Remove default.nix and add script to generate the default.nix #339
Signed-off-by: Chin Yeung Li <[email protected]>
1 parent ae2cb2c commit 86aca15

File tree

2 files changed

+347
-858
lines changed

2 files changed

+347
-858
lines changed

‎build_nix_docker.py‎

Lines changed: 347 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,338 @@
33
Use Docker container to run nix-build.
44
Using Docker approach to ensure a consistent and isolated build environment.
55
6+
Requirement: `toml` and `requests` Python packages and Docker installed.
7+
8+
pip install toml requests
9+
610
To run the script:
711
812
python build_nix_docker.py
913
14+
or
15+
16+
python build_nix_docker.py --generate
17+
18+
The --generate flag is optional and can be used to generate the
19+
default.nix file if needed.
20+
1021
This script will run nix-build and place the built results in the
1122
dist/nix/ directory, it will then run nix-collect-garbage for cleanup.
1223
"""
1324

25+
import argparse
1426
import os
27+
import requests
1528
import shutil
1629
import subprocess
1730
import sys
1831
from pathlib import Path
1932

2033

34+
def read_pyproject_toml():
35+
"""
36+
Read the pyproject.toml file to extract project metadata.
37+
"""
38+
import toml
39+
40+
pyproject_path = Path('pyproject.toml')
41+
if not pyproject_path.exists():
42+
print("Error: pyproject.toml not found in current directory", file=sys.stderr)
43+
sys.exit(1)
44+
45+
with pyproject_path.open('r') as f:
46+
pyproject_data = toml.load(f)
47+
48+
return pyproject_data
49+
50+
def extract_project_meta(pyproject_data):
51+
"""
52+
Extract project metadata from pyproject.toml data.
53+
"""
54+
project_data = pyproject_data['project']
55+
name = project_data.get('name')
56+
version = project_data.get('version')
57+
description = project_data.get('description')
58+
authors = project_data.get('authors')
59+
author_names = [author.get('name', '') for author in authors if 'name' in author]
60+
author_str = ', '.join(author_names)
61+
62+
meta_dict = {
63+
'name': name,
64+
'version': version,
65+
'description': description,
66+
'author': author_str
67+
}
68+
69+
return meta_dict
70+
71+
72+
def extract_project_dependencies(pyproject_data):
73+
"""
74+
Extract project dependencies from pyproject.toml data.
75+
"""
76+
project_data = pyproject_data['project']
77+
dependencies = project_data.get('dependencies', [])
78+
optional_dependencies = project_data.get('optional-dependencies', {})
79+
dev_optional_deps = optional_dependencies.get('dev', [])
80+
all_dep = dependencies + dev_optional_deps
81+
dependencies_list = []
82+
83+
for dep in all_dep:
84+
name_version = dep.split('==')
85+
name = name_version[0]
86+
version = name_version[1]
87+
tmp_dict = {}
88+
tmp_dict['name'] = name
89+
tmp_dict['version'] = version
90+
dependencies_list.append(tmp_dict)
91+
92+
assert len(all_dep) == len(dependencies_list), "Dependency extraction mismatch"
93+
return dependencies_list
94+
95+
96+
def create_defualt_nix(dependencies_list, meta_dict):
97+
"""
98+
Create a default.nix
99+
"""
100+
nix_content = """
101+
{
102+
pkgs ? import <nixpkgs> { },
103+
}:
104+
105+
let
106+
python = pkgs.python313;
107+
108+
# Helper function to override a package to disable tests
109+
disableAllTests =
110+
package: extraAttrs:
111+
package.overrideAttrs (
112+
old:
113+
{
114+
doCheck = false;
115+
doInstallCheck = false;
116+
doPytestCheck = false;
117+
pythonImportsCheck = [];
118+
checkPhase = "echo 'Tests disabled'";
119+
installCheckPhase = "echo 'Install checks disabled'";
120+
pytestCheckPhase = "echo 'Pytest checks disabled'";
121+
__intentionallyOverridingVersion = old.__intentionallyOverridingVersion or false;
122+
}
123+
// extraAttrs
124+
);
125+
126+
pythonOverlay = self: super: {
127+
"""
128+
need_review_packages_list = []
129+
deps_size = len(dependencies_list)
130+
for idx, dep in enumerate(dependencies_list):
131+
print("Processing {}/{}: {}".format(idx + 1, deps_size, dep['name']))
132+
name = dep['name']
133+
version = dep['version']
134+
# Handle 'django_notifications_patched','django-rest-hooks' and 'funcparserlib' separately
135+
if not name == 'django-rest-hooks' and not name == 'django_notifications_patched' and not name == 'funcparserlib':
136+
url = "https://pypi.org/pypi/{name}/{version}/json".format(name=name, version=version)
137+
try:
138+
response = requests.get(url)
139+
response.raise_for_status()
140+
data = response.json()
141+
142+
url_section = data.get("urls", [])
143+
build_from_src = True
144+
package_added = False
145+
for component in url_section:
146+
if component.get("packagetype") == "bdist_wheel":
147+
whl_url = component.get("url")
148+
whl_sha256 = get_sha256_hash(whl_url)
149+
nix_content += ' ' + name + ' = python.pkgs.buildPythonPackage {\n'
150+
nix_content += ' pname = "' + name + '";\n'
151+
nix_content += ' version = "' + version + '";\n'
152+
nix_content += ' format = "wheel";\n'
153+
nix_content += ' src = pkgs.fetchurl {\n'
154+
nix_content += ' url = "' + whl_url + '";\n'
155+
nix_content += ' sha256 = "' + whl_sha256 + '";\n'
156+
nix_content += ' };\n'
157+
nix_content += ' };\n'
158+
build_from_src = False
159+
package_added = True
160+
break
161+
162+
if build_from_src:
163+
for component in url_section:
164+
if component.get("packagetype") == "sdist":
165+
sdist_url = component.get("url")
166+
sdist_sha256 = get_sha256_hash(sdist_url)
167+
nix_content += ' ' + name + ' = disableAllTests super.' + name + ' {\n'
168+
nix_content += ' pname = "' + name + '";\n'
169+
nix_content += ' version = "' + version + '";\n'
170+
nix_content += ' __intentionallyOverridingVersion = true;\n'
171+
nix_content += ' src = pkgs.fetchurl {\n'
172+
nix_content += ' url = "' + sdist_url + '";\n'
173+
nix_content += ' sha256 = "' + sdist_sha256 + '";\n'
174+
nix_content += ' };\n'
175+
nix_content += ' };\n'
176+
package_added = True
177+
break
178+
if not package_added:
179+
need_review_packages_list.append(dep)
180+
except requests.exceptions.RequestException as e:
181+
need_review_packages_list.append(dep)
182+
else:
183+
if name == 'django-rest-hooks' and version == '1.6.1':
184+
nix_content += ' ' + name + ' = python.pkgs.buildPythonPackage {\n'
185+
nix_content += ' pname = "django-rest-hooks";\n'
186+
nix_content += ' version = "1.6.1";\n'
187+
nix_content += ' format = "wheel";\n'
188+
nix_content += ' src = pkgs.fetchurl {\n'
189+
nix_content += ' url = "https://github.com/aboutcode-org/django-rest-hooks/releases/download/1.6.1/django_rest_hooks-1.6.1-py2.py3-none-any.whl";\n'
190+
nix_content += ' sha256 = "1byakq3ghpqhm0mjjkh8v5y6g3wlnri2vvfifyi9ky36l12vqx74";\n'
191+
nix_content += ' };\n'
192+
nix_content += ' };\n'
193+
elif name == 'django_notifications_patched' and version == '2.0.0':
194+
nix_content += ' ' + name + ' = self.buildPythonPackage rec {\n'
195+
nix_content += ' pname = "django_notifications_patched";\n'
196+
nix_content += ' version = "2.0.0";\n'
197+
nix_content += ' format = "setuptools";\n'
198+
nix_content += ' doCheck = false;\n'
199+
nix_content += ' src = pkgs.fetchFromGitHub {\n'
200+
nix_content += ' owner = "dejacode";\n'
201+
nix_content += ' repo = "django-notifications-patched";\n'
202+
nix_content += ' rev = "2.0.0";\n'
203+
nix_content += ' url = "https://github.com/dejacode/django-notifications-patched/archive/refs/tags/2.0.0.tar.gz";\n'
204+
nix_content += ' sha256 = "sha256-RDAp2PKWa2xA5ge25VqkmRm8HCYVS4/fq2xKc80LDX8=";\n'
205+
nix_content += ' };\n'
206+
nix_content += ' };\n'
207+
elif name == 'funcparserlib' and version == '0.3.6':
208+
nix_content += ' ' + name + ' = self.buildPythonPackage rec {\n'
209+
nix_content += ' pname = "funcparserlib";\n'
210+
nix_content += ' version = "0.3.6";\n'
211+
nix_content += ' format = "setuptools";\n'
212+
nix_content += ' doCheck = false;\n'
213+
nix_content += ' src = pkgs.fetchurl {\n'
214+
nix_content += ' url = "https://files.pythonhosted.org/packages/cb/f7/b4a59c3ccf67c0082546eaeb454da1a6610e924d2e7a2a21f337ecae7b40/funcparserlib-0.3.6.tar.gz";\n'
215+
nix_content += ' sha256 = "07f9cgjr3h4j2m67fhwapn8fja87vazl58zsj4yppf9y3an2x6dp";\n'
216+
nix_content += ' };\n\n'
217+
# Original setpy.py: https://github.com/vlasovskikh/funcparserlib/blob/0.3.6/setup.py
218+
# funcparserlib version 0.3.6 uses use_2to3 which is no longer supported in modern setuptools.
219+
# Remove the "use_2to3" from the setup.py
220+
nix_content += " postPatch = ''\n"
221+
nix_content += ' cat > setup.py << EOF\n'
222+
nix_content += ' # -*- coding: utf-8 -*-\n'
223+
nix_content += ' from setuptools import setup\n'
224+
nix_content += ' setup(\n'
225+
nix_content += ' name="funcparserlib",\n'
226+
nix_content += ' version="0.3.6",\n'
227+
nix_content += ' packages=["funcparserlib", "funcparserlib.tests"],\n'
228+
nix_content += ' author="Andrey Vlasovskikh",\n'
229+
nix_content += ' description="Recursive descent parsing library based on functional combinators",\n'
230+
nix_content += ' license="MIT",\n'
231+
nix_content += ' url="http://code.google.com/p/funcparserlib/",\n'
232+
nix_content += ' )\n'
233+
nix_content += ' EOF\n'
234+
nix_content += " '';\n"
235+
nix_content += ' propagatedBuildInputs = with self; [];\n'
236+
nix_content += ' checkPhase = "echo \'Tests disabled for funcparserlib\'";\n'
237+
nix_content += ' };\n'
238+
else:
239+
need_review_packages_list.append(dep)
240+
nix_content += """
241+
};
242+
pythonWithOverlay = python.override {
243+
packageOverrides =
244+
self: super:
245+
let
246+
# Override buildPythonPackage to disable tests for ALL packages
247+
base = {
248+
buildPythonPackage =
249+
attrs:
250+
super.buildPythonPackage (
251+
attrs
252+
// {
253+
doCheck = false;
254+
doInstallCheck = false;
255+
doPytestCheck = false;
256+
pythonImportsCheck = [];
257+
}
258+
);
259+
};
260+
261+
# Apply custom package overrides
262+
custom = pythonOverlay self super;
263+
in
264+
base // custom;
265+
};
266+
267+
pythonApp = pythonWithOverlay.pkgs.buildPythonApplication {
268+
"""
269+
270+
nix_content += ' name = "' + meta_dict['name'] + '";\n'
271+
nix_content += ' version = "' + meta_dict['version'] + '";\n'
272+
273+
nix_content += """
274+
src = ./.;
275+
doCheck = false;
276+
doInstallCheck = false;
277+
doPytestCheck = false;
278+
pythonImportsCheck = [];
279+
280+
format = "pyproject";
281+
282+
nativeBuildInputs = with pythonWithOverlay.pkgs; [
283+
setuptools
284+
wheel
285+
pip
286+
];
287+
288+
propagatedBuildInputs = with pythonWithOverlay.pkgs; [
289+
"""
290+
291+
for dep in dependencies_list:
292+
name = dep['name']
293+
nix_content += ' ' + name + '\n'
294+
295+
nix_content += """
296+
];
297+
298+
meta = with pkgs.lib; {
299+
description = "Automate open source license compliance and ensure supply chain integrity";
300+
license = "AGPL-3.0-only";
301+
maintainers = ["AboutCode.org"];
302+
platforms = platforms.linux;
303+
};
304+
};
305+
306+
in
307+
{
308+
# Default output is the Python application
309+
app = pythonApp;
310+
311+
# Default to the application
312+
default = pythonApp;
313+
}
314+
"""
315+
return nix_content, need_review_packages_list
316+
317+
318+
def get_sha256_hash(url):
319+
"""
320+
Get SHA256 hash of a file using nix-prefetch-url.
321+
"""
322+
try:
323+
result = subprocess.run(
324+
['nix-prefetch-url', url],
325+
capture_output=True,
326+
text=True,
327+
check=True
328+
)
329+
return result.stdout.strip()
330+
except subprocess.CalledProcessError as e:
331+
print(f"Error running nix-prefetch-url for {url}: {e}")
332+
return None
333+
except FileNotFoundError:
334+
print("Error: nix-prefetch-url command not found. Make sure nix is installed.")
335+
return None
336+
337+
21338
def cleanup_nix_store():
22339
"""
23340
Remove the nix-store volume to ensure clean state
@@ -81,7 +398,6 @@ def build_nix_with_docker():
81398

82399
try:
83400
subprocess.run(docker_cmd, check=True)
84-
85401
# Verify if the output directory contains any files or
86402
# subdirectories.
87403
if any(output_dir.iterdir()):
@@ -94,18 +410,42 @@ def build_nix_with_docker():
94410
sys.exit(1)
95411

96412

97-
if __name__ == '__main__':
413+
def main():
98414
# Check if "docker" is available
99415
if not shutil.which('docker'):
100416
print("Error: Docker not found. Please install Docker first.", file=sys.stderr)
101417
sys.exit(1)
102418

103-
# Check if default.nix exists
104-
if not Path('default.nix').exists():
105-
print("Error: default.nix not found in current directory", file=sys.stderr)
106-
sys.exit(1)
419+
parser = argparse.ArgumentParser(description="Package to Nix using Docker.")
420+
parser.add_argument("--generate", action="store_true", help="Generate the default.nix file.")
421+
422+
args = parser.parse_args()
423+
424+
if args.generate or not Path('default.nix').exists():
425+
# Check if "nix-prefetch-url" is available
426+
if not shutil.which("nix-prefetch-url"):
427+
print("nix-prefetch-url is NOT installed.")
428+
sys.exit(1)
429+
430+
print("Generating default.nix")
431+
pyproject_data = read_pyproject_toml()
432+
meta_dict = extract_project_meta(pyproject_data)
433+
dependencies_list = extract_project_dependencies(pyproject_data)
434+
defualt_nix_content, need_review = create_defualt_nix(dependencies_list, meta_dict)
435+
with open("default.nix", "w") as file:
436+
file.write(defualt_nix_content)
437+
438+
print("default.nix file created successfully.")
439+
if need_review:
440+
print("\nThe following packages need manual review as they were not found on PyPI or had issues:")
441+
for pkg in need_review:
442+
print(f" - {pkg['name']}=={pkg['version']}")
443+
print("\nPlease review and add them manually to default.nix and re-run without the --generate.\n")
444+
sys.exit(1)
107445

108446
# Clean up the volume to ensure consistent state
109447
cleanup_nix_store()
110-
111448
build_nix_with_docker()
449+
450+
if __name__ == '__main__':
451+
main()

0 commit comments

Comments
 (0)