Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/mypy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ on:
- "Tools/build/check_warnings.py"
- "Tools/build/compute-changes.py"
- "Tools/build/deepfreeze.py"
- "Tools/build/generate-build-details.py"
- "Tools/build/generate_build_details.py"
- "Tools/build/generate_sbom.py"
- "Tools/build/generate_stdlib_module_names.py"
- "Tools/build/mypy.ini"
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ gmon.out
.pytest_cache/
.ruff_cache/
.DS_Store
__install__.json
build-details.json

*.exe

Expand Down
27 changes: 6 additions & 21 deletions Lib/test/test_build_details.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,18 @@
import importlib
import json
import os
import os.path
import sys
import sysconfig
import string
import unittest
from pathlib import Path

from test.support import is_android, is_apple_mobile, is_wasm32

BASE_PATH = Path(
__file__, # Lib/test/test_build_details.py
'..', # Lib/test
'..', # Lib
'..', # <src/install dir>
).resolve()
MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py'
from test.test_tools import imports_under_tool

try:
# Import "generate-build-details.py" as "generate_build_details"
spec = importlib.util.spec_from_file_location(
"generate_build_details", MODULE_PATH
)
generate_build_details = importlib.util.module_from_spec(spec)
sys.modules["generate_build_details"] = generate_build_details
spec.loader.exec_module(generate_build_details)
except (FileNotFoundError, ImportError):
with imports_under_tool('build'):
import generate_build_details
except ImportError:
generate_build_details = None


Expand Down Expand Up @@ -178,9 +164,8 @@ def test_c_api(self):

@unittest.skipIf(
generate_build_details is None,
"Failed to import generate-build-details"
"Failed to import generate_build_details",
)
@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now')
@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds')
class BuildDetailsRelativePathsTests(unittest.TestCase):
@property
Expand All @@ -191,7 +176,7 @@ def build_details_absolute_paths(self):
@property
def build_details_relative_paths(self):
data = self.build_details_absolute_paths
generate_build_details.make_paths_relative(data, config_path=None)
generate_build_details.make_paths_relative(data, base_path=None)
return data

def test_round_trip(self):
Expand Down
2 changes: 1 addition & 1 deletion Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,7 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS)
fi

build-details.json: pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json
$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate_build_details.py `cat pybuilddir.txt`/build-details.json

# Build static library
$(LIBRARY): $(LIBRARY_OBJS)
Expand Down
10 changes: 10 additions & 0 deletions PC/layout/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
__path__ = [str(Path(__file__).resolve().parent)]

from .support.appxmanifest import *
from .support.build_details import *
from .support.catalog import *
from .support.constants import *
from .support.filesets import *
Expand Down Expand Up @@ -319,6 +320,9 @@ def _c(d):
if ns.include_install_json or ns.include_install_embed_json or ns.include_install_test_json:
yield "__install__.json", ns.temp / "__install__.json"

if ns.include_build_details_json:
yield "build-details.json", ns.temp / "build-details.json"


def _compile_one_py(src, dest, name, optimize, checked=True):
import py_compile
Expand Down Expand Up @@ -426,6 +430,12 @@ def generate_source_files(ns):
with open(ns.temp / "__install__.json", "w", encoding="utf-8") as f:
json.dump(calculate_install_json(ns, for_test=True), f, indent=2)

if ns.include_build_details_json:
log_info("Generating build-details.json in {}", ns.temp)
ns.temp.mkdir(parents=True, exist_ok=True)
base_path = Path(sys.base_prefix, "build-details.json")
write_relative_build_details(ns.temp / "build-details.json", base_path)


def _create_zip_file(ns):
if not ns.zip:
Expand Down
37 changes: 37 additions & 0 deletions PC/layout/support/build_details.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""
Generate the PEP 739 'build-details.json' document.
"""

import sys
from pathlib import Path

PEP739_SCHEMA_VERSION = '1.0'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
PEP739_SCHEMA_VERSION = '1.0'
__all__ = ["write_relative_build_details"]
PEP739_SCHEMA_VERSION = '1.0'


ROOT_DIR = Path(
__file__, # PC/layout/support/build_details.py
'..', # PC/layout/support
'..', # PC/layout
'..', # PC
'..', # <src/install dir>
).resolve()
TOOLS_BUILD_DIR = ROOT_DIR / 'Tools' / 'build'

sys_path = sys.path[:]
try:
sys.path.insert(0, str(TOOLS_BUILD_DIR))
import generate_build_details
except ImportError:
generate_build_details = None
finally:
sys.path = sys_path
del sys_path
Comment on lines +10 to +27
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm afraid this probably needs to replicate most of the functionality from that script and do trickier calculations. The repo containing the running PC/layout isn't necessarily the same as the commit used to build, and the running interpreter isn't necessarily the same as the one we just built.

So the details have to be calculated from the referenced source files (i.e. layout.support.constants and layout.support.arch), not from the current runtime at all.

That said, I haven't dug into the write_build_details function and maybe it does it, but it really doesn't look like enough information is being provided to make it viable (especially when you factor in that {base_path}\python.exe is not executable).

Comment on lines +10 to +27
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need to import the script here instead of invoking it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't do either - we might need to generate the file without being able to launch the runtime. See my comment above on the same section of code.



def write_relative_build_details(out_path, base_path):
if generate_build_details is None:
return
generate_build_details.write_build_details(
schema_version=PEP739_SCHEMA_VERSION,
base_path=base_path,
location=out_path,
)
9 changes: 8 additions & 1 deletion PC/layout/support/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def public(f):
"install-json": {"help": "a PyManager __install__.json file"},
"install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"},
"install-test-json": {"help": "a PyManager __install__.json for the test distro"},
"build-details-json": {"help": "a PEP 739 build-details.json file"},
}


Expand All @@ -56,6 +57,7 @@ def public(f):
"appxmanifest",
"alias",
"alias3x",
"build-details-json",
# XXX: Disabled for now "precompile",
],
},
Expand All @@ -69,9 +71,10 @@ def public(f):
"props",
"nuspec",
"alias",
"build-details-json",
],
},
"iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]},
"iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip", "build-details-json"]},
"default": {
"help": "development kit package",
"options": [
Expand All @@ -85,6 +88,7 @@ def public(f):
"symbols",
"html-doc",
"alias",
"build-details-json",
],
},
"embed": {
Expand All @@ -96,6 +100,7 @@ def public(f):
"flat-dlls",
"underpth",
"precompile",
"build-details-json",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suspect the contents of the file needs to be different for the embeddable package (no Lib directory or Includes/libs, for example). Maybe just omit it for now

],
},
"pymanager": {
Expand All @@ -109,6 +114,7 @@ def public(f):
"dev",
"html-doc",
"install-json",
"build-details-json",
],
},
"pymanager-test": {
Expand All @@ -124,6 +130,7 @@ def public(f):
"symbols",
"tests",
"install-test-json",
"build-details-json",
],
},
}
Expand Down
2 changes: 1 addition & 1 deletion PCbuild/python.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@
<Message Text="Generating $(OutDir)build-details.json" />
<Exec Command='setlocal
set PYTHONPATH=$(PySourcePath)Lib
"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)Tools\build\generate-build-details.py" "$(OutDir)build-details.json"' ContinueOnError="true" />
"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)Tools\build\generate_build_details.py" "$(OutDir)build-details.json"' ContinueOnError="true" />
</Target>
<Target Name="ValidateUcrtbase" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGInstrument' and $(Platform) != 'ARM' and $(Platform) != 'ARM64'">
<PropertyGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,30 @@
import os
import sys
import sysconfig
from pathlib import Path

TYPE_CHECKING = False
if TYPE_CHECKING:
from typing import Any
from typing import Any, Literal

type StrPath = str | os.PathLike[str]

Check failure on line 21 in Tools/build/generate_build_details.py

View workflow job for this annotation

GitHub Actions / lint

Ruff (invalid-syntax)

Tools/build/generate_build_details.py:21:5: invalid-syntax: Cannot use `type` alias statement on Python 3.10 (syntax was added in Python 3.12)
type ValidSchemaVersion = Literal['1.0']

Check failure on line 22 in Tools/build/generate_build_details.py

View workflow job for this annotation

GitHub Actions / lint

Ruff (invalid-syntax)

Tools/build/generate_build_details.py:22:5: invalid-syntax: Cannot use `type` alias statement on Python 3.10 (syntax was added in Python 3.12)


def write_build_details(
*,
schema_version: ValidSchemaVersion,
base_path: StrPath | None,
location: StrPath,
) -> None:
data = generate_data(schema_version)
if base_path is not None:
make_paths_relative(data, base_path)

json_output = json.dumps(data, indent=2)
with open(location, 'w', encoding='utf-8') as f:
f.write(json_output)
f.write('\n')


def version_info_to_dict(obj: sys._version_info) -> dict[str, Any]:
Expand All @@ -29,7 +49,9 @@
return container


def generate_data(schema_version: str) -> collections.defaultdict[str, Any]:
def generate_data(
schema_version: ValidSchemaVersion
) -> collections.defaultdict[str, Any]:
"""Generate the build-details.json data (PEP 739).

:param schema_version: The schema version of the data we want to generate.
Expand All @@ -46,10 +68,15 @@

data['base_prefix'] = sysconfig.get_config_var('installed_base')
#data['base_interpreter'] = sys._base_executable
data['base_interpreter'] = os.path.join(
sysconfig.get_path('scripts'),
'python' + sysconfig.get_config_var('VERSION'),
)
if os.name == 'nt':
data['base_interpreter'] = os.path.join(
data['base_prefix'], os.path.basename(sys._base_executable)
)
else:
data['base_interpreter'] = os.path.join(
sysconfig.get_path('scripts'),
'python' + sysconfig.get_config_var('VERSION'),
)
data['platform'] = sysconfig.get_platform()

data['language']['version'] = sysconfig.get_python_version()
Expand All @@ -72,13 +99,19 @@
#data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES
data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES

LIBDIR = sysconfig.get_config_var('LIBDIR')
if os.name == 'nt':
LIBDIR = data['base_prefix']
else:
LIBDIR = sysconfig.get_config_var('LIBDIR')
LDLIBRARY = sysconfig.get_config_var('LDLIBRARY')
LIBRARY = sysconfig.get_config_var('LIBRARY')
PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY')
LIBPYTHON = sysconfig.get_config_var('LIBPYTHON')
LIBPC = sysconfig.get_config_var('LIBPC')
INCLUDEPY = sysconfig.get_config_var('INCLUDEPY')
if os.name == 'nt':
INCLUDEPY = os.path.join(data['base_prefix'], 'include')
else:
INCLUDEPY = sysconfig.get_config_var('INCLUDEPY')

if os.name == 'posix':
# On POSIX, LIBRARY is always the static library, while LDLIBRARY is the
Expand Down Expand Up @@ -133,11 +166,7 @@
return data


def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None:
# Make base_prefix relative to the config_path directory
if config_path:
data['base_prefix'] = relative_path(data['base_prefix'],
os.path.dirname(config_path))
def make_paths_relative(data: dict[str, Any], base_path: str | None = None) -> None:
base_prefix = data['base_prefix']

# Update path values to make them relative to base_prefix
Expand Down Expand Up @@ -167,8 +196,12 @@
new_path = os.path.join('.', new_path)
container[child] = new_path

if base_path:
# Make base_prefix relative to the base_path directory
config_dir = Path(base_path).resolve().parent
data['base_prefix'] = relative_path(base_prefix, config_dir)

def relative_path(path: str, base: str) -> str:
def relative_path(path: StrPath, base: StrPath) -> str:
if os.name != 'nt':
return os.path.relpath(path, base)

Expand Down Expand Up @@ -201,15 +234,18 @@
)

args = parser.parse_args()

data = generate_data(args.schema_version)
if args.relative_paths:
make_paths_relative(data, args.config_file_path)

json_output = json.dumps(data, indent=2)
with open(args.location, 'w', encoding='utf-8') as f:
f.write(json_output)
f.write('\n')
if os.name == 'nt':
# Windows builds are relocatable; always make paths relative.
base_path = args.config_file_path or args.location
elif args.relative_paths:
base_path = args.config_file_path
else:
base_path = None
write_build_details(
schema_version=args.schema_version,
base_path=base_path,
location=args.location,
)


if __name__ == '__main__':
Expand Down
2 changes: 1 addition & 1 deletion Tools/build/mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ files =
Tools/build/check_warnings.py,
Tools/build/compute-changes.py,
Tools/build/deepfreeze.py,
Tools/build/generate-build-details.py,
Tools/build/generate_build_details.py,
Tools/build/generate_sbom.py,
Tools/build/generate_stdlib_module_names.py,
Tools/build/verify_ensurepip_wheels.py,
Expand Down
Loading