Generate pyconfig.h with cmake, add GRAALPY_VERSION

Author: msimacek

CHANGELOG.md

@@ -16,6 +16,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Rewrite wheelbuilder to be easier to use and contribute to. This version is now the same we run internally to build publishable wheels for some platforms we support, so the community can build the same wheels on their own hardware easily if desired.
 * `pip` is now able to fetch newer versions of GraalPy patches for third-party packages from `graalpython` GitHub repository, allowing us to add new patches to released versions.
   * The patch repository can be overridden using `PIP_GRAALPY_PATCHES_URL` environment variable, which can point to a local path or a URL. It can be disabled by setting it to an empty string.
+* Added `GRAALPY_VERSION` and `GRAALPY_VERSION_NUM` C macros
 
 ## Version 24.1.0
 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'.

graalpython/com.oracle.graal.python.cext/CMakeLists.txt

@@ -45,6 +45,7 @@ if(WIN32)
     require_var(GRAALVM_LLVM_LIB_DIR)
   endif()
 require_var(CAPI_INC_DIR)
+require_var(PYCONFIG_INCLUDE_DIR)
 require_var(TRUFFLE_H_INC)
 require_var(TRUFFLE_NFI_H_INC)
 require_var(GRAALPY_EXT)
@@ -150,6 +151,7 @@ include_directories(
     ${CAPI_SRC}
     "${SRC_DIR}/include"
     "${CAPI_INC_DIR}"
+    "${PYCONFIG_INCLUDE_DIR}"
     "${TRUFFLE_H_INC}"
     "${TRUFFLE_NFI_H_INC}"
 )

graalpython/com.oracle.graal.python.hpy.llvm/CMakeLists.txt

@@ -47,6 +47,7 @@ set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR})
 
 require_var(GRAALVM_HPY_INCLUDE_DIR)
 require_var(GRAALVM_PYTHON_INCLUDE_DIR)
+require_var(PYCONFIG_INCLUDE_DIR)
 require_var(TRUFFLE_H_INC)
 
 set(HPY_SRC "${SRC_DIR}/src")
@@ -91,6 +92,7 @@ target_sources(${TARGET_LIB} PRIVATE ${SRC_FILES} ${HPY_HEADERS})
 target_include_directories(${TARGET_LIB} PRIVATE
     "${GRAALVM_HPY_INCLUDE_DIR}"
     "${GRAALVM_PYTHON_INCLUDE_DIR}"
+    "${PYCONFIG_INCLUDE_DIR}"
     "${TRUFFLE_H_INC}"
 )
 

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_misc.py

@@ -41,9 +41,9 @@
 import os
 import pathlib
 import sys
+import unittest
 
-from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare
-
+from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare, CPyExtType
 
 __global_builtins_dict = builtins.__dict__
 
@@ -370,3 +370,30 @@ class TestMisc(CPyExtTestCase):
         arguments=["PyObject* value"],
         cmpfunc=unhandled_error_compare
     )
+
+
+@unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-only")
+def test_graalpy_version():
+    tester = CPyExtType(
+        "VersionTester",
+        code='''
+        static PyObject* get_version_str(PyObject* unused) {
+            return PyUnicode_FromString(GRAALPY_VERSION);
+        }
+        static PyObject* get_version_num(PyObject* unused) {
+            return PyLong_FromLong(GRAALPY_VERSION_NUM);
+        }
+        ''',
+        tp_methods='''
+        {"get_version_str", (PyCFunction)get_version_str, METH_NOARGS | METH_STATIC, ""},
+        {"get_version_num", (PyCFunction)get_version_num, METH_NOARGS | METH_STATIC, ""}
+        ''',
+    )
+    expected_version = __graalpython__.get_graalvm_version().removesuffix('-dev')
+    assert tester.get_version_str() == expected_version
+    parts = [int(v) for v in expected_version.split('.')] + [0]
+    expected_num = 0
+    for i in range(3):
+        expected_num <<= 8
+        expected_num |= parts[i]
+    assert tester.get_version_num() == expected_num

graalpython/graalpy-pyconfig/CMakeLists.txt

@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2024, Oracle and/or its affiliates.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+# conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list of
+# conditions and the following disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+cmake_minimum_required(VERSION 3.22)
+project(graalpy-pyconfig)
+
+include(CheckSymbolExists)
+include(CheckIncludeFile)
+include (TestBigEndian)
+
+test_big_endian(IS_BIG_ENDIAN)
+
+set(CMAKE_REQUIRED_LINK_OPTIONS "-lm")
+
+check_include_file("dirent.h" HAVE_DIRENT_H)
+check_include_file("errno.h" HAVE_ERRNO_H)
+check_include_file("utime.h" HAVE_UTIME_H)
+check_include_file("signal.h" HAVE_SIGNAL_H)
+check_include_file("fcntl.h" HAVE_FCNTL_H)
+check_include_file("wchar.h" HAVE_WCHAR_H)
+check_include_file("unistd.h" HAVE_UNISTD_H)
+check_include_file("pthread.h" HAVE_PTHREAD_H)
+check_include_file("sys/stat.h" HAVE_SYS_STAT_H)
+check_include_file("sys/wait.h" HAVE_SYS_WAIT_H)
+check_include_file("sys/time.h" HAVE_SYS_TIME_H)
+
+check_symbol_exists(acosh "math.h" HAVE_ACOSH)
+check_symbol_exists(asinh "math.h" HAVE_ASINH)
+check_symbol_exists(atanh "math.h" HAVE_ATANH)
+check_symbol_exists(copysign "math.h" HAVE_COPYSIGN)
+check_symbol_exists(round "math.h" HAVE_ROUND)
+check_symbol_exists(hypot "math.h" HAVE_HYPOT)
+check_symbol_exists(clock "time.h" HAVE_CLOCK)
+check_symbol_exists(sendfile "sys/sendfile.h" HAVE_SENDFILE)
+
+if(HAVE_SYS_TIME_H)
+    set(TIME_WITH_SYS_TIME 1)
+endif()
+if(WIN32)
+    set(NT_THREADS 1)
+endif()
+if(IS_BIG_ENDIAN)
+    set(DOUBLE_IS_BIG_ENDIAN_IEEE754 1)
+else()
+    set(DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1)
+endif()
+
+configure_file("pyconfig.h.in" "pyconfig.h" @ONLY)

graalpython/com.oracle.graal.python.cext/include/pyconfig.h renamed to graalpython/graalpy-pyconfig/pyconfig.h.in

@@ -42,6 +42,8 @@
 #define Py_PYCONFIG_H
 
 #define GRAALVM_PYTHON 1
+#cmakedefine GRAALPY_VERSION "@GRAALPY_VERSION@"
+#cmakedefine GRAALPY_VERSION_NUM @GRAALPY_VERSION_NUM@
 
 // The graalpy build always sets MS_WINDOWS, so when this is not set, we are
 // dealing with an extension build. In that case, if we're on Windows, we need
@@ -148,44 +150,37 @@
 // #define Py_LIMITED_API 1
 // END TRUFFLE DEFS
 
-#define HAVE_ACOSH 1
-#define HAVE_ASINH 1
-#define HAVE_ATANH 1
-#define HAVE_CLOCK 1
-#define HAVE_DIRENT_H 1
-#define HAVE_SENDFILE 1
-#define HAVE_SYS_STAT_H 1
-#define HAVE_ERRNO_H 1
-#define HAVE_UTIME_H
-#define HAVE_SIGNAL_H
-#define HAVE_FCNTL_H
-
-#define HAVE_STDARG_PROTOTYPES
-
-#define HAVE_WCHAR_H 1
-
+#cmakedefine HAVE_ACOSH 1
+#cmakedefine HAVE_ASINH 1
+#cmakedefine HAVE_ATANH 1
+#cmakedefine HAVE_COPYSIGN 1
+#cmakedefine HAVE_ROUND 1
+#cmakedefine HAVE_HYPOT 1
+#cmakedefine HAVE_CLOCK 1
+#cmakedefine HAVE_SENDFILE 1
+#cmakedefine HAVE_DIRENT_H 1
+#cmakedefine HAVE_ERRNO_H 1
+#cmakedefine HAVE_UTIME_H 1
+#cmakedefine HAVE_SIGNAL_H 1
+#cmakedefine HAVE_FCNTL_H 1
+#cmakedefine HAVE_WCHAR_H 1
+#cmakedefine HAVE_UNISTD_H 1
+#cmakedefine HAVE_PTHREAD_H 1
+#cmakedefine HAVE_SYS_WAIT_H 1
+#cmakedefine HAVE_SYS_TIME_H 1
+#cmakedefine HAVE_SYS_STAT_H 1
+
+#cmakedefine TIME_WITH_SYS_TIME 1
+#cmakedefine NT_THREADS 1
+
+#define HAVE_STDARG_PROTOTYPES 1
 #define WITH_THREAD 1
 #define WITH_DOC_STRINGS 1
 
-#ifndef MS_WINDOWS
-#define HAVE_UNISTD_H
-#define HAVE_PTHREAD_H
-#define HAVE_SYS_WAIT_H
-#define HAVE_SYS_TIME_H
-#define TIME_WITH_SYS_TIME 1
-#else
-#define HAVE_COPYSIGN 1
-#define HAVE_ROUND 1
-#define HAVE_HYPOT 1
-#define NT_THREADS 1
-#endif
-
-// The following should have been generated using `configure` command,
-// but instead, are set manully for the time being.
-
 /* Define if C doubles are 64-bit IEEE 754 binary format, stored with the
    least significant byte first */
-#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1
+#cmakedefine DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1
+#cmakedefine DOUBLE_IS_BIG_ENDIAN_IEEE754 1
 
 
 #endif /*Py_PYCONFIG_H*/

mx.graalpython/mx_graalpython.py

@@ -1706,12 +1706,21 @@ def py_version_short(variant=None, **kwargs):
 def graal_version_short(variant=None, **kwargs):
     if variant == 'major_minor_nodot':
         return GRAAL_VERSION_MAJ_MIN.replace(".", "")
+    elif variant == 'major_minor':
+        return GRAAL_VERSION_MAJ_MIN
     elif variant == 'binary':
         # PythonLanguage and PythonResource consume this data, and they assume 3 components, so we cap the list size
         # to 3 although the version may have even more components
         return "".join([chr(int(p) + ord(VERSION_BASE)) for p in GRAAL_VERSION.split(".")[:3]])
+    elif variant == 'hex':
+        parts = GRAAL_VERSION.split(".")
+        num = 0
+        for i in range(3):
+            num <<= 8
+            num |= int(parts[i]) if i < len(parts) else 0
+        return hex(num)
     else:
-        return GRAAL_VERSION_MAJ_MIN
+        return '.'.join(GRAAL_VERSION.split('.')[:3])
 
 
 def release_level(variant=None):

mx.graalpython/suite.py

@@ -672,6 +672,20 @@
             ],
         },
 
+        "graalpy-pyconfig": {
+            "subDir": "graalpython",
+            "class": "CMakeNinjaProject",
+            "max_jobs": "1",
+            "ninja_targets": ["all"],
+            "cmakeConfig": {
+                "GRAALPY_VERSION": "<graal_ver>",
+                "GRAALPY_VERSION_NUM": "<graal_ver:hex>",
+            },
+            "results": [
+                "pyconfig.h",
+            ],
+        },
+
         "com.oracle.graal.python.cext": {
             "subDir": "graalpython",
             "class": "CMakeNinjaProject",
@@ -686,6 +700,7 @@
                     "<others>": {
                         "cmakeConfig": {
                             "CAPI_INC_DIR": "<output_root:com.oracle.graal.python>/jni_gen",
+                            "PYCONFIG_INCLUDE_DIR": "<output_root:graalpy-pyconfig>/<arch>",
                             "GRAALVM_LLVM_LIB_DIR": "<path:SULONG_NATIVE_HOME>/native/lib",
                             "TRUFFLE_H_INC": "<path:SULONG_LEGACY>/include",
                             "TRUFFLE_NFI_H_INC": "<path:com.oracle.truffle.nfi.native>/include",
@@ -706,6 +721,7 @@
                     "<others>": {
                         "cmakeConfig": {
                             "CAPI_INC_DIR": "<output_root:com.oracle.graal.python>/jni_gen",
+                            "PYCONFIG_INCLUDE_DIR": "<output_root:graalpy-pyconfig>/<arch>",
                             "TRUFFLE_H_INC": "<path:SULONG_LEGACY>/include",
                             "TRUFFLE_NFI_H_INC": "<path:com.oracle.truffle.nfi.native>/include",
                             "CMAKE_C_COMPILER": "<toolchainGetToolPath:native,CC>",
@@ -731,6 +747,7 @@
                 "sulong:SULONG_HOME",
                 "sulong:SULONG_BOOTSTRAP_TOOLCHAIN",
                 "sulong:SULONG_LEGACY",
+                "graalpy-pyconfig",
                 "com.oracle.graal.python",
             ],
         },
@@ -749,6 +766,7 @@
                 "bin/<lib:hpy-native>",
             ],
             "buildDependencies": [
+                "graalpy-pyconfig",
                 "sulong:SULONG_HOME",
                 "sulong:SULONG_BOOTSTRAP_TOOLCHAIN",
                 "sulong:SULONG_LEGACY",
@@ -757,6 +775,7 @@
                 "windows": {
                     "<others>": {
                         "cmakeConfig": {
+                            "PYCONFIG_INCLUDE_DIR": "<output_root:graalpy-pyconfig>/<arch>",
                             "GRAALVM_LLVM_LIB_DIR": "<path:SULONG_NATIVE_HOME>/native/lib",
                             "GRAALVM_HPY_INCLUDE_DIR": "<path:com.oracle.graal.python.hpy.llvm>/include",
                             "GRAALVM_PYTHON_INCLUDE_DIR": "<path:com.oracle.graal.python.cext>/include",
@@ -768,6 +787,7 @@
                 "<others>": {
                     "<others>": {
                         "cmakeConfig": {
+                            "PYCONFIG_INCLUDE_DIR": "<output_root:graalpy-pyconfig>/<arch>",
                             "GRAALVM_HPY_INCLUDE_DIR": "<path:com.oracle.graal.python.hpy.llvm>/include",
                             "GRAALVM_PYTHON_INCLUDE_DIR": "<path:com.oracle.graal.python.cext>/include",
                             "TRUFFLE_H_INC": "<path:SULONG_LEGACY>/include",
@@ -783,6 +803,7 @@
             "native": "shared_lib",
             "deliverable": "pythonjni",
             "buildDependencies": [
+                "graalpy-pyconfig",
                 "com.oracle.graal.python", # for the generated JNI header file
             ],
             "use_jdk_headers": True, # the generated JNI header includes jni.h
@@ -801,7 +822,8 @@
                             "-I\"<path:com.oracle.graal.python.cext>/src\"",
                             "-I\"<path:com.oracle.graal.python.hpy.llvm>/include\"",
                             "-I\"<path:com.oracle.graal.python.hpy.llvm>/src\"",
-                            "-I\"<path:com.oracle.truffle.nfi.native>/include\""
+                            "-I\"<path:com.oracle.truffle.nfi.native>/include\"",
+                            "-I\"<output_root:graalpy-pyconfig>/<arch>\"",
                         ],
                     },
                 },
@@ -815,7 +837,8 @@
                             "-I\"<path:com.oracle.graal.python.cext>/src\"",
                             "-I\"<path:com.oracle.graal.python.hpy.llvm>/include\"",
                             "-I\"<path:com.oracle.graal.python.hpy.llvm>/src\"",
-                            "-I\"<path:com.oracle.truffle.nfi.native>/include\""
+                            "-I\"<path:com.oracle.truffle.nfi.native>/include\"",
+                            "-I\"<output_root:graalpy-pyconfig>/<arch>\"",
                         ],
                     },
                 },
@@ -1341,6 +1364,7 @@
             "description": "GraalVM Python header resources",
             "layout": {
                 "./META-INF/resources/include/": [
+                    "dependency:graalpy-pyconfig/pyconfig.h",
                     "file:graalpython/com.oracle.graal.python.cext/include/*",
                     "file:graalpython/com.oracle.graal.python.hpy.llvm/include/*",
                 ],