Skip to content

Commit b361341

Browse files
committed
[GR-47457][GR-42003] Enable sqlite3, ctypes, venv creation, and pip installation on Windows
PullRequest: graalpython/2881
2 parents 8db45d8 + 6c052c0 commit b361341

File tree

26 files changed

+382
-310
lines changed

26 files changed

+382
-310
lines changed

graalpython/com.oracle.graal.python.cext/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,12 +174,12 @@ simple_native_module("_mmap")
174174
simple_native_module("_cpython_sre")
175175
simple_native_module("_cpython_unicodedata")
176176
simple_native_module("_cpython_struct")
177+
simple_native_module("_sqlite3")
177178

178179
if(NOT WIN32)
179180
native_module("_testcapi" FALSE "${SRC_DIR}/modules/_testcapi.c")
180181
simple_native_module("_testmultiphase")
181182
simple_native_module("_ctypes_test")
182-
simple_native_module("_sqlite3")
183183

184184
###################### BZIP2 ########################
185185
if(DEFINED LIBBZ2_BUILD_FILE)

graalpython/com.oracle.graal.python.cext/modules/_sqlite/module.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,10 @@ PyMODINIT_FUNC PyInit__sqlite3(void)
420420
goto error;
421421
}
422422

423-
if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) {
423+
/**
424+
* GraalPy change: use SQLITE_VERSION from builtin sqlite3
425+
*/
426+
if (PyModule_AddStringConstant(module, "sqlite_version", SQLITE_VERSION)) {
424427
goto error;
425428
}
426429

graalpython/com.oracle.graal.python.cext/modules/_sqlite3.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@
3939
* SOFTWARE.
4040
*/
4141

42+
// prevent unresolved symbols
43+
#define SQLITE_EXTERN
44+
4245
#include "_sqlite/cache.c"
4346
#include "_sqlite/connection.c"
4447
#include "_sqlite/cursor.c"

graalpython/com.oracle.graal.python.cext/src/capi.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,7 +1032,7 @@ PyAPI_FUNC(PyMemoryViewObject*) PyTruffle_AllocateMemoryView(PyMemoryViewObject*
10321032
}
10331033

10341034
#define PRIMITIVE_ARRAY_TO_NATIVE(__jtype__, __ctype__, __polyglot_type__, __element_cast__) \
1035-
void* PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
1035+
PyAPI_FUNC(void*) PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
10361036
int64_t i; \
10371037
int64_t size = len + 1; \
10381038
__ctype__* carr = (__ctype__*) malloc(size * sizeof(__ctype__)); \
@@ -1042,7 +1042,7 @@ PyAPI_FUNC(PyMemoryViewObject*) PyTruffle_AllocateMemoryView(PyMemoryViewObject*
10421042
} \
10431043
return polyglot_from_##__polyglot_type__##_array(carr, len); \
10441044
} \
1045-
void* PyTruffle_##__jtype__##ArrayRealloc(const void* array, int64_t len) { \
1045+
PyAPI_FUNC(void*) PyTruffle_##__jtype__##ArrayRealloc(const void* array, int64_t len) { \
10461046
int64_t size = len + 1; \
10471047
__ctype__* carr = (__ctype__*) realloc(array, size * sizeof(__ctype__)); \
10481048
carr[len] = (__ctype__)0; \
@@ -1055,22 +1055,22 @@ PRIMITIVE_ARRAY_TO_NATIVE(Long, int64_t, i64, polyglot_as_i64);
10551055
PRIMITIVE_ARRAY_TO_NATIVE(Double, double, double, polyglot_as_double);
10561056
PRIMITIVE_ARRAY_TO_NATIVE(Object, PyObjectPtr, PyObjectPtr, (PyObjectPtr));
10571057

1058-
void PyTruffle_PrimitiveArrayFree(void* array) {
1058+
PyAPI_FUNC(void) PyTruffle_PrimitiveArrayFree(void* array) {
10591059
free(array);
10601060
}
10611061

1062-
void PyTruffle_ObjectArrayFree(PyObject** array, int32_t size) {
1062+
PyAPI_FUNC(void) PyTruffle_ObjectArrayFree(PyObject** array, int32_t size) {
10631063
for (int i = 0; i < size; i++) {
10641064
Py_DECREF(array[i]);
10651065
}
10661066
free(array);
10671067
}
10681068

1069-
void PyTruffle_SetStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
1069+
PyAPI_FUNC(void) PyTruffle_SetStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
10701070
Py_XSETREF(ptr[index], newitem);
10711071
}
10721072

1073-
void PyTruffle_InitializeStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
1073+
PyAPI_FUNC(void) PyTruffle_InitializeStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
10741074
ptr[index] = newitem;
10751075
}
10761076

graalpython/com.oracle.graal.python.cext/src/dictobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ int PyDict_DelItemString(PyObject *d, const char *key) {
110110
CALL_WITH_STRING(key, int, -1, GraalPyDict_DelItem, d, string);
111111
}
112112

113-
PyObject* _PyObject_GenericGetDict(PyObject* obj) {
113+
PyAPI_FUNC(PyObject*) _PyObject_GenericGetDict(PyObject* obj) {
114114
PyObject** dictptr = _PyObject_GetDictPtr(obj);
115115
if (dictptr == NULL) {
116116
return NULL;

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,6 @@ def add_graalpython_core():
113113
"_sysconfig",
114114
"_weakref",
115115
"builtins",
116-
"bytearray",
117-
"ctypes",
118116
"function",
119117
"java",
120118
"pip_hook",

graalpython/com.oracle.graal.python.test/src/tests/test_venv.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@
4444
import tempfile
4545

4646

47+
BINDIR = 'bin' if sys.platform != 'win32' else 'Scripts'
48+
EXESUF = '' if sys.platform != 'win32' else '.cmd'
49+
50+
4751
class VenvTest():
4852
def setUpClass(self):
4953
self.env_dir = os.path.realpath(tempfile.mkdtemp())
@@ -58,20 +62,21 @@ def test_create_and_use_basic_venv(self):
5862
run_output = ''
5963
try:
6064
subprocess.check_output([sys.executable, "-m", "venv", self.env_dir, "--without-pip"], stderr=subprocess.STDOUT)
61-
run = subprocess.getoutput(". %s/bin/activate; python -m site" % self.env_dir)
65+
run = subprocess.getoutput(f"{self.env_dir}/{BINDIR}/python{EXESUF} -m site")
6266
except subprocess.CalledProcessError as err:
6367
if err.output:
6468
run_output = err.output.decode(errors="replace")
6569
assert run, run_output
6670
assert "ENABLE_USER_SITE: False" in run, run
67-
assert self.env_dir in run, run
71+
if sys.platform != 'win32':
72+
assert self.env_dir in run, run
6873

6974
def test_create_and_use_venv_with_pip(self):
7075
run = None
7176
msg = ''
7277
try:
7378
subprocess.check_output([sys.executable, "-m", "venv", self.env_dir2], stderr=subprocess.STDOUT)
74-
run = subprocess.getoutput("%s/bin/python -m pip list" % self.env_dir2)
79+
run = subprocess.getoutput(f"{self.env_dir2}/{BINDIR}/python{EXESUF} -m pip list")
7580
except subprocess.CalledProcessError as err:
7681
if err.output:
7782
run_output = err.output.decode(errors="replace")

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import java.util.ServiceLoader;
5656
import java.util.logging.Level;
5757

58+
import com.oracle.graal.python.builtins.modules.PyExpatModuleBuiltins;
5859
import org.graalvm.nativeimage.ImageInfo;
5960

6061
import com.oracle.graal.python.PythonLanguage;
@@ -380,7 +381,6 @@ private static TruffleString[] initializeCoreFiles() {
380381
List<TruffleString> coreFiles = new ArrayList<>(Arrays.asList(
381382
toTruffleStringUncached("__graalpython__"),
382383
toTruffleStringUncached("_weakref"),
383-
toTruffleStringUncached("bytearray"),
384384
toTruffleStringUncached("unicodedata"),
385385
toTruffleStringUncached("_sre"),
386386
toTruffleStringUncached("function"),
@@ -630,6 +630,8 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
630630
PythonOptions.WITHOUT_DIGEST ? null : new Blake2sObjectBuiltins(),
631631
PythonOptions.WITHOUT_DIGEST ? null : new HashlibModuleBuiltins(),
632632

633+
new PyExpatModuleBuiltins(),
634+
633635
// itertools
634636
new AccumulateBuiltins(),
635637
new CombinationsBuiltins(),

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,32 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
7777
public void initialize(Python3Core core) {
7878
super.initialize(core);
7979
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
80+
addBuiltinConstant("_LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR", 0x100);
81+
addBuiltinConstant("_LOAD_LIBRARY_SEARCH_DEFAULT_DIRS", 0x1000);
8082
core.removeBuiltinModule(T_POSIX);
8183
} else {
8284
core.removeBuiltinModule(T_NT);
8385
}
8486
}
8587

88+
@Builtin(name = "_getfullpathname", minNumOfPositionalArgs = 1, parameterNames = {"path"})
89+
@ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
90+
@GenerateNodeFactory
91+
abstract static class GetfullpathnameNode extends PythonUnaryClinicBuiltinNode {
92+
@Specialization
93+
@TruffleBoundary
94+
Object getfullpathname(PosixPath path,
95+
@CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib) {
96+
// TODO should call win api
97+
return posixLib.getPathAsString(getPosixSupport(), path.value);
98+
}
99+
100+
@Override
101+
protected ArgumentClinicProvider getArgumentClinic() {
102+
return NtModuleBuiltinsClinicProviders.GetfullpathnameNodeClinicProviderGen.INSTANCE;
103+
}
104+
}
105+
86106
@Builtin(name = "_path_splitroot", minNumOfPositionalArgs = 1, parameterNames = {"path"})
87107
@ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
88108
@GenerateNodeFactory
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.builtins.modules;
42+
43+
import java.util.LinkedHashMap;
44+
import java.util.List;
45+
46+
import com.oracle.graal.python.builtins.Builtin;
47+
import com.oracle.graal.python.builtins.CoreFunctions;
48+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
49+
import com.oracle.graal.python.builtins.PythonBuiltins;
50+
import com.oracle.graal.python.builtins.PythonOS;
51+
import com.oracle.graal.python.builtins.objects.module.PythonModule;
52+
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
53+
import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
54+
import com.oracle.graal.python.builtins.Python3Core;
55+
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
56+
import com.oracle.truffle.api.dsl.NodeFactory;
57+
import com.oracle.truffle.api.dsl.Specialization;
58+
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
59+
60+
import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
61+
62+
@CoreFunctions(defineModule = "pyexpat", os = PythonOS.PLATFORM_WIN32)
63+
public final class PyExpatModuleBuiltins extends PythonBuiltins {
64+
@Override
65+
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
66+
return PyExpatModuleBuiltinsFactory.getFactories();
67+
}
68+
69+
private static enum ContentModelConstant {
70+
XML_CQUANT_NONE(0),
71+
XML_CQUANT_OPT(1),
72+
XML_CQUANT_PLUS(3),
73+
XML_CQUANT_REP(2),
74+
XML_CTYPE_ANY(2),
75+
XML_CTYPE_CHOICE(5),
76+
XML_CTYPE_EMPTY(1),
77+
XML_CTYPE_MIXED(3),
78+
XML_CTYPE_NAME(4),
79+
XML_CTYPE_SEQ(6);
80+
81+
private final int number;
82+
83+
private ContentModelConstant(int number) {
84+
this.number = number;
85+
}
86+
}
87+
88+
private static enum ErrorConstant {
89+
XML_ERROR_NO_MEMORY("out of memory"),
90+
XML_ERROR_SYNTAX("syntax error"),
91+
XML_ERROR_NO_ELEMENTS("no element found"),
92+
XML_ERROR_INVALID_TOKEN("not well-formed (invalid token)"),
93+
XML_ERROR_UNCLOSED_TOKEN("unclosed token"),
94+
XML_ERROR_PARTIAL_CHAR("partial character"),
95+
XML_ERROR_TAG_MISMATCH("mismatched tag"),
96+
XML_ERROR_DUPLICATE_ATTRIBUTE("duplicate attribute"),
97+
XML_ERROR_JUNK_AFTER_DOC_ELEMENT("junk after document element"),
98+
XML_ERROR_PARAM_ENTITY_REF("illegal parameter entity reference"),
99+
XML_ERROR_UNDEFINED_ENTITY("undefined entity"),
100+
XML_ERROR_RECURSIVE_ENTITY_REF("recursive entity reference"),
101+
XML_ERROR_ASYNC_ENTITY("asynchronous entity"),
102+
XML_ERROR_BAD_CHAR_REF("reference to invalid character number"),
103+
XML_ERROR_BINARY_ENTITY_REF("reference to binary entity"),
104+
XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF("reference to external entity in attribute"),
105+
XML_ERROR_MISPLACED_XML_PI("XML or text declaration not at start of entity"),
106+
XML_ERROR_UNKNOWN_ENCODING("unknown encoding"),
107+
XML_ERROR_INCORRECT_ENCODING("encoding specified in XML declaration is incorrect"),
108+
XML_ERROR_UNCLOSED_CDATA_SECTION("unclosed CDATA section"),
109+
XML_ERROR_EXTERNAL_ENTITY_HANDLING("error in processing external entity reference"),
110+
XML_ERROR_NOT_STANDALONE("document is not standalone"),
111+
XML_ERROR_UNEXPECTED_STATE("unexpected parser state - please send a bug report"),
112+
XML_ERROR_ENTITY_DECLARED_IN_PE("entity declared in parameter entity"),
113+
XML_ERROR_FEATURE_REQUIRES_XML_DTD("requested feature requires XML_DTD support in Expat"),
114+
XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING("cannot change setting once parsing has begun"),
115+
XML_ERROR_UNBOUND_PREFIX("unbound prefix"),
116+
XML_ERROR_UNDECLARING_PREFIX("must not undeclare prefix"),
117+
XML_ERROR_INCOMPLETE_PE("incomplete markup in parameter entity"),
118+
XML_ERROR_XML_DECL("XML declaration not well-formed"),
119+
XML_ERROR_TEXT_DECL("text declaration not well-formed"),
120+
XML_ERROR_PUBLICID("illegal character(s) in public id"),
121+
XML_ERROR_SUSPENDED("parser suspended"),
122+
XML_ERROR_NOT_SUSPENDED("parser not suspended"),
123+
XML_ERROR_ABORTED("parsing aborted"),
124+
XML_ERROR_FINISHED("parsing finished"),
125+
XML_ERROR_SUSPEND_PE("cannot suspend in external parameter entity");
126+
127+
private final String message;
128+
129+
private ErrorConstant(String message) {
130+
this.message = message;
131+
}
132+
}
133+
134+
@Override
135+
public void initialize(Python3Core core) {
136+
super.initialize(core);
137+
PythonModule model = core.factory().createPythonModule(toTruffleStringUncached("pyexpat.model"));
138+
for (ContentModelConstant v : ContentModelConstant.values()) {
139+
model.setAttribute(toTruffleStringUncached(v.name()), v.number);
140+
}
141+
addBuiltinConstant("model", model);
142+
143+
PythonModule errors = core.factory().createPythonModule(toTruffleStringUncached("pyexpat.errors"));
144+
LinkedHashMap<String, Object> codes = new LinkedHashMap<>(ErrorConstant.values().length);
145+
LinkedHashMap<Object, Object> messages = new LinkedHashMap<>(ErrorConstant.values().length);
146+
for (ErrorConstant c : ErrorConstant.values()) {
147+
errors.setAttribute(toTruffleStringUncached(c.name()), toTruffleStringUncached(c.message));
148+
codes.put(c.message, c.ordinal() + 1);
149+
messages.put(c.ordinal() + 1, c.message);
150+
}
151+
errors.setAttribute(toTruffleStringUncached("messages"), core.factory().createDictFromMapGeneric(messages));
152+
errors.setAttribute(toTruffleStringUncached("codes"), core.factory().createDictFromMap(codes));
153+
addBuiltinConstant("errors", errors);
154+
}
155+
156+
@Builtin(name = "ParserCreate", parameterNames = {"encoding", "namespace_separator", "intern"}, doc = "Return a new XML parser object.")
157+
@GenerateNodeFactory
158+
abstract static class ParserCreateNode extends PythonTernaryBuiltinNode {
159+
@SuppressWarnings("unused")
160+
@Specialization
161+
@TruffleBoundary
162+
Object fail(Object encoding, Object namespace_separator, Object intern) {
163+
throw raise(PythonBuiltinClassType.NotImplementedError, toTruffleStringUncached("XML pyexpat parser is not implemented"));
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)