[GR-42657] Avoid unresolved symbols in sqlite3 and add test.

PullRequest: graalpython/2879

Author: lukasstadler

graalpython/com.oracle.graal.python.cext/modules/_sqlite/sqlite/sqlite3.c

@@ -174096,7 +174096,10 @@ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION;
 /* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns
 ** a pointer to the to the sqlite3_version[] string constant.
 */
-SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; }
+/*
+* GraalPy change: use SQLITE_VERSION instead of sqlite3_version
+*/
+SQLITE_API const char *sqlite3_libversion(void){ return SQLITE_VERSION; }
 
 /* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a
 ** pointer to a string constant whose value is the same as the

graalpython/com.oracle.graal.python.cext/modules/_sqlite/sqlite/sqlite3.h

@@ -182,7 +182,10 @@ extern "C" {
 **
 ** See also: [sqlite_version()] and [sqlite_source_id()].
 */
-SQLITE_API SQLITE_EXTERN const char sqlite3_version[];
+/*
+* GraalPy change: avoid unresolved symbol "sqlite3_version"
+*/
+// SQLITE_API SQLITE_EXTERN const char sqlite3_version[];
 SQLITE_API const char *sqlite3_libversion(void);
 SQLITE_API const char *sqlite3_sourceid(void);
 SQLITE_API int sqlite3_libversion_number(void);

graalpython/com.oracle.graal.python.cext/src/_warnings.c

@@ -176,4 +176,17 @@ int PyErr_WarnExplicitFormat(PyObject *category,
     return ret;
 }
 
+void _PyErr_ChainExceptions(PyObject* a, PyObject* b, PyObject* c) {
+	// empty dummy implementation
+}
+
+PyObject* _PyErr_FormatFromCause(PyObject *exception, const char *format, ...) {
+	// dummy implementation that ignores the cause
+    va_list args;
+    va_start(args, format);
+    PyObject* formatted_msg = PyUnicode_FromFormatV(format, args);
+    va_end(args);
+    Graal_PyTruffleErr_CreateAndSetException(exception, formatted_msg);
+    return NULL;
+}
 

graalpython/com.oracle.graal.python.jni/src/capi_forwards.h

@@ -4995,15 +4995,9 @@ void (*__target___PyErr_BadInternalCall)(const char*, int) = NULL;
 PyAPI_FUNC(void) _PyErr_BadInternalCall(const char* a, int b) {
     __target___PyErr_BadInternalCall(a, b);
 }
-PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject* a, PyObject* b, PyObject* c) {
-    unimplemented("_PyErr_ChainExceptions"); exit(-1);
-}
 PyAPI_FUNC(int) _PyErr_CheckSignals() {
     unimplemented("_PyErr_CheckSignals"); exit(-1);
 }
-PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause(PyObject* a, const char* b, ...) {
-    unimplemented("_PyErr_FormatFromCause"); exit(-1);
-}
 PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState* a, PyObject** b, PyObject** c, PyObject** d) {
     unimplemented("_PyErr_GetExcInfo"); exit(-1);
 }

graalpython/com.oracle.graal.python.test/src/tests/test_sqlite3.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+
+try:
+    import __graalpython__
+    managed_launcher = __graalpython__.is_managed_launcher()
+except ImportError:
+    managed_launcher = False
+
+def test_basic_functionality():
+    """
+    This is a basic test to ensure that the module can be imported.
+    The main sqlite3 test suite will be silently skipped if the 
+    "_sqlite3" module is not available.
+    """
+    if not managed_launcher and os.name != 'nt':
+        import sqlite3
+        import _sqlite3
+        conn = sqlite3.connect(':memory:')
+        rows = conn.execute("select sqlite_version()")
+        assert len(next(rows)[0]) >= 5
+        conn.close()
+

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiCodeGen.java

@@ -52,6 +52,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -62,6 +63,7 @@
 import java.util.function.IntFunction;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltinRegistry;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiBuiltin;
@@ -480,13 +482,60 @@ private static boolean generateBuiltinRegistry(List<CApiBuiltinDesc> javaBuiltin
         return writeGenerated(Path.of("com.oracle.graal.python", "src", "com", "oracle", "graal", "python", "builtins", "modules", "cext", "PythonCextBuiltinRegistry.java"), lines);
     }
 
+    private static void checkUnimplementedAPI(Path path, List<CApiBuiltinDesc> additionalBuiltins) {
+        List<String> lines;
+        try {
+            lines = Files.readAllLines(path);
+        } catch (IOException e) {
+            System.out.println("    Error while reading " + path + ": " + e.getMessage());
+            return;
+        }
+        boolean msg = false;
+        for (CApiBuiltinDesc builtin : additionalBuiltins) {
+            if (builtin.call == NotImplemented) {
+                for (int i = 0; i < lines.size(); i++) {
+                    String line = lines.get(i);
+                    int offset = line.indexOf(builtin.name);
+                    // avoid recognizing, e.g. "_PyObject_CallMethodId" in
+                    // "_PyObject_CallMethodIdNoArgs"
+                    if (offset > 0 && Character.isUnicodeIdentifierPart(line.charAt(offset - 1))) {
+                        continue;
+                    }
+                    if (offset + builtin.name.length() < line.length() && Character.isUnicodeIdentifierPart(line.charAt(offset + builtin.name.length()))) {
+                        continue;
+                    }
+                    if (line.contains(builtin.name)) {
+                        if (!msg) {
+                            msg = true;
+                            System.out.println("Checking " + path);
+                        }
+                        System.out.println("    " + builtin.name + " used in " + path + " line " + (i + 1) + ": " + line);
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * Entry point for the "mx python-capi-forwards" command.
      */
     public static void main(String[] args) throws IOException {
         List<CApiBuiltinDesc> javaBuiltins = CApiFunction.getJavaBuiltinDefinitions();
         List<CApiBuiltinDesc> additionalBuiltins = CApiFunction.getOtherBuiltinDefinitions();
 
+        /*
+         * Calling with arguments "check <path>" will recursively check all files in the path for
+         * unimplemented C API functions.
+         */
+        if (args.length == 2 && "check".equals(args[0])) {
+            System.out.println("Checking usages of unimplemented API:");
+            String path = args[1];
+            try (Stream<Path> stream = Files.walk(Paths.get(path))) {
+                stream.filter(Files::isRegularFile).forEach(p -> checkUnimplementedAPI(p, additionalBuiltins));
+            }
+            return;
+        }
+
         List<CApiBuiltinDesc> allBuiltins = new ArrayList<>();
         allBuiltins.addAll(additionalBuiltins);
         for (var entry : javaBuiltins) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java

@@ -373,6 +373,8 @@ public final class CApiFunction {
     @CApiBuiltin(name = "PyByteArray_Size", ret = Py_ssize_t, args = {PyObject}, call = CImpl)
     @CApiBuiltin(name = "PyGILState_Ensure", ret = PY_GIL_STATE_STATE, args = {}, acquiresGIL = false, call = CImpl)
     @CApiBuiltin(name = "PyGILState_Release", ret = Void, args = {PY_GIL_STATE_STATE}, acquiresGIL = false, call = CImpl)
+    @CApiBuiltin(name = "_PyErr_ChainExceptions", ret = Void, args = {PyObject, PyObject, PyObject}, call = CImpl)
+    @CApiBuiltin(name = "_PyErr_FormatFromCause", ret = PyObject, args = {PyObject, ConstCharPtrAsTruffleString, VARARGS}, call = CImpl)
 
     /*
      * Functions that are implemented in C code that needs to run on Sulong:
@@ -782,9 +784,7 @@ public final class CApiFunction {
     @CApiBuiltin(name = "_PyDict_SizeOf", ret = Py_ssize_t, args = {PYDICTOBJECT_PTR}, call = NotImplemented)
     @CApiBuiltin(name = "_PyDictView_Intersect", ret = PyObject, args = {PyObject, PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "_PyDictView_New", ret = PyObject, args = {PyObject, PyTypeObject}, call = NotImplemented)
-    @CApiBuiltin(name = "_PyErr_ChainExceptions", ret = Void, args = {PyObject, PyObject, PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "_PyErr_CheckSignals", ret = Int, args = {}, call = NotImplemented)
-    @CApiBuiltin(name = "_PyErr_FormatFromCause", ret = PyObject, args = {PyObject, ConstCharPtrAsTruffleString, VARARGS}, call = NotImplemented)
     @CApiBuiltin(name = "_PyErr_GetExcInfo", ret = Void, args = {PyThreadState, PyObjectPtr, PyObjectPtr, PyObjectPtr}, call = NotImplemented)
     @CApiBuiltin(name = "_PyErr_GetTopmostException", ret = _PYERR_STACKITEM_PTR, args = {PyThreadState}, call = NotImplemented)
     @CApiBuiltin(name = "_PyErr_ProgramDecodedTextObject", ret = PyObject, args = {PyObject, Int, ConstCharPtr}, call = NotImplemented)