[GR-37836] Update HPy

PullRequest: graalpython/2191

Author: qunaibit

docs/contributor/DEV_TASKS.md

@@ -25,3 +25,32 @@ our imported sources from CPython and PyPy:
 It prints a fairly long help. Note that you'll need to make sure you also pushed
 your `python-import` branch after doing the update, so that any conflicts you
 resolved don't have to be resolved again by the next person.
+
+### Updating hpy
+
+Follow these steps to update HPy.
+
+
+  1. Merge updated hpy sources. To do so, clone hpy somewhere next to
+     graalpython. Then run the following command on a new branch of graalpython:
+
+        mx python-update-hpy-import --pull /path/to/clone/of/hpy
+
+     Follow the instructions.
+  2. We need to fix compilation. We patch the hpy sources, and the merge may
+     have introduced new API or types, and for these we need to apply
+     patches. At the time of this writing, we redefine hpy types conditionally
+     on `#ifdef GRAALVM_PYTHON_LLVM` (grep for this to find some
+     examples). Also, we use macros to convert between the structured hpy types
+     and plain pointers for our native interface, see the uses of the `WRAP` and
+     `UNWRAP` macros.
+  3. Once compilation is working, we try to run the tests and go on fixing
+     them. If new API was added, `GraalHPyContext` needs to be adapted with the
+     new signatures and/or types. This may include:
+
+      - Updating the HPyContextMember enum
+      - Updating the HPyContextSignatureType enum
+      - Adding `GraalHPyContextFunction` implementations for the new APIs
+      - Updating the `createMembers` method to assign the appropriate
+        implementations to the context members
+

graalpython/com.oracle.graal.python.cext/hpy/hpy.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -56,7 +56,10 @@ static HPyContext *g_debug_ctx;
 typedef HPyDef* HPyDefPtr;
 
 POLYGLOT_DECLARE_TYPE(HPy)
+POLYGLOT_DECLARE_TYPE(HPyField)
+POLYGLOT_DECLARE_TYPE(HPyGlobal)
 POLYGLOT_DECLARE_TYPE(HPyContext)
+POLYGLOT_DECLARE_TYPE(HPyThreadState)
 POLYGLOT_DECLARE_TYPE(HPyDef)
 POLYGLOT_DECLARE_TYPE(HPyDef_Kind)
 POLYGLOT_DECLARE_TYPE(HPyDefPtr)
@@ -82,6 +85,7 @@ int graal_hpy_init(HPyContext *context, void *initObject) {
 	// register the native type of HPy
 	polyglot_invoke(initObject, "setHPyContextNativeType", polyglot_HPyContext_typeid());
 	polyglot_invoke(initObject, "setHPyNativeType", polyglot_HPy_typeid());
+	polyglot_invoke(initObject, "setHPyFieldNativeType", polyglot_HPyField_typeid());
 	polyglot_invoke(initObject, "setHPyArrayNativeType", polyglot_array_typeid(polyglot_HPy_typeid(), 0));
 
 	// register size of wchar_t
@@ -142,11 +146,11 @@ void* graal_hpy_from_HPyType_SpecParam_array(HPyType_SpecParam *ptr) {
 }
 
 void* graal_hpy_get_m_name(HPyModuleDef *moduleDef) {
-	return polyglot_from_string(moduleDef->m_name, SRC_CS);
+	return polyglot_from_string(moduleDef->name, SRC_CS);
 }
 
 void* graal_hpy_get_m_doc(HPyModuleDef *moduleDef) {
-	const char *m_doc = moduleDef->m_doc;
+	const char *m_doc = moduleDef->doc;
 	if (m_doc) {
 		return polyglot_from_string(m_doc, SRC_CS);
 	}
@@ -167,6 +171,22 @@ HPyFunc_Signature graal_hpy_meth_get_signature(HPyMeth *methodDef) {
 	return methodDef->signature;
 }
 
+void* graal_hpy_get_field_i(HPyField *hf) {
+	return hf->_i;
+}
+
+void graal_hpy_set_field_i(HPyField *hf, void* i) {
+	hf->_i = i;
+}
+
+void* graal_hpy_get_global_i(HPyGlobal *hg) {
+	return hg->_i;
+}
+
+void graal_hpy_set_global_i(HPyGlobal *hg, void* i) {
+	hg->_i = i;
+}
+
 /* getters for HPySlot */
 
 HPySlot_Slot graal_hpy_slot_get_slot(HPySlot *slot) {
@@ -297,6 +317,17 @@ void* graal_hpy_from_string(const char *ptr) {
 	return polyglot_from_string(ptr, SRC_CS);
 }
 
+int graal_hpy_get_errno() {
+	return errno;
+}
+
+void* graal_hpy_get_strerror(int i) {
+	if (i != 0) {
+		return polyglot_from_string(strerror(i), SRC_CS);
+	}
+	return polyglot_from_string("Error", SRC_CS);
+}
+
 uint64_t graal_hpy_strlen(const char *ptr) {
 	return strlen(ptr);
 }
@@ -624,12 +655,17 @@ HPy_buffer *graal_hpy_allocate_buffer() {
 #define _h2py(_x) NULL
 
 typedef HPy* _HPyPtr;
+typedef HPyField* _HPyFieldPtr;
 typedef HPy _HPyConst;
+typedef HPyGlobal* _HPyGlobalPtr;
 
 #define HPy void*
 #define HPyListBuilder void*
 #define HPyTupleBuilder void*
 #define HPyTracker void*
+#define HPyField void*
+#define HPyThreadState void*
+#define HPyGlobal void*
 
 #define SELECT_CTX(__ctx__) ((__ctx__) == g_universal_ctx ? g_universal_ctx : g_debug_ctx)
 
@@ -728,6 +764,14 @@ HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME(Long_AsSsize_t)(HPyContext *ctx, HPy h
 	return (HPy_ssize_t) UPCALL_I64(ctx_Long_AsSsize_t, ctx, h);
 }
 
+HPyAPI_STORAGE void* _HPy_IMPL_NAME(Long_AsVoidPtr)(HPyContext *ctx, HPy h) {
+     return (void *) UPCALL_I64(ctx_Long_AsVoidPtr, ctx, h); 
+}
+
+HPyAPI_STORAGE double _HPy_IMPL_NAME(Long_AsDouble)(HPyContext *ctx, HPy h) {
+     return UPCALL_DOUBLE(ctx_Long_AsDouble, ctx, h); 
+}
+
 HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Float_FromDouble)(HPyContext *ctx, double v)
 {
 	return UPCALL_HPY(ctx_Float_FromDouble, ctx, v);
@@ -943,11 +987,23 @@ HPyAPI_STORAGE void _HPy_IMPL_NAME(Err_SetObject)(HPyContext *ctx, HPy h_type, H
 	UPCALL_VOID(ctx_Err_SetObject, ctx, h_type, h_value);
 }
 
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Err_SetFromErrnoWithFilename)(HPyContext *ctx, HPy h_type, const char *filename_fsencoded) {
+     return UPCALL_HPY(ctx_Err_SetFromErrnoWithFilename, ctx, h_type, filename_fsencoded); 
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Err_SetFromErrnoWithFilenameObjects)(HPyContext *ctx, HPy h_type, HPy filename1, HPy filename2) {
+     return UPCALL_HPY(ctx_Err_SetFromErrnoWithFilenameObjects, ctx, h_type, filename1, filename2); 
+}
+
 HPyAPI_STORAGE int _HPy_IMPL_NAME(Err_Occurred)(HPyContext *ctx)
 {
     return (int) polyglot_as_i32(UPCALL_HPY0(ctx_Err_Occurred, ctx));
 }
 
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Err_ExceptionMatches)(HPyContext *ctx, HPy exc) {
+     return (int) UPCALL_I32(ctx_Err_ExceptionMatches, ctx, exc); 
+}
+
 HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Err_NoMemory)(HPyContext *ctx)
 {
 	return UPCALL_HPY0(ctx_Err_NoMemory, ctx);
@@ -966,6 +1022,18 @@ HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Err_NewExceptionWithDoc)(HPyContext *ctx, cons
     return UPCALL_HPY(ctx_Err_NewExceptionWithDoc, ctx, name, doc, base, dict);
 }
 
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Err_WarnEx)(HPyContext *ctx, HPy category, const char *message, HPy_ssize_t stack_level) {
+     return (int) UPCALL_I32(ctx_Err_WarnEx, ctx, category, message, stack_level); 
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(Err_WriteUnraisable)(HPyContext *ctx, HPy obj) {
+     UPCALL_VOID(ctx_Err_WriteUnraisable, ctx, obj); 
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(FatalError)(HPyContext *ctx, const char *msg) {
+     UPCALL_VOID(ctx_FatalError, ctx, msg); 
+}
+
 HPyAPI_STORAGE int _HPy_IMPL_NAME_NOPREFIX(IsTrue)(HPyContext *ctx, HPy h)
 {
 	return UPCALL_HPY(ctx_IsTrue, ctx, h);
@@ -1026,6 +1094,10 @@ HPyAPI_STORAGE HPy _HPy_IMPL_NAME_NOPREFIX(GetItem_s)(HPyContext *ctx, HPy obj,
 	return UPCALL_HPY(ctx_GetItem_s, ctx, obj, key);
 }
 
+HPyAPI_STORAGE int _HPy_IMPL_NAME_NOPREFIX(Contains)(HPyContext *ctx, HPy container, HPy key) {
+     return (int) UPCALL_I32(ctx_Contains, ctx, container, key); 
+}
+
 HPyAPI_STORAGE int _HPy_IMPL_NAME_NOPREFIX(SetItem)(HPyContext *ctx, HPy obj, HPy key, HPy value)
 {
 	return UPCALL_HPY(ctx_SetItem, ctx, obj, key, value);
@@ -1151,6 +1223,14 @@ HPyAPI_STORAGE int _HPy_IMPL_NAME(Unicode_Check)(HPyContext *ctx, HPy h)
 	return (int) UPCALL_I32(ctx_Unicode_Check, ctx, h);
 }
 
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_AsASCIIString)(HPyContext *ctx, HPy h) {
+     return UPCALL_HPY(ctx_Unicode_AsASCIIString, ctx, h); 
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_AsLatin1String)(HPyContext *ctx, HPy h) {
+     return UPCALL_HPY(ctx_Unicode_AsLatin1String, ctx, h);
+}
+
 HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_AsUTF8String)(HPyContext *ctx, HPy h)
 {
 	return UPCALL_HPY(ctx_Unicode_AsUTF8String, ctx, h);
@@ -1170,6 +1250,26 @@ HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_DecodeFSDefault)(HPyContext *ctx, cons
 	return UPCALL_HPY(ctx_Unicode_DecodeFSDefault, ctx, v);
 }
 
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_DecodeFSDefaultAndSize)(HPyContext *ctx, const char *v, HPy_ssize_t size) {
+     return UPCALL_HPY(ctx_Unicode_DecodeFSDefaultAndSize, ctx, v, size); 
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_EncodeFSDefault)(HPyContext *ctx, HPy h) {
+     return UPCALL_HPY(ctx_Unicode_EncodeFSDefault, ctx, h); 
+}
+
+HPyAPI_STORAGE uint32_t _HPy_IMPL_NAME(Unicode_ReadChar)(HPyContext *ctx, HPy h, HPy_ssize_t index) {
+     return (uint32_t) UPCALL_I32(ctx_Unicode_ReadChar, ctx, h, index);
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_DecodeLatin1)(HPyContext *ctx, const char *s, HPy_ssize_t size, const char *errors) {
+     return UPCALL_HPY(ctx_Unicode_DecodeLatin1, ctx, s, size, errors ); 
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_DecodeASCII)(HPyContext *ctx, const char *s, HPy_ssize_t size, const char *errors) {
+     return UPCALL_HPY(ctx_Unicode_DecodeASCII, ctx, s, size, errors ); 
+}
+
 HPyAPI_STORAGE int _HPy_IMPL_NAME(List_Check)(HPyContext *ctx, HPy h)
 {
 	return (int) UPCALL_I32(ctx_List_Check, ctx, h);
@@ -1271,6 +1371,30 @@ HPyAPI_STORAGE void _HPy_IMPL_NAME(Tracker_Close)(HPyContext *ctx, HPyTracker ht
 	UPCALL_VOID(ctx_Tracker_Close, ctx, ht);
 }
 
+HPyAPI_STORAGE void _HPy_IMPL_NAME(Field_Store)(HPyContext *ctx, HPy target_object, _HPyFieldPtr target_field, HPy h) {
+	UPCALL_VOID(ctx_Field_Store, ctx, target_object, target_field, h);
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Field_Load)(HPyContext *ctx, HPy source_object, HPyField source_field) {
+	return UPCALL_HPY(ctx_Field_Load, ctx, source_object, source_field);
+}
+
+HPyAPI_STORAGE HPyThreadState _HPy_IMPL_NAME(LeavePythonExecution)(HPyContext *ctx) {
+	return UPCALL_HPY0(ctx_LeavePythonExecution, ctx);
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(ReenterPythonExecution)(HPyContext *ctx, HPyThreadState state) {
+	UPCALL_VOID(ctx_ReenterPythonExecution, ctx, state);
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(Global_Store)(HPyContext *ctx, _HPyGlobalPtr global, HPy h) {
+	UPCALL_VOID(ctx_Global_Store, ctx, global, h);
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Global_Load)(HPyContext *ctx, HPyGlobal global) {
+	return UPCALL_HPY(ctx_Global_Load, ctx, global);
+}
+
 HPyAPI_STORAGE void _HPy_IMPL_NAME(Dump)(HPyContext *ctx, HPy h) {
 	UPCALL_VOID(ctx_Dump, ctx, h);
 }
@@ -1279,6 +1403,9 @@ HPyAPI_STORAGE void _HPy_IMPL_NAME(Dump)(HPyContext *ctx, HPy h) {
 #undef HPyListBuilder
 #undef HPyTupleBuilder
 #undef HPyTracker
+#undef HPyField
+#undef HPyThreadState
+#undef HPyGlobal
 
 #undef _HPy_IMPL_NAME_NOPREFIX
 #undef _HPy_IMPL_NAME
@@ -1292,14 +1419,14 @@ HPyContext *graal_hpy_context_to_native(HPyContext *managed_context, HPyContext
 	HPyContext *native_context = graal_native_context_get_hpy_context(full_native_context);
 
 #define COPY(__member) native_context->__member = managed_context->__member
-	COPY(name);
-	COPY(ctx_version);
-	COPY(h_None);
-	COPY(h_True);
-	COPY(h_False);
-	COPY(h_NotImplemented);
-	COPY(h_Ellipsis);
-	COPY(h_BaseException);
+    COPY(name);
+    COPY(ctx_version);
+    COPY(h_None);
+    COPY(h_True);
+    COPY(h_False);
+    COPY(h_NotImplemented);
+    COPY(h_Ellipsis);
+    COPY(h_BaseException);
     COPY(h_Exception);
     COPY(h_StopAsyncIteration);
     COPY(h_StopIteration);
@@ -1365,7 +1492,9 @@ HPyContext *graal_hpy_context_to_native(HPyContext *managed_context, HPyContext
     COPY(h_ResourceWarning);
     COPY(h_BaseObjectType);
     COPY(h_TypeType);
+    COPY(h_BoolType);
     COPY(h_LongType);
+    COPY(h_FloatType);
     COPY(h_UnicodeType);
     COPY(h_TupleType);
     COPY(h_ListType);
@@ -1449,6 +1578,7 @@ HPyContext *graal_hpy_context_to_native(HPyContext *managed_context, HPyContext
     HPY_CTX_UPCALL(ctx_GetItem);
     HPY_CTX_UPCALL(ctx_GetItem_i);
     HPY_CTX_UPCALL(ctx_GetItem_s);
+    HPY_CTX_UPCALL(ctx_Contains);
     HPY_CTX_UPCALL(ctx_SetItem);
     HPY_CTX_UPCALL(ctx_SetItem_i);
     HPY_CTX_UPCALL(ctx_SetItem_s);
@@ -1500,7 +1630,29 @@ HPyContext *graal_hpy_context_to_native(HPyContext *managed_context, HPyContext
     HPY_CTX_UPCALL(ctx_Tracker_Add);
     HPY_CTX_UPCALL(ctx_Tracker_ForgetAll);
     HPY_CTX_UPCALL(ctx_Tracker_Close);
+    HPY_CTX_UPCALL(ctx_Field_Store);
+    HPY_CTX_UPCALL(ctx_Field_Load);
+    HPY_CTX_UPCALL(ctx_LeavePythonExecution);
+    HPY_CTX_UPCALL(ctx_ReenterPythonExecution);
+    HPY_CTX_UPCALL(ctx_Global_Store);
+    HPY_CTX_UPCALL(ctx_Global_Load);
     HPY_CTX_UPCALL(ctx_Dump);
+    HPY_CTX_UPCALL(ctx_Long_AsVoidPtr);
+    HPY_CTX_UPCALL(ctx_Long_AsDouble);
+    HPY_CTX_UPCALL(ctx_Err_SetFromErrnoWithFilename);
+    HPY_CTX_UPCALL(ctx_Err_SetFromErrnoWithFilenameObjects);
+    HPY_CTX_UPCALL(ctx_Err_ExceptionMatches);
+    HPY_CTX_UPCALL(ctx_Err_WarnEx);
+    HPY_CTX_UPCALL(ctx_Err_WriteUnraisable);
+    HPY_CTX_UPCALL(ctx_FatalError);
+    HPY_CTX_UPCALL(ctx_Unicode_AsASCIIString);
+    HPY_CTX_UPCALL(ctx_Unicode_AsLatin1String);
+    HPY_CTX_UPCALL(ctx_Unicode_DecodeFSDefaultAndSize);
+    HPY_CTX_UPCALL(ctx_Unicode_EncodeFSDefault);
+    HPY_CTX_UPCALL(ctx_Unicode_ReadChar);
+    HPY_CTX_UPCALL(ctx_Unicode_DecodeLatin1);
+    HPY_CTX_UPCALL(ctx_Unicode_DecodeASCII);
+
 #undef HPY_CTX_UPCALL
 
 	return native_context;