[GR-15770] [GR-15772] [GR-16311] [GR-15769] [GR-16445] [GR-14742] Use toolchain to build C API and extensions.

PullRequest: graalpython/553

Author: fangerer

.gitignore

@@ -34,6 +34,7 @@ language
 /*.py
 *.o
 *.so
+*.dylib
 *.rej
 .checkstyle
 /graalpython/com.oracle.graal.python.cext/compile_flags.txt
@@ -57,6 +58,7 @@ Python3.g4.stamp
 *.orig
 /graalpython/com.oracle.graal.python.test/src/tests/cpyext/Py*.c
 /graalpython/com.oracle.graal.python.test/src/tests/cpyext/Test*.c
+/graalpython/com.oracle.graal.python.test/src/tests/cpyext/*.sha256
 /*.diff
 ## generated from: pyhocon -i ci.hocon -f json -o ci.json
 /ci.json

README.md

@@ -32,6 +32,41 @@ source <dir-to-venv>/bin/activate
 
 In the venv multiple executables are available, like `python`, `python3` and `graalpython`. 
 
+### Using modules with C extensions
+
+This Python implementation is able to load and run modules with C extensions.
+Supporting C extensions is one of the most difficult features for any Python 
+implementation since it requires to be compatible to CPython's C API.
+
+However, GraalVM's Python implementation is capable of executing C extensions and
+there is also no optimization boundary.
+
+In order to be able to run C extensions, a user must first build the C API
+runtime library. It is recommended to build the C API in any case because
+it will only be used if necessary. 
+The recommended way to do so is to create a venv (see [Create a virtual environment](#create-a-virtual-environment))
+and run everything within the venv.
+
+If you don't want to create and use a venv, the C API can be built using
+following command:
+```
+mx python -m build_capi
+```
+
+You can test if everything was built correctly by, for example, using 
+a memoryvew object:
+```
+(your-venv) graalpython -c "print(repr(memoryview(b'')))"
+```
+
+in the venv or
+
+```
+mx python -c "print(repr(memoryview(b'')))"
+```
+
+without a venv.
+
 ### Installing packages
 
 At the moment not enough of the standard library is implemented to run the

graalpython/com.oracle.graal.python.cext/Makefile

@@ -36,68 +36,26 @@
 # SOFTWARE.
 
 QUIETLY$(MX_VERBOSE) = @
-LIBDIR=../lib-graalpython
-TARGET_LIB=${LIBDIR}/capi.bc
-
+.PHONY: default clean
 
-SOURCE_FILES=$(wildcard src/*.c)
-OBJ_FILES=${SOURCE_FILES:%.c=%.o}
 INCLUDE_FILES=$(wildcard include/*.h)
-
-MODULE_SOURCES=$(wildcard modules/*.c)
-MODULE_OBJ_FILES=$(MODULE_SOURCES:%.c=%.o)
-MODULE_TARGETS=$(MODULE_SOURCES:modules/%.c=${LIBDIR}/modules/%.bc)
+LEGACY_INCLUDES=$(wildcard $(TRUFFLE_H_INC)/*.h)
 
 # These are just needed to allow compilation from source, because include needs
 # to be next to lib-graalpython
-HEADER_TARGETS=$(INCLUDE_FILES:include/%.h=../include/%.h)
-
-.PHONY: default clean
-default: ${TARGET_LIB} ${MODULE_TARGETS} ${HEADER_TARGETS} compile_flags.txt
-
-
-CFLAGS=${LLVM_TARGET_FLAGS} -O1 -ggdb -emit-llvm
-OPT_FLAGS=-mem2reg -globalopt -simplifycfg -constprop -always-inline -instcombine -dse -loop-simplify -reassociate -licm -gvn
-WARNINGS=-Wno-int-to-pointer-cast -Wno-int-conversion -Wno-incompatible-pointer-types-discards-qualifiers -Wno-pointer-type-mismatch
-INCLUDES=-I${POLYGLOT_INC} -Iinclude
-
+HEADER_TARGETS=$(INCLUDE_FILES:include/%.h=../include/%.h) 
+LEGACY_TARGETS=$(LEGACY_INCLUDES:$(TRUFFLE_H_INC)/%.h=../include/%.h)
 
-rebuild:
-	$(MAKE) clean
-	$(MAKE) default
+default: ${HEADER_TARGETS} ${LEGACY_TARGETS}
 
-${LIBDIR}/modules/%.bc: modules/%.o
-	$(QUIETLY) mkdir -p ${LIBDIR}/modules
-	$(QUIETLY) llvm-link -o $@ $+
-
-${TARGET_LIB}: ${OBJ_FILES}
-	$(QUIETLY) llvm-link -o $@ $+
-
-src/%.o: src/%.c Makefile src/capi.h ${INCLUDE_FILES}
-	$(QUIETLY) clang ${CFLAGS} ${WARNINGS} ${INCLUDES} -o $@ -c $<
-	$(QUIETLY) opt -o $@ $@ ${OPT_FLAGS}
-
-modules/%.o: modules/%.c Makefile src/capi.h ${INCLUDE_FILES}
-	$(QUIETLY) clang ${CFLAGS} ${WARNINGS} ${INCLUDES} -o $@ -c $<
-	$(QUIETLY) opt -o $@ $@ ${OPT_FLAGS}
-
-../include/%.h: ${INCLUDE_FILES}
+$(HEADER_TARGETS): ${INCLUDE_FILES}
 	$(QUIETLY) mkdir -p ../include/
 	$(QUIETLY) cp $(@:../include/%.h=include/%.h) $@
 
+$(LEGACY_TARGETS): ${LEGACY_INCLUDES}
+	$(QUIETLY) mkdir -p ../include/
+	$(QUIETLY) cp $(@:../include/%.h=$(TRUFFLE_H_INC)/%.h) $@
+
 clean:
-	$(QUIETLY) rm -f ${TARGET_LIB}
-	$(QUIETLY) rm -f ${OBJ_FILES}
-	$(QUIETLY) rm -f ${MODULE_TARGETS}
-	$(QUIETLY) rm -f ${MODULE_OBJ_FILES}
 	$(QUIETLY) rm -f ${HEADER_TARGETS}
-	$(QUIETLY) rm -f compile_flags.txt
-ifeq ($(wildcard ../com.oracle.graal.python.test/src/tests/cpyext/*.bc),)
-	rm -f $(wildcard ../com.oracle.graal.python.test/src/tests/cpyext/*.bc)
-endif
-
-# compile_flags.txt is useful with clangd as language server
-compile_flags.txt: Makefile
-	$(QUIETLY) $(file > $@,-O1)
-	$(QUIETLY) $(foreach var,$(WARNINGS),$(file >> $@,$(var)))
-	$(QUIETLY) $(foreach var,$(INCLUDES),$(file >> $@,$(var)))
+	$(QUIETLY) rm -f $(LEGACY_TARGETS)