Use static analyzer to check pinning state (#49)

This PR introduces some preliminary work to use GCChecker to check pinning state. The clangsa tests pass. But the checker found issues with the current runtime code base. Future PRs will fix that.

Author: qinsoon

src/clangsa/GCChecker.cpp

@@ -5,41 +5,49 @@
 
 #ifdef LIBRARY_EXPORTS
 
+// Generated file, needs to be searched in include paths so that the builddir
+// retains priority
+#include <jl_internal_funcs.inc>
+#undef jl_setjmp
+#undef jl_longjmp
+#undef jl_egal
+#endif
+
+#include "julia_fasttls.h"
+#include "libsupport.h"
+#include <stdint.h>
+#include <string.h>
+
+#include "htable.h"
+#include "arraylist.h"
+#include "analyzer_annotations.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
 extern unsigned char mmtk_pin_object(void* obj);
+extern unsigned char mmtk_unpin_object(void* obj);
+#ifdef __clang_gcanalyzer__
+extern void PTRHASH_PIN(void* key) JL_NOTSAFEPOINT;
+extern void PTRHASH_UNPIN(void* key) JL_NOTSAFEPOINT;
+#else
 // FIXME: Pinning objects that get hashed in the ptrhash table
 // until we implement address space hashing.
 #ifdef MMTK_GC
 #define PTRHASH_PIN(key) mmtk_pin_object(key);
+#define PTRHASH_UNPIN(key) mmtk_unpin_object(key);
 #else
 #define PTRHASH_PIN(key)
+#define PTRHASH_UNPIN(key)
+#endif
 #endif
 
 #ifdef __cplusplus
 }
 #endif
 
-// Generated file, needs to be searched in include paths so that the builddir
-// retains priority
-#include <jl_internal_funcs.inc>
-#undef jl_setjmp
-#undef jl_longjmp
-#undef jl_egal
-#endif
-
-#include "julia_fasttls.h"
-#include "libsupport.h"
-#include <stdint.h>
-#include <string.h>
-
-#include "htable.h"
-#include "arraylist.h"
-#include "analyzer_annotations.h"
-
 #include <setjmp.h>
 #ifndef _OS_WINDOWS_
 #  define jl_jmp_buf sigjmp_buf
@@ -907,6 +915,24 @@ extern void _JL_GC_PUSHARGS(jl_value_t **, size_t) JL_NOTSAFEPOINT;
 
 extern void JL_GC_POP() JL_NOTSAFEPOINT;
 
+#ifdef MMTK_GC
+extern void JL_GC_PUSH1_NO_TPIN(void *) JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH2_NO_TPIN(void *, void *) JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH3_NO_TPIN(void *, void *, void *)  JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH4_NO_TPIN(void *, void *, void *, void *)  JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH5_NO_TPIN(void *, void *, void *, void *, void *)  JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH7_NO_TPIN(void *, void *, void *, void *, void *, void *, void *)  JL_NOTSAFEPOINT;
+extern void JL_GC_PUSH8_NO_TPIN(void *, void *, void *, void *, void *, void *, void *, void *)  JL_NOTSAFEPOINT;
+extern void _JL_GC_PUSHARGS_NO_TPIN(jl_value_t **, size_t) JL_NOTSAFEPOINT;
+// This is necessary, because otherwise the analyzer considers this undefined
+// behavior and terminates the exploration
+#define JL_GC_PUSHARGS_NO_TPIN(rts_var, n)     \
+  rts_var = (jl_value_t **)alloca(sizeof(void*) * (n)); \
+  memset(rts_var, 0, sizeof(void*) * (n)); \
+  _JL_GC_PUSHARGS_NO_TPIN(rts_var, (n));
+
+#endif
+
 #else
 
 #define JL_GC_PUSH1(arg1)                                                                               \

src/julia.h

@@ -9,10 +9,21 @@ extern "C" {
 
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
 extern unsigned char mmtk_pin_object(void* obj);
+extern unsigned char mmtk_unpin_object(void* obj);
+
+#ifdef __clang_gcanalyzer__
+extern void PTR_PIN(void* key) JL_NOTSAFEPOINT;
+extern void PTR_UNPIN(void* key) JL_NOTSAFEPOINT;
+#else
+
 #ifdef MMTK_GC
 #define PTR_PIN(key) mmtk_pin_object(key);
+#define PTR_UNPIN(key) mmtk_unpin_object(key);
 #else
 #define PTR_PIN(key)
+#define PTR_UNPIN(key)
+#endif
+
 #endif
 
 #ifdef __cplusplus

src/julia_internal.h

@@ -13,7 +13,10 @@
 #define JL_PROPAGATES_ROOT __attribute__((annotate("julia_propagates_root")))
 #define JL_NOTSAFEPOINT __attribute__((annotate("julia_not_safepoint")))
 #define JL_MAYBE_UNROOTED __attribute__((annotate("julia_maybe_unrooted")))
+#define JL_MAYBE_UNPINNED __attribute__((annotate("julia_maybe_unpinned")))
 #define JL_GLOBALLY_ROOTED __attribute__((annotate("julia_globally_rooted")))
+#define JL_GLOBALLY_PINNED __attribute__((annotate("julia_globally_pinned")))
+#define JL_GLOBALLY_TPINNED __attribute__((annotate("julia_globally_tpinned")))
 #define JL_ROOTING_ARGUMENT __attribute__((annotate("julia_rooting_argument")))
 #define JL_ROOTED_ARGUMENT __attribute__((annotate("julia_rooted_argument")))
 #define JL_GC_DISABLED __attribute__((annotate("julia_gc_disabled")))
@@ -34,7 +37,15 @@ extern "C" {
 #define JL_PROPAGATES_ROOT
 #define JL_NOTSAFEPOINT
 #define JL_MAYBE_UNROOTED
+#define JL_MAYBE_UNPINNED
+// The runtime may mark any object that is reachable from a global root as globally rooted.
+// So JL_GLOBALLY_ROOTED does not need to an actual root. Thus we don't know anything
+// about pining state.
 #define JL_GLOBALLY_ROOTED
+// A root may be pinned. But any object reachable from the root is not pinned.
+#define JL_GLOBALLY_PINNED
+// A root may be transitively pinned, and any object reachable from the root is transitively pinned.
+#define JL_GLOBALLY_TPINNED
 #define JL_ROOTING_ARGUMENT
 #define JL_ROOTED_ARGUMENT
 #define JL_GC_DISABLED

src/support/analyzer_annotations.h

@@ -1,6 +1,6 @@
 // This file is a part of Julia. License is MIT: https://julialang.org/license
 
-// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -Xclang -verify -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -x c++ %s
+// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -Xclang -verify -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CLANGSA_CXXFLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -x c++ %s
 
 #include "julia.h"
 #include <string>

test/clangsa/GCPushPop.cpp

@@ -1,7 +1,7 @@
 // This file is a part of Julia. License is MIT: https://julialang.org/license
 
-// RUN: clang-tidy %s --checks=-*,concurrency-implicit-atomics -load libImplicitAtomicsPlugin%shlibext -- -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -x c -std=c11 | FileCheck --check-prefixes=CHECK,CHECK-C %s
-// RUN: clang-tidy %s --checks=-*,concurrency-implicit-atomics -load libImplicitAtomicsPlugin%shlibext -- -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} ${CXXFLAGS} -x c++ -std=c++11 | FileCheck --check-prefixes=CHECK,CHECK-CXX %s
+// RUN: clang-tidy %s --checks=-*,concurrency-implicit-atomics -load libImplicitAtomicsPlugin%shlibext -- -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CLANGSA_CXXFLAGS} ${CPPFLAGS} ${CFLAGS} -x c -std=c11 | FileCheck --check-prefixes=CHECK,CHECK-C %s
+// RUN: clang-tidy %s --checks=-*,concurrency-implicit-atomics -load libImplicitAtomicsPlugin%shlibext -- -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CLANGSA_CXXFLAGS} ${CPPFLAGS} ${CFLAGS} ${CXXFLAGS} -x c++ -std=c++11 | FileCheck --check-prefixes=CHECK,CHECK-CXX %s
 
 #include "julia_atomics.h"
 

test/clangsa/ImplicitAtomicsTest.c

@@ -13,7 +13,7 @@ TESTS = $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/*.c) $(wildcard $(SRCDIR)/
 	PATH=$(build_bindir):$(build_depsbindir):$$PATH \
 	LD_LIBRARY_PATH="${build_libdir}:$$LD_LIBRARY_PATH" \
 	CLANGSA_FLAGS="${CLANGSA_FLAGS}" \
-	CLANGSACXX_FLAGS="${CLANGSACXX_FLAGS}" \
+	CLANGSA_CXXFLAGS="${CLANGSA_CXXFLAGS}" \
 	CPPFLAGS_FLAGS="${CPPFLAGS_FLAGS}" \
 	CFLAGS_FLAGS="${CFLAGS_FLAGS}" \
 	CXXFLAGS_FLAGS="${CXXFLAGS_FLAGS}" \

test/clangsa/Makefile

@@ -0,0 +1,56 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CLANGSA_CXXFLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -Xclang -verify -v -x c %s
+
+#include "julia.h"
+#include "julia_internal.h"
+
+extern void look_at_value(jl_value_t *v);
+
+void unpinned_argument() {
+    jl_svec_t *val = jl_svec1(NULL);  // expected-note{{Started tracking value here}}
+    JL_GC_PROMISE_ROOTED(val);        // expected-note{{Value was rooted here}}
+    look_at_value((jl_value_t*) val); // expected-warning{{Passing non-pinned value as argument to function that may GC}}
+                                      // expected-note@-1{{Passing non-pinned value as argument to function that may GC}}                                      
+}
+
+void pinned_argument() {
+    jl_svec_t *val = jl_svec1(NULL);
+    JL_GC_PROMISE_ROOTED(val);
+    PTR_PIN(val);
+    look_at_value((jl_value_t*) val);
+    PTR_UNPIN(val);
+}
+
+void missing_pin_before_safepoint() {
+    jl_svec_t *val = jl_svec1(NULL);    // expected-note{{Started tracking value here}}
+    JL_GC_PROMISE_ROOTED(val);          // expected-note{{Value was rooted here}}
+    jl_gc_safepoint();
+    look_at_value((jl_value_t*) val);   // expected-warning{{Argument value may have been moved}}
+                                        // expected-note@-1{{Argument value may have been moved}}
+}
+
+void proper_pin_before_safepoint() {
+    jl_svec_t *val = jl_svec1(NULL);
+    JL_GC_PROMISE_ROOTED(val);
+    PTR_PIN(val);
+    jl_gc_safepoint();
+    look_at_value((jl_value_t*) val);
+    PTR_UNPIN(val);
+}
+
+void push_tpin_value() {
+    jl_svec_t *val = jl_svec1(NULL);
+    JL_GC_PUSH1(&val);
+    jl_gc_safepoint();
+    look_at_value((jl_value_t*) val);
+    JL_GC_POP();
+}
+
+void push_no_tpin_value() {
+    jl_svec_t *val = jl_svec1(NULL);
+    JL_GC_PUSH1_NO_TPIN(&val);
+    jl_gc_safepoint();
+    look_at_value((jl_value_t*) val);
+    JL_GC_POP();
+}

test/clangsa/MissingPinning.c

@@ -1,12 +1,12 @@
 // This file is a part of Julia. License is MIT: https://julialang.org/license
 
-// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -Xclang -verify -x c %s
+// RUN: clang -D__clang_gcanalyzer__ --analyze -Xanalyzer -analyzer-output=text -Xclang -load -Xclang libGCCheckerPlugin%shlibext -I%julia_home/src -I%julia_home/src/support -I%julia_home/usr/include ${CLANGSA_FLAGS} ${CLANGSA_CXXFLAGS} ${CPPFLAGS} ${CFLAGS} -Xclang -analyzer-checker=core,julia.GCChecker --analyzer-no-default-checks -Xclang -verify -x c %s
 
 #include "julia.h"
 #include "julia_internal.h"
 
 extern void look_at_value(jl_value_t *v);
-extern void process_unrooted(jl_value_t *maybe_unrooted JL_MAYBE_UNROOTED);
+extern void process_unrooted(jl_value_t *maybe_unrooted JL_MAYBE_UNROOTED JL_MAYBE_UNPINNED);
 extern void jl_gc_safepoint();
 
 void unrooted_argument() {
@@ -167,7 +167,7 @@ int unrooted() {
                            // expected-note@-1{{Trying to access value which may have been GCed}}
 }
 
-extern jl_value_t *global_value JL_GLOBALLY_ROOTED;
+extern jl_value_t *global_value JL_GLOBALLY_ROOTED JL_GLOBALLY_PINNED;
 void globally_rooted() {
   jl_value_t *val = global_value;
   jl_gc_safepoint();
@@ -239,7 +239,7 @@ void pushargs_as_args()
   JL_GC_POP();
 }
 
-static jl_typemap_entry_t *this_call_cache[10] JL_GLOBALLY_ROOTED;
+static jl_typemap_entry_t *this_call_cache[10] JL_GLOBALLY_ROOTED JL_GLOBALLY_TPINNED;
 void global_array2() {
   jl_value_t *val = NULL;
   JL_GC_PUSH1(&val);
@@ -296,12 +296,12 @@ void tparam0(jl_value_t *atype) {
    look_at_value(jl_tparam0(atype));
 }
 
-extern jl_value_t *global_atype JL_GLOBALLY_ROOTED;
+extern jl_value_t *global_atype JL_GLOBALLY_ROOTED JL_GLOBALLY_TPINNED;
 void tparam0_global() {
    look_at_value(jl_tparam0(global_atype));
 }
 
-static jl_value_t *some_global JL_GLOBALLY_ROOTED;
+static jl_value_t *some_global JL_GLOBALLY_ROOTED JL_GLOBALLY_PINNED;
 void global_copy() {
     jl_value_t *local = NULL;
     jl_gc_safepoint();