- Update to QATzip v0.2.5 (#1)

- Fix can't find some symbols when use ldd

Author: xieqi

hadoop_qat_wrapper/build/native/Makefile

@@ -37,7 +37,7 @@ INC = -I $(JAVA_HOME)/include \
       -I .
 
 libqatcodec.so: QatCompressor.o QatDecompressor.o util.o
-	gcc -fpic -shared -o ../../target/libqatcodec.so QatCompressor.o \
+	gcc -fpic -shared -o ../../target/libqatcodec.so -ldl QatCompressor.o \
 	  QatDecompressor.o util.o
 
 QatCompressor.o: ../../src/main/native/QatCompressor.c javah

hadoop_qat_wrapper/src/main/native/QatCompressor.c

@@ -42,23 +42,23 @@ static jfieldID QatCompressor_uncompressedDirectBufLen;
 static jfieldID QatCompressor_compressedDirectBuf;
 static jfieldID QatCompressor_directBufferSize;
 
-__thread QzSession  g_qzCompressSession = {
+__thread QzSession_T  g_qzCompressSession = {
     .internal = NULL,
 };
 
 #ifdef UNIX
-unsigned char* (*dlsym_qzMalloc)(int, int, int); 
-static int (*dlsym_qzCompress)(QzSession *sess, const unsigned char* src,
+unsigned char* (*dlsym_qzMalloc)(int, int, int);
+static int (*dlsym_qzCompress)(QzSession_T *sess, const unsigned char* src,
     unsigned int* src_len, unsigned char* dest, unsigned int* dest_len,
     unsigned int last);
+int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults);
+int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults);
 #endif
 
-//QzSession sess;
-QzSessionParams params;
 JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_qat_QatCompressor_initIDs
 (JNIEnv *env, jclass clazz, jint level){
+  QzSessionParams_T params;
 #ifdef UNIX
-
   // Load libqatzip.so
   void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL);
   if (!libqatzip) {
@@ -74,6 +74,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_qat_QatCompressor_init
   dlerror();                                 // Clear any existing error
   LOAD_DYNAMIC_SYMBOL(dlsym_qzCompress, env, libqatzip, "qzCompress");
   LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc");
+  LOAD_DYNAMIC_SYMBOL(dlsym_qzGetDefaults, env, libqatzip, "qzGetDefaults");
+  LOAD_DYNAMIC_SYMBOL(dlsym_qzSetDefaults, env, libqatzip, "qzSetDefaults");
 #endif
 
   QatCompressor_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
@@ -88,10 +90,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_qat_QatCompressor_init
                                                          "Ljava/nio/Buffer;");
   QatCompressor_directBufferSize = (*env)->GetFieldID(env, clazz,
                                                        "directBufferSize", "I");
-  qzGetDefaults(&params);
-  params.compLvl = level;
+  dlsym_qzGetDefaults(&params);
+  params.comp_lvl = level;
   fprintf(stderr, "compression level is %d, tid is %d\n", level, syscall(__NR_gettid));
-  qzSetDefaults(&params);
+  dlsym_qzSetDefaults(&params);
 }
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_qat_QatCompressor_compressBytesDirect
@@ -132,7 +134,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_qat_QatCompressor_comp
   ret = dlsym_qzCompress(&g_qzCompressSession, uncompressed_bytes, &src_len,
         compressed_bytes, &buf_len, 1);
   if (ret != QZ_OK){
-    THROW(env, "java/lang/InternalError", "Could not compress data. Buffer length is too small.");
+    THROW(env, "java/lang/InternalError", "Could not compress data, return " + ret);
     return 0;
   }
   if (buf_len > JINT_MAX) {
@@ -170,10 +172,10 @@ Java_org_apache_hadoop_io_compress_qat_QatCompressor_getLibraryName(JNIEnv *env,
 #endif
 }
 
-JNIEXPORT jobject JNICALL 
+JNIEXPORT jobject JNICALL
 Java_org_apache_hadoop_io_compress_qat_QatCompressor_nativeAllocateBB(JNIEnv *env,
  jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){
-//void *buf = dlsym_qat_malloc(capacity);  
+//void *buf = dlsym_qat_malloc(capacity);
 //printf("compressor: DBB address is 0x%lx\n",(unsigned long)buf);
 //fflush(stdout);
 //  return (*env)->NewDirectByteBuffer(env, buf, capacity);

hadoop_qat_wrapper/src/main/native/QatDecompressor.c

@@ -38,14 +38,14 @@ static jfieldID QatDecompressor_directBufferSize;
 #define qaePinnedMemFree(x)      qaeMemFreeNUMA((void **)&(x))
 
 #ifdef UNIX
-static int (*dlsym_qzDecompress)(QzSession *sess, const unsigned char* src,
+static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
     unsigned int* compressed_buf_len, unsigned char* dest,
     unsigned int* uncompressed_buffer_len);
-unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned); 
+unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned);
 #endif
 
 #ifdef WINDOWS
-typedef int (__cdecl *__dlsym_qzDecompress)(QzSession *sess, const unsigned char* src,
+typedef int (__cdecl *__dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
     unsigned int* compressed_buf_len, unsigned char* dest,
     unsigned int* uncompressed_buffer_len);
 static __dlsym_qzDecompress dlsym_qzDecompress;
@@ -133,23 +133,19 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_qat_QatDecompressor_de
   compressed_buf_len = compressed_direct_buf_len;
   ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes, &compressed_buf_len,
         uncompressed_bytes, &uncompressed_direct_buf_len);
-  if (ret == QZ_PARAMS){
-    THROW(env, "java/lang/InternalError", "Could not decompress data. Buffer length is too small.");
-  } else if (ret == QZ_FAIL){
-    THROW(env, "java/lang/InternalError", "Could not decompress data. Functionality is not supported.");
-  } else if (ret != QZ_OK){
-    THROW(env, "java/lang/InternalError", "Could not decompress data.");
+  if (ret != QZ_OK) {
+    THROW(env, "java/lang/InternalError", "Could not decompress data, return " + ret);
   }
 
   (*env)->SetIntField(env, thisj, QatDecompressor_compressedDirectBufLen, 0);
 
   return (jint)uncompressed_direct_buf_len;
 }
 
-JNIEXPORT jobject JNICALL 
+JNIEXPORT jobject JNICALL
 Java_org_apache_hadoop_io_compress_qat_QatDecompressor_nativeAllocateBB(JNIEnv *env,
  jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){
-/*void *buf = dlsym_qzMalloc(capacity,0,1);  
+/*void *buf = dlsym_qzMalloc(capacity,0,1);
 if (NULL == buf){
 fprintf(stderr,"decompressor: DBB address is 0x%lx\n",(unsigned long)buf);
 fflush(stderr);

hadoop_qat_wrapper/src/main/native/qatcodec.h

@@ -42,4 +42,4 @@
 
 #define HADOOP_QAT_LIBRARY "libqatzip.so"
 
-extern __thread QzSession  g_qzCompressSession; 
+extern __thread QzSession_T  g_qzCompressSession;

spark_qat_wrapper/src/main/native/QatCodecJNI.c

@@ -40,14 +40,14 @@
 
 #define QAT_ZIP_LIBRARY_NAME "libqatzip.so"
 
-typedef int (*dlsym_qzCompress)(QzSession *sess, const unsigned char* src,
+typedef int (*dlsym_qzCompress)(QzSession_T *sess, const unsigned char* src,
     unsigned int* src_len, unsigned char* dest, unsigned int* dest_len,
     unsigned int last);
-typedef int (*dlsym_qzDecompress)(QzSession *sess, const unsigned char* src,
+typedef int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
     unsigned int* compressed_buf_len, unsigned char* dest,
     unsigned int* uncompressed_buffer_len);
-typedef int (*dlsym_qzGetDefaults)(QzSessionParams *defaults);
-typedef int (*dlsym_qzSetDefaults)(QzSessionParams *defaults);
+typedef int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults);
+typedef int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults);
 
 
 typedef struct qat_wrapper_context {                                                                                                     int magic;
@@ -59,7 +59,7 @@ typedef struct qat_wrapper_context {
 
 qat_wrapper_context_t g_qat_wrapper_context;
 
-__thread QzSession  g_qzSession = {
+__thread QzSession_T  g_qzSession = {
     .internal = NULL,
 };
 
@@ -140,9 +140,9 @@ Java_com_intel_qat_jni_QatCodecJNI_createCompressContext(
         JNIEnv *env, jclass cls, jint level)
 {
     qat_wrapper_context_t *qat_wrapper_context = &g_qat_wrapper_context;
-    QzSessionParams params;
+    QzSessionParams_T params;
     qat_wrapper_context->getDefaults(&params);
-    params.compLvl = level;
+    params.comp_lvl = level;
     //fprintf(stderr, "compression level is %d, tid is %ld\n", level, syscall(__NR_gettid));
     qat_wrapper_context->setDefaults(&params);
     return (jlong)1;