Skip to content
This repository was archived by the owner on Aug 29, 2023. It is now read-only.

Commit 58f1d95

Browse files
zhongijoywinningsix
zhongijoy
authored and
winningsix
committed
carbondata qat codec (#2)
1 parent af2faa6 commit 58f1d95

File tree

9 files changed

+614
-0
lines changed

9 files changed

+614
-0
lines changed

‎carbondata_qat_wrapper/pom.xml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<parent>
6+
<groupId>com.intel.qat</groupId>
7+
<artifactId>qat-parent</artifactId>
8+
<version>1.0.0</version>
9+
<relativePath>../pom.xml</relativePath>
10+
</parent>
11+
12+
<groupId>com.intel.qat</groupId>
13+
<artifactId>carbondata_qat_wrapper</artifactId>
14+
<name>Apache Carbondata QAT Codec</name>
15+
<packaging>jar</packaging>
16+
17+
<properties>
18+
<basedir>./</basedir>
19+
<maven-antrun-plugin.version>1.8</maven-antrun-plugin.version>
20+
<maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version>
21+
</properties>
22+
23+
<dependencies>
24+
<dependency>
25+
<groupId>junit</groupId>
26+
<artifactId>junit</artifactId>
27+
<version>4.11</version>
28+
<scope>test</scope>
29+
</dependency>
30+
<dependency>
31+
<groupId>junit</groupId>
32+
<artifactId>junit</artifactId>
33+
<version>RELEASE</version>
34+
<scope>test</scope>
35+
</dependency>
36+
</dependencies>
37+
38+
<build>
39+
<plugins>
40+
<plugin>
41+
<artifactId>maven-surefire-plugin</artifactId>
42+
<version>${maven-surefire-plugin.version}</version>
43+
<configuration>
44+
<systemPropertyVariables>
45+
<propertyName>java.library.path</propertyName>
46+
<buildDirectory>target</buildDirectory>
47+
</systemPropertyVariables>
48+
<environmentVariables>
49+
<LD_LIBRARY_PATH>${basedir}/src/main/native</LD_LIBRARY_PATH>
50+
</environmentVariables>
51+
</configuration>
52+
</plugin>
53+
<plugin>
54+
<groupId>org.apache.maven.plugins</groupId>
55+
<artifactId>maven-antrun-plugin</artifactId>
56+
<version>${maven-antrun-plugin.version}</version>
57+
<executions>
58+
<execution>
59+
<id>build-native</id>
60+
<phase>compile</phase>
61+
<goals>
62+
<goal>run</goal>
63+
</goals>
64+
<configuration>
65+
<tasks>
66+
<echo>Java Home: ${java.home}</echo>
67+
<property name="native.src.dir" value="${basedir}/src/main/native"/>
68+
<exec dir="${native.src.dir}" executable="make" failonerror="true">
69+
<env key="JAVA_HOME" value="${java.home}/.." />
70+
</exec>
71+
</tasks>
72+
</configuration>
73+
</execution>
74+
75+
<execution>
76+
<id>clean-native</id>
77+
<phase>clean</phase>
78+
<goals>
79+
<goal>run</goal>
80+
</goals>
81+
<configuration>
82+
<tasks>
83+
<exec dir="src/main/native" executable="make" failonerror="true">
84+
<arg value="clean" />
85+
</exec>
86+
</tasks>
87+
</configuration>
88+
</execution>
89+
</executions>
90+
</plugin>
91+
</plugins>
92+
</build>
93+
</project>

‎carbondata_qat_wrapper/src/main/java/com/intel/qat/func/QatCompressor.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.intel.qat.func;
2+
3+
import com.intel.qat.jni.CarbondataQatJNI;
4+
5+
import java.io.IOException;
6+
7+
8+
public class QatCompressor {
9+
10+
public synchronized byte [] compress(byte[] unCompInput) throws IOException {
11+
if (unCompInput == null) {
12+
throw new NullPointerException();
13+
}
14+
15+
int maxDesLen = CarbondataQatJNI.maxCompressedLength(unCompInput.length);
16+
byte[] des = new byte[maxDesLen];
17+
int compressedLen = CarbondataQatJNI.compress(unCompInput, unCompInput.length, des);
18+
19+
byte[] res = new byte[compressedLen];
20+
System.arraycopy(des, 0, res, 0, compressedLen);
21+
return res;
22+
}
23+
24+
public synchronized byte [] compress(byte[] unCompInput, int srcLen) throws IOException {
25+
if (unCompInput == null) {
26+
throw new NullPointerException();
27+
}
28+
if (srcLen < 0) {
29+
throw new ArrayIndexOutOfBoundsException();
30+
}
31+
32+
srcLen = Math.min(srcLen, unCompInput.length);
33+
34+
int maxDesLen = CarbondataQatJNI.maxCompressedLength(srcLen);
35+
byte[] des = new byte[maxDesLen];
36+
int compressedLen = CarbondataQatJNI.compress(unCompInput, srcLen, des);
37+
38+
byte[] res = new byte[compressedLen];
39+
System.arraycopy(des, 0, res, 0, compressedLen);
40+
return res;
41+
}
42+
43+
}

‎carbondata_qat_wrapper/src/main/java/com/intel/qat/func/QatDecompressor.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package com.intel.qat.func;
2+
3+
import com.intel.qat.jni.CarbondataQatJNI;
4+
5+
import java.io.IOException;
6+
7+
public class QatDecompressor {
8+
9+
public synchronized byte [] decompress(byte[] compInput) throws IOException {
10+
if (compInput == null) {
11+
throw new NullPointerException();
12+
}
13+
14+
byte[] des = new byte[compInput.length*2];
15+
int uncompressedLen = CarbondataQatJNI.decompress(compInput, 0, compInput.length, des);
16+
17+
byte[] res = new byte[uncompressedLen];
18+
System.arraycopy(des, 0, res, 0, uncompressedLen);
19+
return res;
20+
}
21+
22+
public synchronized byte [] decompress(byte[] compInput, int srcOff, int srcLen) throws IOException{
23+
if (compInput == null) {
24+
throw new NullPointerException();
25+
}
26+
if (srcOff < 0 || srcLen < 0 || srcOff >= compInput.length) {
27+
throw new ArrayIndexOutOfBoundsException();
28+
}
29+
30+
byte[] des = new byte[compInput.length*2];
31+
int uncompressedLen = CarbondataQatJNI.decompress(compInput, srcOff, srcLen, des);
32+
33+
byte[] res = new byte[uncompressedLen];
34+
System.arraycopy(des, 0, res, 0, uncompressedLen);
35+
return res;
36+
}
37+
}

‎carbondata_qat_wrapper/src/main/java/com/intel/qat/jni/CarbondataQatJNI.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.intel.qat.jni;
2+
3+
public class CarbondataQatJNI {
4+
5+
static{
6+
System.loadLibrary("CarbondataQatJNI");
7+
init();
8+
}
9+
10+
public static native void init();
11+
12+
public static native int compress(byte[] src, int srcLen, byte[] des);
13+
14+
public static native int decompress(byte[] src, int srcOff, int srcLen, byte[] des);
15+
16+
public static native int maxCompressedLength(int srcLen);
17+
}

‎carbondata_qat_wrapper/src/main/native/CarbondataQatJNI.c‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#include "com_intel_qat_jni_CarbondataQatJNI.h"
2+
#include "qatzip.h"
3+
4+
#include <sys/syscall.h>
5+
#include <unistd.h>
6+
#include <dlfcn.h>
7+
#include <stdlib.h>
8+
#include <string.h>
9+
#include <stdint.h>
10+
11+
#define QAT_ZIP_LIBRARY_NAME "libqatzip.so"
12+
13+
/* A helper macro to 'throw' a java exception. */
14+
#define THROW(env, exception_name, message) \
15+
{ \
16+
jclass ecls = (*env)->FindClass(env, exception_name); \
17+
if (ecls) { \
18+
(*env)->ThrowNew(env, ecls, message); \
19+
(*env)->DeleteLocalRef(env, ecls); \
20+
} \
21+
}
22+
23+
typedef int (*dlsym_qzCompress)(QzSession_T *sess, const unsigned char* src,
24+
unsigned int* src_len, unsigned char* dest, unsigned int* dest_len,
25+
unsigned int last);
26+
typedef int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
27+
unsigned int* compressed_buf_len, unsigned char* dest,
28+
unsigned int* uncompressed_buffer_len);
29+
typedef int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults);
30+
typedef unsigned int (*dlsym_qzMaxCompressedLength)(unsigned int src_sz);
31+
32+
33+
typedef struct qat_wrapper_context {
34+
dlsym_qzCompress compress;
35+
dlsym_qzDecompress decompress;
36+
dlsym_qzGetDefaults getDefaults;
37+
dlsym_qzMaxCompressedLength maxCompressedLength;
38+
} qat_wrapper_context_t;
39+
40+
qat_wrapper_context_t g_qat_wrapper_context;
41+
42+
43+
__thread QzSession_T g_qzSession = {
44+
.internal = NULL,
45+
};
46+
47+
JNIEXPORT void JNICALL Java_com_intel_qat_jni_CarbondataQatJNI_init
48+
(JNIEnv *env, jclass cls) {
49+
50+
qat_wrapper_context_t *qat_wrapper_context = &g_qat_wrapper_context;
51+
void *lib = dlopen(QAT_ZIP_LIBRARY_NAME, RTLD_LAZY | RTLD_GLOBAL);
52+
if (!lib){
53+
char msg[128];
54+
snprintf(msg, 128, "Can't load %s due to %s", QAT_ZIP_LIBRARY_NAME, dlerror());
55+
THROW(env, "java/lang/UnsatisfiedLinkError", msg);
56+
}
57+
58+
dlerror(); // Clear any existing error
59+
60+
qat_wrapper_context->compress = dlsym(lib, "qzCompress");
61+
if (qat_wrapper_context->compress == NULL) {
62+
THROW(env, "java/lang/UnsatisfiedLinkError", "Failed to load qzCompress");
63+
}
64+
65+
qat_wrapper_context->decompress = dlsym(lib, "qzDecompress");
66+
if (qat_wrapper_context->compress == NULL) {
67+
THROW(env, "java/lang/UnsatisfiedLinkError", "Failed to load qzDecompress");
68+
}
69+
70+
qat_wrapper_context->getDefaults = dlsym(lib, "qzGetDefaults");
71+
if (qat_wrapper_context->getDefaults == NULL) {
72+
THROW(env, "java/lang/UnsatisfiedLinkError", "Failed to load qzGetDefaults");
73+
}
74+
75+
qat_wrapper_context->maxCompressedLength = dlsym(lib, "qzMaxCompressedLength");
76+
if (qat_wrapper_context->maxCompressedLength == NULL) {
77+
THROW(env, "java/lang/UnsatisfiedLinkError", "why Failed to load qzMaxCompressedLength");
78+
}
79+
}
80+
81+
82+
JNIEXPORT jint JNICALL Java_com_intel_qat_jni_CarbondataQatJNI_compress
83+
(JNIEnv *env, jclass cls, jbyteArray src, jint srcLen, jbyteArray des){
84+
85+
jbyte *in = (*env)->GetByteArrayElements(env, src, 0);
86+
if (in == NULL) {
87+
THROW(env, "java/lang/OutOfMemoryError", "Can't get compressor input buffer");
88+
}
89+
90+
jbyte *out = (*env)->GetByteArrayElements(env, des, 0);
91+
if (out == NULL) {
92+
THROW(env, "java/lang/OutOfMemoryError", "Can't get compressor output buffer");
93+
}
94+
95+
uint32_t uncompressed_size = srcLen;
96+
uint32_t compressed_size = srcLen+1000;
97+
qat_wrapper_context_t *qat_wrapper_context = &g_qat_wrapper_context;
98+
int ret = qat_wrapper_context->compress(&g_qzSession, in, &uncompressed_size, out, &compressed_size, 1);
99+
if (ret == QZ_OK) {
100+
}
101+
else if (ret == QZ_PARAMS) {
102+
THROW(env, "java/lang/InternalError", "Could not compress data. *sess is NULL or member of params is invalid.");
103+
}
104+
else if (ret == QZ_FAIL) {
105+
THROW(env, "java/lang/InternalError", "Could not compress data. Function did not succeed.");
106+
}
107+
else {
108+
char temp[256];
109+
sprintf(temp, "Could not compress data. Return error code %d", ret);
110+
THROW(env, "java/lang/InternalError", temp);
111+
}
112+
113+
(*env)->ReleaseByteArrayElements(env, src, in, JNI_COMMIT);
114+
(*env)->ReleaseByteArrayElements(env, des, out, JNI_COMMIT);
115+
116+
return compressed_size;
117+
}
118+
119+
120+
JNIEXPORT jint JNICALL Java_com_intel_qat_jni_CarbondataQatJNI_decompress
121+
(JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jint srcLen, jbyteArray des){
122+
123+
jbyte *in = (*env)->GetByteArrayElements(env, src, 0);
124+
if (in == NULL) {
125+
THROW(env, "java/lang/OutOfMemoryError", "Can't get decompressor input buffer");
126+
}
127+
in += srcOff;
128+
129+
jbyte *out = (*env)->GetByteArrayElements(env, des, 0);
130+
if (out == NULL) {
131+
THROW(env, "java/lang/OutOfMemoryError", "Can't get decompressor output buffer");
132+
}
133+
134+
uint32_t compressed_size = srcLen;
135+
uint32_t uncompressed_size = srcLen*2;
136+
137+
qat_wrapper_context_t *qat_wrapper_context = &g_qat_wrapper_context;
138+
int ret = qat_wrapper_context->decompress(&g_qzSession, in, &compressed_size, out, &uncompressed_size);
139+
if (ret == QZ_OK) {
140+
}
141+
else if (ret == QZ_PARAMS) {
142+
THROW(env, "java/lang/InternalError", "Could not decompress data. *sess is NULL or member of params is invalid");
143+
}
144+
else if (ret == QZ_FAIL) {
145+
THROW(env, "java/lang/InternalError", "Could not decompress data. Function did not succeed.");
146+
}
147+
else {
148+
char temp[256];
149+
sprintf(temp, "Could not decompress data. Return error code %d", ret);
150+
THROW(env, "java/lang/InternalError", temp);
151+
}
152+
153+
(*env)->ReleaseByteArrayElements(env, src, in, JNI_COMMIT);
154+
(*env)->ReleaseByteArrayElements(env, des, out, JNI_COMMIT);
155+
// if ((*env)->ExceptionCheck(env)){
156+
// THROW(env, "java/lang/InternalError", "Could not release array. Function did not succeed.");
157+
// }
158+
return uncompressed_size;
159+
}
160+
161+
162+
JNIEXPORT jint JNICALL Java_com_intel_qat_jni_CarbondataQatJNI_maxCompressedLength
163+
(JNIEnv *env, jclass cls, jint length){
164+
165+
qat_wrapper_context_t *qat_wrapper_context = &g_qat_wrapper_context;
166+
uint32_t ret = qat_wrapper_context->maxCompressedLength((uint32_t)length);
167+
return ret;
168+
}
169+

0 commit comments

Comments
 (0)