Skip to content

Commit e359ec6

Browse files
lesserwhirlslesserwhirls
authored
Merge pull request #1426 from lesserwhirls/gh-753
Add basic support for libaec decoding using native libraries
2 parents 1ef165d + 552432e commit e359ec6

File tree

10 files changed

+676
-0
lines changed

10 files changed

+676
-0
lines changed

‎.github/workflows/test-native-compression.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
on:
2+
workflow_dispatch:
3+
pull_request:
4+
paths:
5+
- 'native-compression/libaec-jna/**'
6+
- 'native-compression/libaec-native/**'
7+
- 'native-compression/build.gradle'
8+
9+
jobs:
10+
tests:
11+
strategy:
12+
matrix:
13+
os: [
14+
ubuntu-24.04,
15+
ubuntu-24.04-arm,
16+
windows-2022,
17+
macos-14,
18+
macos-13
19+
]
20+
name: netCDF-Java Native Compression Tests
21+
runs-on: ${{ matrix.os }}
22+
steps:
23+
- uses: actions/checkout@v4
24+
- name: Set up JDK 11
25+
uses: actions/setup-java@v4
26+
with:
27+
distribution: 'temurin'
28+
java-version: '11'
29+
- name: Cache Gradle packages
30+
uses: actions/cache@v4
31+
with:
32+
path: |
33+
~/.gradle/caches
34+
~/.gradle/wrapper
35+
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
36+
restore-keys: |
37+
${{ runner.os }}-gradle-
38+
- name: Run libaec JNA tests
39+
run: ./gradlew clean :native-compression:libaec-jna:test
40+
- uses: actions/upload-artifact@v4
41+
if: failure()
42+
with:
43+
name: NativeCompression_JUnit_Results_${{ github.sha }}_-${{ matrix.os }}
44+
path: native-compression/libaec-jna/build/reports/tests

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,6 @@ thredds.tgz
6161

6262
# dependency check NVD files
6363
project-files/owasp-dependency-check/nvd
64+
65+
# native binary distribution files
66+
project-files/native

‎native-compression/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
description = 'Native binding for compression'
2+
ext.title = 'native compression binding'

‎native-compression/libaec-jna/build.gradle‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apply from: "$rootDir/gradle/any/dependencies.gradle"
2+
apply from: "$rootDir/gradle/any/java-library.gradle"
3+
4+
description = 'Java bindings for decoding libaec compression using JNA'
5+
ext.title = 'libaec compression decoder using JNA'
6+
7+
dependencies {
8+
api enforcedPlatform(project(':netcdf-java-platform'))
9+
testImplementation enforcedPlatform(project(':netcdf-java-testing-platform'))
10+
11+
api 'net.java.dev.jna:jna'
12+
implementation 'org.slf4j:slf4j-api'
13+
14+
testImplementation project(':cdm-test-utils')
15+
16+
testImplementation 'com.google.truth:truth'
17+
18+
testRuntimeOnly project(':native-compression:libaec-native')
19+
testRuntimeOnly 'ch.qos.logback:logback-classic'
20+
}

‎native-compression/libaec-jna/src/main/java/edu/ucar/unidata/compression/jna/libaec/LibAec.java‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
* Copyright (c) 2025 University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE for license information.
4+
*/
5+
6+
package edu.ucar.unidata.compression.jna.libaec;
7+
8+
import com.sun.jna.Memory;
9+
import com.sun.jna.Native;
10+
import com.sun.jna.Pointer;
11+
import com.sun.jna.Structure;
12+
import com.sun.jna.ptr.PointerByReference;
13+
import java.io.File;
14+
import java.io.IOException;
15+
import java.util.Arrays;
16+
import java.util.List;
17+
18+
/**
19+
* JNA access to libaec. Not a full implementation, just the functions
20+
* actually used for decoding (and testing). This is a transliteration
21+
* of the libaec library file include/libaec.h.
22+
*
23+
* @author sarms
24+
* @since 5.7.1
25+
*/
26+
27+
public final class LibAec {
28+
29+
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(LibAec.class);
30+
private static final String libName = "aec";
31+
32+
static {
33+
try {
34+
File library = Native.extractFromResourcePath(libName);
35+
Native.register(library.getAbsolutePath());
36+
log.debug("Using libaec library from libaec-native.jar");
37+
} catch (IOException e) {
38+
try {
39+
Native.register(libName);
40+
log.debug("Using libaec library from system");
41+
} catch (UnsatisfiedLinkError ule) {
42+
String message =
43+
"libaec C library not present. To read this data, include the libaec-native jar in your classpath "
44+
+ "(edu.ucar:libaec-native) or install libaec on your system.";
45+
log.error(message);
46+
throw new RuntimeException(message, ule);
47+
}
48+
}
49+
}
50+
51+
public static class AecStream extends Structure {
52+
public static AecStream create(int bitsPerSample, int blockSize, int rsi, int flags) {
53+
AecStream aecStream = new AecStream();
54+
aecStream.bits_per_sample = bitsPerSample;
55+
aecStream.block_size = blockSize;
56+
aecStream.rsi = rsi;
57+
aecStream.flags = flags;
58+
59+
return aecStream;
60+
}
61+
62+
public void setInputMemory(Memory inputMemory) {
63+
this.next_in = inputMemory;
64+
this.avail_in = new SizeT(inputMemory.size());
65+
}
66+
67+
public void setOutputMemory(Memory outputMemory) {
68+
this.next_out = outputMemory;
69+
this.avail_out = new SizeT(outputMemory.size());
70+
}
71+
72+
public Pointer next_in;
73+
// number of bytes available at next_in
74+
public SizeT avail_in;
75+
76+
// total number of input bytes read so far
77+
public SizeT total_in;
78+
79+
public Pointer next_out;
80+
81+
// remaining free space at next_out
82+
public SizeT avail_out;
83+
84+
// total number of bytes output so far
85+
public SizeT total_out;
86+
87+
// resolution in bits per sample (n = 1, ..., 32)
88+
public int bits_per_sample;
89+
90+
// block size in samples
91+
public int block_size;
92+
93+
// Reference sample interval, the number of blocks
94+
// between consecutive reference samples (up to 4096)
95+
public int rsi;
96+
97+
public int flags;
98+
99+
public volatile PointerByReference state;
100+
101+
@Override
102+
protected List<String> getFieldOrder() {
103+
return Arrays.asList("next_in", "avail_in", "total_in", "next_out", "avail_out", "total_out", "bits_per_sample",
104+
"block_size", "rsi", "flags", "state");
105+
}
106+
}
107+
108+
// Sample data description flags
109+
110+
// Samples are signed. Telling libaec this results in a slightly
111+
// better compression ratio. Default is unsigned.
112+
static final int AEC_DATA_SIGNED = 1;
113+
114+
// 24 bit samples are coded in 3 bytes
115+
static final int AEC_DATA_3BYTE = 2;
116+
117+
// Samples are stored with their most significant bit first. This has
118+
// nothing to do with the endianness of the host. Default is LSB.
119+
static final int AEC_DATA_MSB = 4;
120+
121+
// Set if preprocessor should be used
122+
static final int AEC_DATA_PREPROCESS = 8;
123+
124+
// Use restricted set of code options
125+
static final int AEC_RESTRICTED = 16;
126+
127+
128+
// Pad RSI to byte boundary. Only used for decoding some CCSDS sample
129+
// data. Do not use this to produce new data as it violates the
130+
// standard.
131+
static final int AEC_PAD_RSI = 32;
132+
133+
// Do not enforce standard regarding legal block sizes.
134+
static final int AEC_NOT_ENFORCE = 64;
135+
136+
// Return codes of library functions
137+
138+
public static final int AEC_OK = 0;
139+
public static final int AEC_CONF_ERROR = (-1);
140+
public static final int AEC_STREAM_ERROR = (-2);
141+
public static final int AEC_DATA_ERROR = (-3);
142+
public static final int AEC_MEM_ERROR = (-4);
143+
public static final int AEC_RSI_OFFSETS_ERROR = (-5);
144+
145+
// Options for flushing
146+
147+
// Do not enforce output flushing. More input may be provided with
148+
// later calls. So far only relevant for encoding.
149+
public static final int AEC_NO_FLUSH = 0;
150+
151+
// Flush output and end encoding. The last call to aec_encode() must
152+
// set AEC_FLUSH to drain all output.
153+
// It is not possible to continue encoding of the same stream after it
154+
// has been flushed. For one, the last block may be padded zeros after
155+
// preprocessing. Secondly, the last encoded byte may be padded with
156+
// fill bits.
157+
public static final int AEC_FLUSH = 1;
158+
159+
// Declare native methods corresponding to the functions in libaec.h
160+
// package private - for round trip testing only
161+
static native int aec_buffer_encode(AecStream strm);
162+
163+
public static native int aec_buffer_decode(AecStream strm);
164+
}

‎native-compression/libaec-jna/src/main/java/edu/ucar/unidata/compression/jna/libaec/SizeT.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE for license information.
4+
*/
5+
6+
package edu.ucar.unidata.compression.jna.libaec;
7+
8+
import com.sun.jna.IntegerType;
9+
import com.sun.jna.Native;
10+
11+
/**
12+
* Map a native size_t with JNA.
13+
*
14+
* @see <a href="https://github.com/twall/jna/issues/191" />
15+
*/
16+
public class SizeT extends IntegerType {
17+
public SizeT() {
18+
this(0);
19+
}
20+
21+
public SizeT(long value) {
22+
super(Native.SIZE_T_SIZE, value, true);
23+
}
24+
25+
public String toString() {
26+
return String.format("%d", super.longValue());
27+
}
28+
}

0 commit comments

Comments
 (0)