Skip to content

Commit 5d2fd26

Browse files
ayzkayzk
authored
SZ3.3 (#94)
* Algorithmic updates * replace pointwise iterator by blockwise iterator * replace pointwise iterator by blockwise iterator * Merge test units (#95) * update python support for Windows * update config to support save and load to ini * update error handling * update conf * Speeding up SZ3 Python API Compress and Decompress functions (#99)
1 parent dd09ba7 commit 5d2fd26

File tree

174 files changed

+3110
-66364
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

174 files changed

+3110
-66364
lines changed

‎.github/workflows/cmake.yml‎

Lines changed: 239 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,248 @@
1-
name: CMake
1+
name: Cross-Platform SZ3 Verification
22

3-
on: [push]
3+
on: [ push ]
44

55
env:
6-
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
76
BUILD_TYPE: Release
7+
TEST_URL: https://raw.githubusercontent.com/szcompressor/SZ3/master/tools/sz3/testfloat_8_8_128.dat
8+
DIMS: "128 8 8"
9+
MODE: ABS
10+
TOL: 1
811

912
jobs:
10-
build:
11-
# The CMake configure and build commands are platform agnostic and should work equally
12-
# well on Windows or Mac. You can convert this to a matrix build if you need
13-
# cross-platform coverage.
14-
# See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
13+
build-linux:
14+
name: Build & Test (Linux)
1515
runs-on: ubuntu-latest
16+
outputs:
17+
digest: ${{ steps.calc.outputs.digest }}
18+
steps:
19+
- uses: actions/checkout@v2
20+
21+
- name: Install HDF5
22+
run: |
23+
sudo apt-get update
24+
sudo apt-get install -y libhdf5-dev
25+
26+
- name: Create build dir
27+
run: cmake -E make_directory build
28+
29+
- name: Configure
30+
shell: bash
31+
working-directory: build
32+
run: |
33+
cmake .. \
34+
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
35+
-DBUILD_TESTING=ON \
36+
-DBUILD_H5Z_FILTER=ON
37+
38+
- name: Build
39+
shell: bash
40+
working-directory: build
41+
run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4
42+
43+
- name: Run CTest
44+
shell: bash
45+
working-directory: build
46+
run: ctest --output-on-failure -C ${{ env.BUILD_TYPE }}
47+
48+
- name: Download test data
49+
shell: bash
50+
run: |
51+
curl -L -o input.dat ${{ env.TEST_URL }}
52+
53+
- name: Run sz3 and check error
54+
shell: bash
55+
working-directory: build
56+
run: |
57+
./tools/sz3/sz3 -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \
58+
-o out.dat -z out.sz3 \
59+
| tee sz3.log
60+
ERR=$(grep -E 'Max absolute error =' sz3.log \
61+
| sed -E 's/.*Max absolute error = *([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?).*/\1/')
62+
63+
[ -n "$ERR" ] || { echo "Could not parse Max absolute error!"; exit 1; }
64+
echo "Parsed error: $ERR"
65+
awk -v err="$ERR" -v tol="${{ env.TOL }}" 'BEGIN { if (err > tol) exit 1; }'
66+
67+
- name: Compute SHA256 of compressed
68+
id: calc
69+
shell: bash
70+
working-directory: build
71+
run: |
72+
DIGEST=$(sha256sum out.sz3 | awk '{print $1}')
73+
echo "::set-output name=digest::$DIGEST"
74+
75+
- name: Upload Linux-compressed artifact
76+
uses: actions/upload-artifact@v4
77+
with:
78+
name: temp-linux-out-sz3
79+
path: build/out.sz3
80+
81+
build-macos:
82+
name: Build & Test (macOS)
83+
runs-on: macos-latest
84+
outputs:
85+
digest: ${{ steps.calc.outputs.digest }}
86+
steps:
87+
- uses: actions/checkout@v2
88+
89+
- name: Install HDF5
90+
run: |
91+
brew update
92+
brew install hdf5
93+
94+
- name: Create build dir
95+
run: cmake -E make_directory build
96+
97+
- name: Configure
98+
shell: bash
99+
working-directory: build
100+
run: |
101+
cmake .. \
102+
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
103+
-DBUILD_TESTING=ON \
104+
-DBUILD_H5Z_FILTER=ON
105+
106+
- name: Build
107+
shell: bash
108+
working-directory: build
109+
run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4
110+
111+
- name: Run CTest
112+
shell: bash
113+
working-directory: build
114+
run: ctest --output-on-failure -C ${{ env.BUILD_TYPE }}
115+
116+
- name: Download test data
117+
shell: bash
118+
run: |
119+
curl -L -o input.dat ${{ env.TEST_URL }}
120+
121+
- name: Run sz3 and check error
122+
shell: bash
123+
working-directory: build
124+
run: |
125+
./tools/sz3/sz3 -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \
126+
-o out.dat -z out.sz3 \
127+
| tee sz3.log
128+
ERR=$(grep -E 'Max absolute error =' sz3.log \
129+
| sed -E 's/.*Max absolute error = *([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?).*/\1/')
16130
131+
[ -n "$ERR" ] || { echo "Could not parse Max absolute error!"; exit 1; }
132+
echo "Parsed error: $ERR"
133+
awk -v err="$ERR" -v tol="${{ env.TOL }}" 'BEGIN { if (err > tol) exit 1; }'
134+
135+
- name: Compute SHA256 of compressed
136+
id: calc
137+
shell: bash
138+
working-directory: build
139+
run: |
140+
DIGEST=$(shasum -a 256 out.sz3 | awk '{print $1}')
141+
echo "::set-output name=digest::$DIGEST"
142+
143+
build-windows:
144+
name: Build & Test (Windows)
145+
runs-on: windows-latest
146+
needs: build-linux
147+
steps:
148+
- uses: actions/checkout@v2
149+
150+
# - name: Bootstrap vcpkg & install HDF5
151+
# shell: pwsh
152+
# run: |
153+
# git clone https://github.com/microsoft/vcpkg.git C:\vcpkg
154+
# cd C:\vcpkg
155+
# .\bootstrap-vcpkg.bat
156+
# .\vcpkg.exe install hdf5:x64-windows
157+
# echo "VCPKG_ROOT=C:\vcpkg" | Out-File -FilePath $Env:GITHUB_ENV -Encoding ASCII
158+
159+
- name: Create build dir
160+
shell: pwsh
161+
run: cmake -E make_directory build
162+
163+
- name: Configure
164+
shell: pwsh
165+
working-directory: build
166+
run: |
167+
cmake .. `
168+
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
169+
170+
- name: Build
171+
shell: pwsh
172+
working-directory: build
173+
run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4
174+
175+
# - name: Run CTest
176+
# shell: pwsh
177+
# working-directory: build
178+
# run: ctest --output-on-failure -C ${{ env.BUILD_TYPE }}
179+
180+
- name: Download test data
181+
shell: pwsh
182+
run: Invoke-WebRequest -UseBasicParsing -Uri ${{ env.TEST_URL }} -OutFile input.dat
183+
184+
- name: Run sz3 and check error
185+
shell: pwsh
186+
working-directory: build
187+
run: |
188+
.\tools\sz3\Release\sz3.exe -f -i ..\input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a `
189+
-o out.dat -z out.sz3 2>&1 | Tee-Object sz3.log
190+
$match = Select-String -Pattern 'Max absolute error =\s*([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)' -Path sz3.log
191+
if (-not $match) {
192+
Write-Error "Could not parse Max absolute error!"
193+
exit 1
194+
}
195+
196+
$err = [double]$match.Matches[0].Groups[1].Value
197+
Write-Host "Parsed error: $err"
198+
199+
if ($err -gt [double]${{ env.TOL }}) {
200+
Write-Error "Error $err exceeds tolerance ${{ env.TOL }}"
201+
exit 1
202+
}
203+
204+
205+
- name: Download Linux-compressed artifact
206+
uses: actions/download-artifact@v4
207+
with:
208+
name: temp-linux-out-sz3
209+
path: .
210+
211+
- name: Run sz3 to decompress and check error
212+
shell: pwsh
213+
working-directory: build
214+
run: |
215+
.\tools\sz3\Release\sz3.exe -f -i ..\input.dat -z ..\out.sz3 -a -o out_recon.dat `
216+
2>&1 | Tee-Object sz3.log
217+
218+
$match = Select-String -Pattern 'Max absolute error =\s*([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)' -Path sz3.log
219+
if (-not $match) {
220+
Write-Error "Could not parse Max absolute error!"
221+
exit 1
222+
}
223+
224+
$err = [double]$match.Matches[0].Groups[1].Value
225+
Write-Host "Parsed error: $err"
226+
227+
if ($err -gt [double]${{ env.TOL }}) {
228+
Write-Error "Error $err exceeds tolerance ${{ env.TOL }}"
229+
exit 1
230+
}
231+
232+
compare:
233+
name: Compare Digests (Linux vs macOS)
234+
runs-on: ubuntu-latest
235+
needs: [ build-linux, build-macos ]
17236
steps:
18-
- uses: actions/checkout@v2
19-
20-
- name: Create Build Environment
21-
# Some projects don't allow in-source building, so create a separate build directory
22-
# We'll use this as our working directory for all subsequent commands
23-
run: cmake -E make_directory ${{github.workspace}}/build
24-
25-
- name: Configure CMake
26-
# Use a bash shell so we can use the same syntax for environment variable
27-
# access regardless of the host operating system
28-
shell: bash
29-
working-directory: ${{github.workspace}}/build
30-
# Note the current convention is to use the -S and -B options here to specify source
31-
# and build directories, but this is only available with CMake 3.13 and higher.
32-
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
33-
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
34-
35-
- name: Build
36-
working-directory: ${{github.workspace}}/build
37-
shell: bash
38-
# Execute the build. You can specify a specific target with "--target <NAME>"
39-
run: cmake --build . --config $BUILD_TYPE
40-
41-
# - name: Test
42-
# working-directory: ${{github.workspace}}/build
43-
# shell: bash
44-
# # Execute tests defined by the CMake configuration.
45-
# # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
46-
# run: ctest -C $BUILD_TYPE
237+
- name: Check that Linux and macOS digests match
238+
shell: bash
239+
run: |
240+
L=${{ needs.build-linux.outputs.digest }}
241+
M=${{ needs.build-macos.outputs.digest }}
242+
echo "linux: $L"
243+
echo "macos: $M"
244+
if [[ "$L" != "$M" ]]; then
245+
echo "::error ::Compressed outputs differ between Linux and macOS!"
246+
exit 1
247+
fi
248+
echo "Linux and macOS compressed outputs are identical."

‎CMakeLists.txt‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
cmake_minimum_required(VERSION 3.18)
2-
project(SZ3 VERSION 3.2.2)
2+
project(SZ3 VERSION 3.3.0)
33

44
#data version defines the version of the compressed data format
55
#it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0)
66
#only update data version if the new version of the program changes compressed data format
7-
set(SZ3_DATA_VERSION 3.2.1)
7+
set(SZ3_DATA_VERSION 3.3.0)
88

99
include(GNUInstallDirs)
10-
include(CTest)
10+
1111

1212
option(BUILD_SHARED_LIBS "build shared libraries by default" ON)
1313
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -60,6 +60,13 @@ else ()
6060
target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_DEBUG_TIMINGS=0)
6161
endif ()
6262

63+
option(SZ3_USE_SKA_HASH "use ska hashmap instead of std hashmap, default is ska for better speed" ON)
64+
if (SZ3_USE_SKA_HASH)
65+
target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_USE_SKA_HASH=1)
66+
else ()
67+
target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_USE_SKA_HASH=0)
68+
endif ()
69+
6370
pkg_search_module(ZSTD IMPORTED_TARGET libzstd)
6471
if (ZSTD_FOUND AND NOT SZ3_USE_BUNDLED_ZSTD)
6572
target_link_libraries(${PROJECT_NAME} INTERFACE PkgConfig::ZSTD)
@@ -68,6 +75,12 @@ else ()
6875
target_link_libraries(${PROJECT_NAME} INTERFACE zstd)
6976
endif ()
7077

78+
option(BUILD_TESTING "Build tests" OFF)
79+
if (BUILD_TESTING)
80+
enable_testing()
81+
add_subdirectory(tools/test)
82+
endif ()
83+
7184
find_package(GSL)
7285
if (GSL_FOUND)
7386
target_compile_definitions(${PROJECT_NAME} INTERFACE -DSZ3_ENABLE_GSL)

0 commit comments

Comments
 (0)