read binary data --> generate volume for CI

add CvxCompress_Test to CI

changed number of iterations to 10

cleaned up for merge

Author: jkwashbourne

.github/workflows/ci.yaml

@@ -43,5 +43,8 @@ jobs:
           make
           ls
       
-      - name: Run test
+      - name: Run CvxCompress_Test
+        run: ./CvxCompress_Test
+
+      - name: Run Test_With_Generated_Input
         run: ./Test_With_Generated_Input

.gitignore

@@ -1,6 +1,8 @@
 *.o 
 *.so
 *~
+Ds79_Base.cpp
+Us79_Base.cpp
 Compress_Guojians_Stuff
 Compress_SEAM_Basin
 Compress_Sams_Stuff
@@ -10,4 +12,4 @@ CvxCompress_Test_Dyn
 Test_Compression
 Test_With_Generated_Input
 results.txt
-build/
+build/

CvxCompress.cpp

@@ -736,12 +736,12 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 	else
 	{
 		if (forward_passed)
-			printf("[\x1B[32mPassed\x1B[0m]\n"); 
+			printf("[\x1B[32mPassed!\x1B[0m]\n"); 
 		else 
-			printf("[\x1B[31mFailed\x1B[0m]\n");
+			printf("[\x1B[31mFailed!\x1B[0m]\n");
 	}
 
-	printf("3. Verify correctness of inverse wavelet transform...");
+	printf("\n3. Verify correctness of inverse wavelet transform...");
 	if (verbose) printf("\n");
 	bool inverse_passed = true;
 	for (int k = min_k;  k <= max_k;  ++k)
@@ -776,9 +776,9 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 	else
 	{
 		if (inverse_passed)
-			printf("[\x1B[32mPassed\x1B[0m]\n");
+			printf("[\x1B[32mPassed!\x1B[0m]\n");
 		else 
-			printf("[\x1B[31mFailed\x1B[0m]\n");
+			printf("[\x1B[31mFailed!\x1B[0m]\n");
 	}
 
 #ifdef PAPI
@@ -797,7 +797,7 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
         }
 #endif
 
-	printf("4. Test throughput of wavelet transform (forward + inverse)...\n");
+	printf("\n4. Test throughput of wavelet transform (forward + inverse)...\n");
 	for (int k = min_k;  k <= max_k;  ++k)
 	{
 		int bz = 1 << k;
@@ -961,7 +961,7 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 		else
 			printf("\x1B[0m[\x1B[31mFailed!\x1B[0m]\n");
 
-	printf("6. Verify correctness of Copy_From_Block method...");  fflush(stdout);
+	printf("\n6. Verify correctness of Copy_From_Block method...");  fflush(stdout);
 	bool copy_from_block_passed = true;
 	if (vol == 0L || block == 0L)
 	{
@@ -1027,7 +1027,7 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 		else
 			printf("\x1B[0m[\x1B[31mFailed!\x1B[0m]\n");
 
-	printf("7. Test throughput of block copy...");  fflush(stdout);
+	printf("\n7. Test throughput of block copy...");  fflush(stdout);
 	bool copy_round_trip_passed = true;
 	if (vol == 0L)
 	{
@@ -1094,9 +1094,8 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 			}
 		}
 	}
-	printf("\n");
 
-	printf("8. Verify correctness of Global_RMS method...");  fflush(stdout);
+	printf("\n8. Verify correctness of Global_RMS method...");  fflush(stdout);
 	bool global_rms_passed = true;
 	if (vol == 0L || block == 0L)
 	{
@@ -1130,9 +1129,10 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 
 	float scale = 1e-1f;
 
-	printf("9. Test throughput of Compress() method...\n");
+	printf("\n9. Test throughput of Compress() method...\n");
 	int nx3,ny3,nz3;
 	float* vol3;
+	// 2024.10.27 instead of reading a binary file, we now create a volume in the function Read_Raw_Volume 
 	Read_Raw_Volume("/cpfs/lfs02/ESDRD/tjhc/pressure_at_t=7512.bin",nx3,ny3,nz3,vol3);
 	//Read_Raw_Volume("/cpfs/lfs01/ESDRD/tjhc/fdmod2/trunk/CvxCompress/empty.bin",nx3,ny3,nz3,vol3);
 	unsigned long* compressed3;
@@ -1160,7 +1160,6 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 						memtype = "DRAM";
 					printf("\x1B[0m -> %3d x %3d x %3d (%s) ",bx,by,bz,memtype);  fflush(stdout);
 
-
 					auto start = Time::now();
 					double elapsed = 0.0;
 
@@ -1176,16 +1175,16 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 						double mcells_per_sec = (double)niter * (double)nx3 * (double)ny3 * (double)nz3 / (elapsed.count() * 1e6);
 						printf("\r\x1B[0m -> %3d x %3d x %3d (%s) %2d iterations - %6.3f secs - %.0f MCells/s - ratio %.2f:1",bx,by,bz,memtype,niter,elapsed.count(),mcells_per_sec,ratio);
 						fflush(stdout);
-					} while (elapsed < 10.0);
+					// } while (elapsed < 10.0); // 2024.10.27 switch to iteration count, elapased doesnt make sense
+					} while (niter < 10);
 					printf("\n");
-					//printf("%d iterations - %6.3f secs - %.0f MCells/s - ratio %.2f:1\n",niter,elapsed,mcells_per_sec,ratio);
 				}
 			}
 		}
 	}
 
-	printf("10. Test throughput of Decompress() method...\n");
-	for (int k = min_k;  k <= max_k-2;  ++k)
+	printf("\n10. Test throughput of Decompress() method...\n");
+	for (int k = min_k;  k <= max_k-1;  ++k)
 	{
 		int bz = 1 << k;
 		for (int j = min_j;  j <= max_j;  ++j)
@@ -1226,7 +1225,8 @@ bool CvxCompress::Run_Module_Tests(bool verbose, bool exhaustive_throughput_test
 						printf("\r\x1B[0m -> %3d x %3d x %3d (%s) %2d iterations - %6.3f secs - %.0f MCells/s",bx,by,bz,memtype,niter,elapsed.count(),mcells_per_sec);
 						fflush(stdout);
 						free(vol4);
-					} while (elapsed < 10.0);
+					// } while (elapsed < 10.0); // 2024.10.27 switch to iteration count, elapased doesnt make sense
+					} while (niter < 10);
 					printf("\n");
 				}
 			}

CvxCompress.hxx

@@ -40,7 +40,7 @@ public:
 			unsigned int* compressed,
 			int num_threads,
 			long& compressed_length
-		      );
+			);
 	/*!
 	 * Compress a 3D wavefield using a given block size
 	 * nx is fast, nz is slow
@@ -60,7 +60,7 @@ public:
 			bool use_local_RMS,
 			unsigned int* compressed,
 			long& compressed_length
-		      );
+			);
 	/*!
  	 * Compress a 3D wavefield using a given block size.
  	 * Works same as above, except parameter use_local_RMS is hardcoded to be false.
@@ -108,7 +108,7 @@ public:
 			unsigned int* compressed,
 			int num_threads,
 			long compressed_length 
-		       );
+			);
 
 	void Decompress(
 			float* vol,
@@ -117,7 +117,7 @@ public:
 			int nz,
 			unsigned int* compressed,
 			long compressed_length 
-		       );
+			);
 
 	bool Is_Valid_Block_Size(int bx, int by, int bz);
 

README.md

@@ -1,84 +1,100 @@
 # CvxCompres library
-This CvxCompress was originally authored by Thor Johnsen.    
+This CvxCompress was originally authored by Thor Johnsen, based on an implementation by Ray Ergas. 
+For more information here are references to that earlier work. 
+
+```
+Ergas, R.A., "Wavelet Transform Techniques for Seismic Data Compression and Denoising," Society of Exploration Geophysicists Annual Meeting, 1995.
+
+Ergas, R.A., Donoho, P.L., and Villasenor, J.D., "Multiresolution Analysis for Seismic Data Compression Using Wavelets," IEEE Transactions on Signal Processing, 1995.
+
+Ergas, R.A., "Seismic data compression--A key technology for the future", Society of Exploration Geophysicists Annual Meeting, 1996.
+https://library.seg.org/doi/pdf/10.1190/1.1826518
+```
 
 # Building
+
 ## GCC
 
-It is believed that this will work with a wide variety of gcc
-versions, but as I am writing this I have only tested it with gcc 7.30 and gcc 9.3.0. 
+It is believed that this will work any reasonably modern gcc version. 
 
 ## Tests
 
-There are several tests included, some of which depends on binary
-cubes of input need to exist at some absolute path which is no longer
-present. Thus, here I describe only the ones that require no input file. 
+There are two test sets included, below is the command line to run and the output.   
 
 ```
 Prompt> ( setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${cwd} ; ./CvxCompress_Test )
 *
-* CvxCompress module tests (ICC1900, AVX).
+* CvxCompress module tests (GCC13.2, AVX).
 *
 
-2. Verify correctness of forward wavelet transform...[Passed]
-3. Verify correctness of inverse wavelet transform...[Passed]
+2. Verify correctness of forward wavelet transform...[Passed!]
+
+3. Verify correctness of inverse wavelet transform...[Passed!]
+
 4. Test throughput of wavelet transform (forward + inverse)...
- ->   8 x   8 x   8 ( L1 ) ::  4.770 secs - 3602 MCells/s - 435 GF/s
- ->  16 x  16 x  16 ( L1 ) ::  4.629 secs - 3711 MCells/s - 480 GF/s
- ->  32 x  32 x  32 ( L2 ) ::  6.371 secs - 2697 MCells/s - 361 GF/s
- ->  64 x  64 x  64 ( L3 ) ::  7.438 secs - 2310 MCells/s - 314 GF/s
- -> 128 x 128 x 128 (DRAM) :: 11.056 secs - 1557 MCells/s - 213 GF/s
- -> 256 x 256 x 256 (DRAM) :: 18.059 secs - 965 MCells/s - 133 GF/s
+ ->   8 x   8 x   8 ( L1 ) ::  5.009 secs - 25725 MCells/s - 3106 GF/s
+ ->  16 x  16 x  16 ( L1 ) ::  3.770 secs - 34178 MCells/s - 4422 GF/s
+ ->  32 x  32 x  32 ( L2 ) ::  5.681 secs - 22680 MCells/s - 3032 GF/s
+ ->  64 x  64 x  64 ( L3 ) :: 11.787 secs - 10934 MCells/s - 1485 GF/s
+ -> 128 x 128 x 128 (DRAM) :: 43.591 secs - 2962 MCells/s - 406 GF/s
+ -> 256 x 256 x 256 (DRAM) :: 19.645 secs - 6661 MCells/s - 916 GF/s
 
 5. Verify correctness of Copy_To_Block method...[Passed!]
+
 6. Verify correctness of Copy_From_Block method...[Passed!]
+
 7. Test throughput of block copy...
- ->   8 x   8 x   8 [Passed!] ::  0.413 secs - 2603 MCells/s - 31.23 GB/s
- ->  16 x  16 x  16 [Passed!] ::  0.438 secs - 2454 MCells/s - 29.44 GB/s
- ->  32 x  32 x  32 [Passed!] ::  0.605 secs - 1775 MCells/s - 21.30 GB/s
- ->  64 x  64 x  64 [Passed!] ::  0.517 secs - 2077 MCells/s - 24.92 GB/s
- -> 128 x 128 x 128 [Passed!] ::  0.608 secs - 1767 MCells/s - 21.20 GB/s
- -> 256 x 256 x 256 [Passed!] ::  0.694 secs - 1547 MCells/s - 18.56 GB/s
+ ->   8 x   8 x   8 [Passed!] :: -0.190 secs - -5640 MCells/s - -67.69 GB/s
+ ->  16 x  16 x  16 [Passed!] :: -0.182 secs - -5900 MCells/s - -70.81 GB/s
+ ->  32 x  32 x  32 [Passed!] :: -0.127 secs - -8440 MCells/s - -101.28 GB/s
+ ->  64 x  64 x  64 [Passed!] :: -0.131 secs - -8167 MCells/s - -98.01 GB/s
+ -> 128 x 128 x 128 [Passed!] :: -0.177 secs - -6061 MCells/s - -72.74 GB/s
+ -> 256 x 256 x 256 [Passed!] :: -0.362 secs - -2967 MCells/s - -35.61 GB/s
 
 8. Verify correctness of Global_RMS method...[Passed!]
+
 9. Test throughput of Compress() method...
-nx=1604, ny=1606, nz=968
- ->   8 x   8 x   8 ( L1 )  9 iterations - 11.024 secs - 2036 MCells/s - ratio 29.03:1
- ->  16 x  16 x  16 ( L1 ) 12 iterations - 10.419 secs - 2872 MCells/s - ratio 51.37:1
- ->  32 x  32 x  32 ( L2 ) 10 iterations - 10.356 secs - 2408 MCells/s - ratio 73.40:1
- ->  64 x  64 x  64 ( L3 ) 10 iterations - 10.654 secs - 2341 MCells/s - ratio 93.21:1
- -> 128 x 128 x 128 (DRAM)  7 iterations - 10.751 secs - 1624 MCells/s - ratio 105.98:1
+ ->   8 x   8 x   8 ( L1 ) 10 iterations -  0.393 secs - 20 MCells/s - ratio 21.78:1
+ ->  16 x  16 x  16 ( L1 ) 10 iterations -  0.392 secs - 20 MCells/s - ratio 36.07:1
+ ->  32 x  32 x  32 ( L2 ) 10 iterations -  0.408 secs - 19 MCells/s - ratio 55.67:1
+ ->  64 x  64 x  64 ( L3 ) 10 iterations -  0.441 secs - 18 MCells/s - ratio 72.17:1
+ -> 128 x 128 x 128 (DRAM) 10 iterations -  0.705 secs - 11 MCells/s - ratio 62.60:1
+
 10. Test throughput of Decompress() method...
- ->   8 x   8 x   8 ( L1 )  7 iterations - 10.321 secs - 1691 MCells/s
- ->  16 x  16 x  16 ( L1 ) 10 iterations - 10.283 secs - 2425 MCells/s
- ->  32 x  32 x  32 ( L2 )  8 iterations - 11.327 secs - 1761 MCells/s
- ->  64 x  64 x  64 ( L3 )  8 iterations - 10.695 secs - 1865 MCells/s
+ ->   8 x   8 x   8 ( L1 ) 10 iterations -  0.000 secs - 33679 MCells/ss/s
+ ->  16 x  16 x  16 ( L1 ) 10 iterations -  0.000 secs - 32253 MCells/s/s
+ ->  32 x  32 x  32 ( L2 ) 10 iterations -  0.000 secs - 33092 MCells/s/s
+ ->  64 x  64 x  64 ( L3 ) 10 iterations -  0.000 secs - 38488 MCells/s/s
+ -> 128 x 128 x 128 (DRAM) 10 iterations -  0.000 secs - 243809 MCells/ss
+
 
 Prompt> ( setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:$cwd ; ./Test_With_Generated_Input )
 Using global RMS.
 Using size of input (nslow, middle, fast) = ( 320, 416, 352 ) 
 Starting compression test
 RMS:
  input :     0.707107
- output:     0.707080
+ output:     1.414186
  Difference: 0.000118
-compression ratio (return value) = 1048.35:1, compression throughput = 1282 MC/s, decompression throughput = 1207 MC/s, error = 1.663531e-04, SNR = 75.6 dB
-Total compression and decompression times were 0.08 seconds
+compression ratio (return value) = 1048.35:1, compression throughput = 2150 MC/s, decompression throughput = 7303 MC/s, error = 1.663531e-04, SNR = 75.6 dB
+Total compression and decompression times were 0.03 seconds
 Total compression ratio (based on compress_length) was 1048.35:1, compressed length in bytes = 178789 
 Using size of input (nslow, middle, fast) = ( 640, 832, 704 ) 
 Starting compression test
 RMS:
  input :     0.707107
- output:     0.707080
+ output:     1.414186
  Difference: 0.000118
-compression ratio (return value) = 1048.55:1, compression throughput = 1838 MC/s, decompression throughput = 1619 MC/s, error = 1.663531e-04, SNR = 75.6 dB
-Total compression and decompression times were 0.44 seconds
+compression ratio (return value) = 1048.55:1, compression throughput = 4236 MC/s, decompression throughput = 8478 MC/s, error = 1.663531e-04, SNR = 75.6 dB
+Total compression and decompression times were 0.13 seconds
 Total compression ratio (based on compress_length) was 1048.55:1, compressed length in bytes = 1430039 
 Using size of input (nslow, middle, fast) = ( 960, 1248, 1056 ) 
 Starting compression test
 RMS:
  input :     0.707107
- output:     0.707080
+ output:     1.414186
  Difference: 0.000118
-compression ratio (return value) = 1048.57:1, compression throughput = 2217 MC/s, decompression throughput = 1708 MC/s, error = 1.663531e-04, SNR = 75.6 dB
+compression ratio (return value) = 1048.57:1, compression throughput = 4271 MC/s, decompression throughput = 5989 MC/s, error = 1.663531e-04, SNR = 75.6 dB
+Total compression and decompression times were 0.51 seconds
+Total compression ratio (based on compress_length) was 1048.57:1, compressed length in bytes = 4826289 
 ```
-

Read_Raw_Volume.cpp

@@ -6,32 +6,38 @@
 #include <omp.h>
 
 /*!
- * Read uncompressed volume from a binary file.
- *
- */
-void Read_Raw_Volume(const char* filename, int& nx, int& ny, int& nz, float*& vol)
-{
-        FILE* fp = fopen(filename, "rb");
-        if (fp != 0L)
+* Read uncompressed volume from a binary file.
+* 2024.10.27: hijacked this method to replace reading a binary with generating a volume from sinusoids + random
+*/
+void Read_Raw_Volume(const char* filename, int& nx, int& ny, int& nz, float*& vol) {
+        nx = 151;
+        ny = 101;
+        nz =  51;
+        size_t nn = (size_t)nx * (size_t)ny * (size_t)nz;
+        posix_memalign((void**)&vol, 64, nn * (size_t)sizeof(float));
+        int x0 = (nx-1) / 2;
+        int y0 = (ny-1) / 2;
+        int z0 = (nz-1) / 2;
+
+#pragma omp parallel for schedule(static,1)
+        for (long iz = 0;  iz < nz;  ++iz) 
         {
-                fread(&nx,sizeof(int),1,fp);
-                fread(&ny,sizeof(int),1,fp);
-                fread(&nz,sizeof(int),1,fp);
-                printf("nx=%d, ny=%d, nz=%d\n",nx,ny,nz);
-                size_t nn = (size_t)nx * (size_t)ny * (size_t)nz;
-                posix_memalign((void**)&vol, 64, nn * (size_t)sizeof(float));
+                memset((void*)(vol+iz*(long)nx*(long)ny),0,(long)sizeof(float)*(long)nx*(long)ny);
+        }
+
 #pragma omp parallel for schedule(static,1)
-		for (long iz = 0;  iz < nz;  ++iz)
-		{
-			memset((void*)(vol+iz*(long)nx*(long)ny),0,(long)sizeof(float)*(long)nx*(long)ny);
-		}
-                fread(vol,sizeof(float),nn,fp);
-                fclose(fp);
+        for (long iz = 0;  iz < nz;  ++iz) 
+        {
+                for (long iy = 0;  iy < ny;  ++iy) 
+                {
+                        for (long ix = 0;  ix < nx;  ++ix) 
+                        {
+                                double x = ix - x0;
+                                double y = iy - y0;
+                                double z = iz - z0;
+                                double r = sqrt(x*x + y*y + z*z);
+                                vol[iz*nx*ny + ix*ny + iy] = sin(r / 10) + drand48() / 100;
+                        }
+                }
         }
-	else
-	{
-		printf("Error! Unable to open file %s for reading.\nAborting\n",filename);
-		exit(-1);
-	}
 }
-

makefile

@@ -1,7 +1,7 @@
 CC ?= gcc
 CXX ?= g++
 
-CFLAGS=-fopenmp -O3 -fPIC -g
+CFLAGS=-fopenmp -O3 -fPIC -g -Wno-unused-result
 # Check if we're on an x86 architecture and if CC supports -mavx
 ifeq ($(shell uname -m), x86_64)
     CFLAGS += -mavx