Release v0.2 (#21)

* Created reading functions that are lax on corrupt records. They return `([recs], int|None)`, where the second element is the byte where the corrupted records start, or None if no records are corrupted. These functions have the same python signature, but are named `read_dmap_lax`, `read_iqdat_lax`, etc.
* Used macro_rules to reduce similar code blocks.
* Renamed `GenericRecord` to `DmapRecord`, and moved into dmap.rs file.
* Moved dmap.rs file containing `Record` trait to src/record.rs
* Added a few unit tests to types.rs
* Added new functions to read from / write to bytes via Python API.
* Fleshed out README.md with a developer guide to explain the crate structure and what the files roughly contain.
* Version number bump
* Run all OS builds in CI for PR
* Updated CI.yml with new `maturin-ci generate` command.

Author: RemingtonRohel

.github/workflows/CI.yml

@@ -1,4 +1,4 @@
-# This file is autogenerated by maturin v1.7.0
+# This file is autogenerated by maturin v1.9.0
 # To update, run
 #
 #    maturin generate-ci github --zig
@@ -25,18 +25,18 @@ jobs:
     strategy:
       matrix:
         platform:
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: x86_64
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: x86
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: aarch64
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: armv7
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: s390x
-#          - runner: ubuntu-latest
-#            target: ppc64le
+    #          - runner: ubuntu-22.04
+    #            target: ppc64le
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -51,6 +51,15 @@ jobs:
           manylinux: auto
           before-script-linux: |
             sudo apt update -y && sudo apt-get install -y libssl-dev openssl pkg-config
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist --zig -i python3.13t
+#          sccache: 'false'
+#          manylinux: auto
+#          before-script-linux: |
+#            sudo apt update -y && sudo apt-get install -y libssl-dev openssl pkg-config
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -59,17 +68,16 @@ jobs:
 
   musllinux:
     runs-on: ${{ matrix.platform.runner }}
-    if: "startsWith(github.ref, 'refs/tags/')"
     strategy:
       matrix:
         platform:
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: x86_64
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: x86
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: aarch64
-          - runner: ubuntu-latest
+          - runner: ubuntu-22.04
             target: armv7
     steps:
       - uses: actions/checkout@v4
@@ -83,6 +91,15 @@ jobs:
           args: --release --out dist
           sccache: 'false'
           manylinux: musllinux_1_2
+          before-script-linux: |
+            sudo apt update -y && sudo apt-get install -y libssl-dev openssl pkg-config
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'false'
+#          manylinux: musllinux_1_2
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -91,12 +108,13 @@ jobs:
 
   windows:
     runs-on: ${{ matrix.platform.runner }}
-    if: "startsWith(github.ref, 'refs/tags/')"
     strategy:
       matrix:
         platform:
           - runner: windows-latest
             target: x64
+          - runner: windows-latest
+            target: x86
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -109,6 +127,16 @@ jobs:
           target: ${{ matrix.platform.target }}
           args: --release --out dist
           sccache: 'false'
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.13t
+          architecture: ${{ matrix.platform.target }}
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'false'
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -117,11 +145,10 @@ jobs:
 
   macos:
     runs-on: ${{ matrix.platform.runner }}
-    if: "startsWith(github.ref, 'refs/tags/')"
     strategy:
       matrix:
         platform:
-          - runner: macos-12
+          - runner: macos-13
             target: x86_64
           - runner: macos-14
             target: aarch64
@@ -136,6 +163,12 @@ jobs:
           target: ${{ matrix.platform.target }}
           args: --release --out dist
           sccache: 'false'
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'false'
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -146,9 +179,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.x
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
@@ -163,18 +193,26 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    if: "startsWith(github.ref, 'refs/tags/')"
+    if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
     needs: [linux, musllinux, windows, macos, sdist]
     permissions:
+      # Use to sign the release artifacts
       id-token: write
-    environment: release
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
     steps:
       - uses: actions/download-artifact@v4
-      - uses: actions/setup-python@v5
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v2
         with:
-          python-version: 3.x
+          subject-path: 'wheels-*/*'
       - name: Publish to PyPI
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
         uses: PyO3/maturin-action@v1
+        #        env:
+        #          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
         with:
           command: upload
           args: --non-interactive --skip-existing wheels-*/*

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dmap"
-version = "0.1.6"
+version = "0.2.0"
 edition = "2021"
 rust-version = "1.63.0"
 
@@ -22,6 +22,7 @@ thiserror = "1.0.63"
 zerocopy = { version = "0.7.35", features = ["byteorder"] }
 lazy_static = "1.5.0"
 bzip2 = "0.4.4"
+paste = "1.0.15"
 
 [dev-dependencies]
 criterion = { version = "0.4", features = ["html_reports"] }

README.md

@@ -1,3 +1,49 @@
 # Dmap
 
 Rust tools for SuperDARN DMAP file format operations.
+
+This project exposes both Rust and Python APIs for handling DMAP I/O.
+I/O can be conducted either on byte buffers, or directly to/from files.
+
+The SuperDARN DMAP file formats are all supported (IQDAT, RAWACF, FITACF, GRID, MAP, and SND)
+as well as a generic DMAP format that is unaware of any required fields or types 
+(e.g. char, int32) for any fields.
+
+## Developer Guidelines
+
+### `src/record.rs`
+This file contains the `Record` trait, which defines a set of functions that specific DMAP formats must implement.
+For example, `read_file(infile: &PathBuf) -> Result<Vec<Self>, DmapError>` is defined in the `Record` trait, and handles
+reading in records from a file at the specified path. This function is generic, in that it doesn't know what type of records
+(RAWACF, FITACF, etc.) are expected. Also, since it is a trait function, you can only use it through a struct which implements
+the trait. For example, the `FitacfRecord` struct defined in `src/formats/fitacf.rs` implements the `Record` trait, and so
+you can call `FitacfRecord::read_file(...)` to read a FITACF file, but you couldn't invoke `Record::read_file(...)`.
+
+### `src/types.rs`
+This file defines necessary structs and enums for encapsulating basic types (`i8`, `u32`, `String`, etc.) into
+objects like `DmapField`, `DmapScalar`, `DmapVec`, etc. that abstract over the supported underlying types.
+For instance, when reading scalar from a DMAP file, the underlying data type is inferred from the `type` field in the 
+scalar's metadata, so it can't be known beforehand. This requires some encapsulating type, `DmapScalar` in this case,
+which contains the metadata of the field and has a known size for the stack memory. 
+
+This file defines the `Fields` struct, which is used to hold the names and types of the required and optional
+scalar and vector fields for a type of DMAP record (RAWACF, FITACF, etc.).
+
+This file defines the `DmapType` trait and implements it for supported data types that can be in DMAP records, namely
+`u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64`, `f32`, `f64`, and `String`. The implementation of the trait for
+these types only means that other types, e.g. `i128`, cannot be stored in DMAP records.
+
+Lastly, functions for parsing scalars and vectors from a byte buffer are defined in this file.
+
+### `src/formats`
+This directory holds the files that define the DMAP record formats: IQDAT, RAWACF, FITACF, GRID, MAP, SND, and the generic DMAP.
+If you are defining a new DMAP format, you will need to make a new file in this directory following the structure of the
+existing files. Essentially, you define the scalar and vector fields, both required and optional, and the groups of vector
+fields which must have identical dimensions, then call a macro to autogenerate the struct code for you. 
+
+### `tests`
+In `tests.rs`, integration tests for reading and writing all file types are present. Small example files
+are contained in `tests/test_files`.
+
+### `benches/io_benchmarking.rs`
+This file contains benchmarking functions for checking the performance of the basic read functions.

benches/io_benchmarking.rs

@@ -1,11 +1,12 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use dmap::formats::dmap::Record;
 use dmap::formats::fitacf::FitacfRecord;
 use dmap::formats::grid::GridRecord;
 use dmap::formats::iqdat::IqdatRecord;
 use dmap::formats::map::MapRecord;
 use dmap::formats::rawacf::RawacfRecord;
 use dmap::formats::snd::SndRecord;
+use dmap::record::Record;
+use paste::paste;
 use std::fs::File;
 
 fn criterion_benchmark(c: &mut Criterion) {
@@ -15,12 +16,6 @@ fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("Read GRID", |b| b.iter(|| read_grid()));
     c.bench_function("Read SND", |b| b.iter(|| read_snd()));
     c.bench_function("Read MAP", |b| b.iter(|| read_map()));
-    // c.bench_function("Read Full-size RAWACF", |b| {
-    //     b.iter(|| read_fullsize_rawacf())
-    // });
-    // c.bench_function("Read Full-size FITACF", |b| {
-    //     b.iter(|| read_fullsize_fitacf())
-    // });
 
     // let records = read_iqdat();
     // c.bench_with_input(
@@ -30,52 +25,34 @@ fn criterion_benchmark(c: &mut Criterion) {
     // );
 }
 
-fn read_fitacf() -> Vec<FitacfRecord> {
-    let file = File::open("tests/test_files/test.fitacf").expect("Test file not found");
-    FitacfRecord::read_records(file).unwrap()
+/// Generates benchmark functions for a given DMAP record type.
+macro_rules! read_type {
+    ($type:ident) => {
+        paste! {
+            fn [< read_ $type >]() -> Vec<[< $type:camel Record >]> {
+                let file = File::open(format!("tests/test_files/test.{}", stringify!($type))).expect("Test file not found");
+                [< $type:camel Record >]::read_records(file).unwrap()
+            }
+        }
+    }
 }
 
-fn read_rawacf() -> Vec<RawacfRecord> {
-    let file = File::open("tests/test_files/test.rawacf").expect("Test file not found");
-    RawacfRecord::read_records(file).unwrap()
-}
-
-fn read_fullsize_rawacf() -> Vec<RawacfRecord> {
-    let file = File::open("tests/test_files/20210607.1801.00.cly.a.rawacf.mean")
-        .expect("Test file not found");
-    RawacfRecord::read_records(file).unwrap()
-}
-
-fn read_fullsize_fitacf() -> Vec<FitacfRecord> {
-    let file =
-        File::open("tests/test_files/20210607.1801.00.cly.a.fitacf").expect("Test file not found");
-    FitacfRecord::read_records(file).unwrap()
-}
-
-fn read_iqdat() -> Vec<IqdatRecord> {
-    let file = File::open("tests/test_files/test.iqdat").expect("Test file not found");
-    IqdatRecord::read_records(file).unwrap()
-}
+read_type!(iqdat);
+read_type!(rawacf);
+read_type!(fitacf);
+read_type!(grid);
+read_type!(map);
+read_type!(snd);
 
 // fn write_iqdat(records: &Vec<RawDmapRecord>) {
 //     let file = File::open("tests/test_files/test.iqdat").expect("Test file not found");
 //     dmap::read_records(file).unwrap();
 //     dmap::to_file("tests/test_files/temp.iqdat", records).unwrap();
 // }
 
-fn read_grid() -> Vec<GridRecord> {
-    let file = File::open("tests/test_files/test.grid").expect("Test file not found");
-    GridRecord::read_records(file).unwrap()
-}
-
-fn read_map() -> Vec<MapRecord> {
-    let file = File::open("tests/test_files/test.map").expect("Test file not found");
-    MapRecord::read_records(file).unwrap()
-}
-
-fn read_snd() -> Vec<SndRecord> {
-    let file = File::open("tests/test_files/test.snd").expect("Test file not found");
-    SndRecord::read_records(file).unwrap()
+criterion_group! {
+    name = benches;
+    config = Criterion::default();
+    targets = criterion_benchmark
 }
-criterion_group!(benches, criterion_benchmark);
 criterion_main!(benches);

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "darn-dmap"
-version = "0.1.6"
+version = "0.2.0"
 requires-python = ">=3.8"
 authors = [
     { name = "Remington Rohel" }

src/error.rs

@@ -33,7 +33,7 @@ pub enum DmapError {
 
     /// Errors when reading in multiple records
     #[error("First error: {1}\nRecords with errors: {0:?}")]
-    BadRecords(Vec<usize>, String)
+    BadRecords(Vec<usize>, String),
 }
 
 impl From<DmapError> for PyErr {