Update formats (#16)

* add reading hd examiner peptide pool

* prepare for identify by path

* add protein filter field

* refactor loader to reader, rework formats

* remove combined schema

* fix peptide pool reader and add tests

* add kingfisher HD examiner example file

* refactor peptide pool reader

* refactor: update format handling and improve peptide loading functionality

* example of only reading files

* fix loading datasets example

* add protein field to docs

* add hd examiner peptides files

* cast exposure can also raise ValueError

* group by protein and state such that dataframes can be aggregated in batch

* fix hd examiner format identification

* aggregate replicate, cluster and charge count

* add summary function

* add uptake summary table

* add load method to formats

* fix format names

* add uptake summary converter

* renew test data with n_charges column

* expand docs on n_charges

* add function to find offset between structure and peptides

* update docstring and allow selecting mutiple columns on join

* make publication title required

* support loading from .zip files

* allow loading form dir also when there is only 1 dataset present

* delete comment

* pass exception when format is not a string

* formatting

* remove unused code from zip example

* check for stringIO type, format file

* add loading .zip test

Author: Jhsmit

docs/fields.md

@@ -13,6 +13,12 @@ residue number of the last amino acid in the peptide
 ### sequence (str)
 fasta sequence of the peptide
 
+### protein (str)
+protein name or identifier
+
+HDExaminer name: Protein
+DynamX name: Protein
+
 ### state (str)
 state label
 
@@ -93,6 +99,9 @@ These fields are derived from other fields defined in the above sections.
 added after data aggregation
 Total number of replicates that were aggregated together
 
+### n_charges
+Total number of different charged states that were aggregated together
+
 ### n_clusters
 added after data aggregation
 Total number of isotopic clusters that were aggregated together. When replicates include multiple isotopic clusters (different charged states), this value will be larger than n_replicates.

docs/hd_examiner_files/HDX export file test.csv

@@ -159,6 +159,25 @@ FD control: 'MAX' (older version)
 Comments:
 
 
+### Kingfisher HD examiner example
+
+File: HDX export file test.csv
+Source: https://github.com/juan2089/Kingfisher-HDX/blob/Kingfisher-v1.1/www/HDX%20export%20file%20test.csv
+
+Columns:
+The first line is a header with exposure times. 
+
+The second line has the column names, starting with:
+'State,Protein,Start,End,Sequence,Search RT,Charge,Max D,'
+
+Followed by repeating blocks of:
+'Start RT,End RT,#D,%D,#D right,%D right,Score,Conf,'
+Format: (almost!) HD examiner summary file
+
+This is a HD examiner 'peptide pool' file
+
+
+
 ## HD Examiner manual on exporting data
 
 **Peptide Pool Results / Uptake Summary Table**

docs/hd_examiner_formats.md

@@ -4,7 +4,7 @@
 from typing import Optional
 
 from hdxms_datasets.database import populate_known_ids, submit_dataset
-from hdxms_datasets.loader import (
+from hdxms_datasets.reader import (
     read_hxms,
 )
 from hdxms_datasets.models import (

examples/from_hxms_file.py

@@ -0,0 +1,17 @@
+from hdxms_datasets import load_dataset
+from pathlib import Path
+
+DATA_ID = "HDX_C1198C76"  # SecA DynamX state data
+DATA_ID = "HDX_D9096080"  # SecB DynamX state data
+
+fname = "HDX_3BAE2080.zip"  # Example dataset in a zip file
+
+# %%
+test_pth = Path(__file__).parent.parent / "tests"
+database_dir = test_pth / "datasets"
+
+dataset = load_dataset(database_dir / fname)  # Should load the dataset from the zip file
+
+print(dataset.states)
+
+# %%

examples/from_zip_file.py

@@ -53,7 +53,7 @@
 plot_peptides(selected, domain=(0, 1), value="frac_max_uptake")
 
 # %%
-peptides = dataset.states[0].peptides[0]
+peptides = dataset.states[0].peptides[0].load()
 StructureView(dataset.structure).peptide_coverage(peptides)
 
 # %%

examples/load_local_dynamx_cluster.py

@@ -44,9 +44,9 @@
 # load the partially deuterated peptides
 df = state.peptides[0].load(
     convert=True,
-    aggregate=True,
-    # sort_rows=True,
-    # sort_columns=True,
+    aggregate=None,  # dynamx state data is already aggregated
+    sort_rows=True,
+    sort_columns=True,
 )
 print(df.columns)
 # > ['start', 'end', 'sequence', 'state', 'exposure', 'centroid_mz', 'rt', 'rt_sd', 'uptake', 'uptake_sd']
@@ -112,12 +112,12 @@
 # %%
 # show a single peptide
 start, end = processed["start", "end"].row(10)
-view = StructureView(dataset.structure).color_peptide(start, end, chain=["A"])
+view = StructureView(dataset.structure).color_peptide(start, end)
 view
 
 # %%
 # select a set of peptides for further viusualization
-peptides = dataset.states[0].peptides[0]
+peptides = dataset.states[0].peptides[0].load()
 
 # %%
 # show regions of the structure that are covered by peptides

examples/load_local_dynamx_state.py

@@ -35,7 +35,6 @@
 selected = processed.filter(nw.col("exposure") == exposure_value)
 plot_peptides(selected.to_polars(), value="frac_max_uptake", domain=(0, 1))
 # %%
-# %%
 
 peptides = dataset.states[0].peptides[0]
 StructureView(dataset.structure).peptide_coverage(selected)

examples/load_local_hdexaminer.py

@@ -0,0 +1,34 @@
+# %%
+
+from pathlib import Path
+
+from hdxms_datasets import identify_format
+
+# %%
+
+cwd = Path(__file__).parent
+
+# %%
+
+# read a hxms file
+f = cwd / "test_data" / "ecDHFR" / "ecDHFR_2025-09-23_APO.hxms"
+
+fmt_spec = identify_format(f)
+# read to dataframe
+df = fmt_spec.read(f)
+
+# convert to open-hdx format
+df_converted = fmt_spec.convert(df)
+df_converted.to_native()
+
+# %%
+# read an dynamx file
+f = cwd / "test_data" / "ecSecB" / "ecSecB_apo.csv"
+fmt_spec = identify_format(f)
+# read to dataframe
+df = fmt_spec.read(f)
+
+# convert to open-hdx format
+df_converted = fmt_spec.convert(df)
+df_converted.to_native()
+# %%

examples/read_files.py

@@ -4,3 +4,7 @@ the HXMS file format manuscript.
 Correct source: 
 https://www.biorxiv.org/content/10.1101/2025.10.14.682397v1.supplementary-material
 
+
+
+ecDHFR tutorial.csv
+Source: https://huggingface.co/spaces/glasgow-lab/PFLink