Merge pull request #92 from P2GX/lc/obo2json

Fix the release to zenodo process

Author: leokim-l

.github/workflows/create_prompts_and_upload.yml

@@ -15,8 +15,12 @@ jobs:
         uses: actions/setup-java@v3
         with:
           java-version: '22.0.1'
-          distribution: 'temurin' 
-        
+          distribution: 'temurin'
+          cache: maven
+
+      - name: Build JAR with Maven
+        run: mvn clean package -DskipTests
+
       - name: Download latest HPO
         run: |
           java -jar target/phenopacket2prompt.jar download -w
@@ -32,19 +36,12 @@ jobs:
  
       - name: Run batch processing
         run: |
-          java -jar target/phenopacket2prompt.jar batch -d phenopackets-data/${{steps.dirout.outputs.DIR_NAME}} -p --full-translations=true
-
-      #- name: Check generated files # For debugging purposes
-      #  run: |
-      #    ls prompts/en | head
-      #    ls prompts/it | head
-      #    cat $(ls -p prompts/en | grep -v / | head -n 2 | sed 's|^|prompts/en/|')
+          java -jar target/phenopacket2prompt.jar batch -d phenopackets-data/${{steps.dirout.outputs.DIR_NAME}} -p --full-translations -j
 
-      - name: Set up zipped structure for upload
+      - name: Zip original phenopackets for upload
         run: |
-          for dir in prompts/*/; do
-            zip -r "${dir%/}.zip" "$dir" && rm -r "$dir"
-          done
+          zip -r prompts/original_phenopackets.zip prompts/original_phenopackets
+          rm -r prompts/original_phenopackets
 
       - name: Set up Python for subsequent upload to zenodo
         uses: actions/setup-python@v4
@@ -58,6 +55,8 @@ jobs:
         env:
           ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
           ZENODO_DEPOSITION_ID: ${{ secrets.ZENODO_DEPOSITION_ID }}
+          LATEST_STORE_TAG: ${{ inputs. LATEST_STORE }}
+          LATEST_HPO_TAG: ${{ inputs.LATEST_HPO }}
         run: python scripts/update_and_release_to_zenodo.py prompts/
 
 

.github/workflows/look4updates.yml

@@ -11,6 +11,11 @@ jobs:
     permissions:
       contents: read
       actions: write
+    outputs:
+      latest_store_tag: ${{ steps.check.outputs.latest_store_tag }}
+      latest_hpo_tag: ${{ steps.check.outputs.latest_hpo_tag }}
+      new_release: ${{ steps.check.outputs.new_release }}
+
 
     steps:
       - uses: actions/checkout@v4
@@ -28,6 +33,13 @@ jobs:
             GH_TOKEN: ${{ secrets.GH_PAT_LEO }}
         run: python scripts/check_ppktstore_version.py
 
-      - name: Run follow-up action
-        if: steps.check.outputs.new_release == 'true'
-        uses: ./.github/workflows/create_prompts_and_upload.yml
+  follow_up_prompt_creation:
+      needs: check_release
+      if: needs.check_release.outputs.new_release == 'true'
+      uses: ./.github/workflows/create_prompts_and_upload.yml
+      with:
+        LATEST_STORE: ${{ needs.check_release.outputs.latest_store_tag }}
+        LATEST_HPO: ${{ needs.check_release.outputs.latest_hpo_tag }}
+      secrets:
+        ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
+        ZENODO_DEPOSITION_ID: ${{ secrets.ZENODO_DEPOSITION_ID }}

.gitignore

@@ -24,6 +24,7 @@ hs_err_pid*
 replay_pid*
 /.idea/
 /data/
+target/
 /prompts/
 /fenominal-mined.txt
 /p2p_test.txt

scripts/check_ppktstore_version.py

@@ -34,9 +34,10 @@ def fetch_with_retry(url, headers, max_retries=3, timeout=10):
 
 
 ppktstore_repo = "monarch-initiative/phenopacket-store"
+hpo_repo = "obophenotype/human-phenotype-ontology"
 this_repo = os.environ["GITHUB_REPOSITORY"]
 token  = os.environ["GH_TOKEN"]
-var_name = "LAST_RUN_RELEASE"
+ppkt_last_run_version = "LAST_RUN_RELEASE"
 
 # Get phenopacket-store latest version
 latest = fetch_with_retry(
@@ -45,41 +46,54 @@ def fetch_with_retry(url, headers, max_retries=3, timeout=10):
 ).json().get("tag_name")
 
 if not latest:
-    print("Error: Could not fetch latest release tag!")
+    print("Error: Could not fetch latest phenopacket-store release tag!")
     sys.exit(1)
-    
+
+# Get HPO latest version
+latest_hpo = fetch_with_retry(
+    f"https://api.github.com/repos/{hpo_repo}/releases/latest",
+    headers={"Accept": "application/vnd.github+json"}
+).json().get("tag_name")
+
+if not latest_hpo:
+    print("Error: Could not fetch latest HPO release tag!")
+    sys.exit(1)
+
 # Get last version of phenopacket-store that ppkt2prompt ran
 r = fetch_with_retry(
-    f"https://api.github.com/repos/{this_repo}/actions/variables/{var_name}",
+    f"https://api.github.com/repos/{this_repo}/actions/variables/{ppkt_last_run_version}",
     headers={
         "Authorization": f"token {token}",
         "Accept": "application/vnd.github+json"
         }
 )
 
 if r.status_code != 200:
-    print(f"Error: Could not fetch repository variable '{var_name}'! Status code: {r.status_code}")
+    print(f"Error: Could not fetch repository variable '{ppkt_last_run_version}'! Status code: {r.status_code}")
     sys.exit(1)
 
 stored = r.json().get("value")
 if stored is None:
-    print(f"Error: Repository variable '{var_name}' returned no value.")
+    print(f"Error: Repository variable '{ppkt_last_run_version}' returned no value.")
     sys.exit(1)
 
 latest = latest.strip()
 stored = stored.strip()
+latest_hpo = latest_hpo.strip()
+
 new_release = (latest != stored)
 
 with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
-    gh_out.write(f"latest_tag={latest}\n")
     gh_out.write(f"new_release={str(new_release).lower()}\n")
+    gh_out.write(f"latest_store_tag={latest}\n")
+    gh_out.write(f"latest_hpo_tag={latest_hpo}\n")
 
 
 # Update variable if needed
 if new_release:
-    payload = {"name": var_name, "value": latest}
+    payload = {"name": ppkt_last_run_version, "value": latest}
     res = requests.patch(
-        f"https://api.github.com/repos/{this_repo}/actions/variables/{var_name}",
+        f"https://api.github.com/repos/{this_repo}/actions/variables/{ppkt_last_run_version}",
         headers={"Authorization": f"token {token}",
                  "Accept": "application/vnd.github+json"},
         json=payload,

scripts/update_and_release_to_zenodo.py

@@ -8,6 +8,8 @@
 ZENODO_API_BASE = "https://zenodo.org/api/deposit/depositions"
 ACCESS_TOKEN = os.getenv("ZENODO_ACCESS_TOKEN")
 DEPOSITION_ID = os.getenv("ZENODO_DEPOSITION_ID")
+LATEST_PPKT_STORE = os.getenv("LATEST_STORE_TAG")
+LATEST_HPO = os.getenv("LATEST_HPO_TAG")
 
 HEADERS = {"Authorization": f"Bearer {ACCESS_TOKEN}"}
 
@@ -43,7 +45,9 @@ def create_new_version():
 
     today_date = datetime.today().strftime("%Y-%m-%d")
     metadata["publication_date"] = today_date
+    metadata["notes"] = f"Used phenopacket-store version {LATEST_PPKT_STORE} and HPO version {LATEST_HPO}. Beyond this record, please also cite https://doi.org/10.1016/j.ebiom.2025.105957"
     metadata_update = {"metadata": metadata}
+    #TODO maybe add further data here about number of prompts in each language? Should be easy to do with JSONL
 
     response = requests.put(
         f"{ZENODO_API_BASE}/{new_id}",

scripts/zenodo_metadata.json

@@ -21,5 +21,6 @@
   "custom": {
     "code:codeRepository": "https://github.com/monarch-initiative/phenopacket2prompt",
     "code:programmingLanguage": [{"id": "java", "title": {"en": "Java"}}]
-  }
+  },
+  "references": "https://doi.org/10.1016/j.ebiom.2025.105957"
 }

src/main/java/org/monarchinitiative/phenopacket2prompt/cmd/DownloadCommand.java

@@ -43,11 +43,9 @@ public Integer call() throws FileDownloadException, MalformedURLException, URISy
         BioDownloaderBuilder builder = BioDownloader.builder(destination);
         builder.hpoJson();
         builder.overwrite(overwrite);
-        // TODO there might be a bug in newer hp-international.obo, revert to latest after it is fixed.
-        //String hpoInternational = "https://github.com/obophenotype/human-phenotype-ontology/releases/latest/download/hp-international.obo";
-        String hpoInternational = "https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2025-05-06/hp-international.obo";
+        String hpoInternational = "http://purl.obolibrary.org/obo/hp/translations/hp-all.babelon.json";
         URL hpoInternationalUrl  = new URI(hpoInternational).toURL() ;
-        builder.custom("hp-international.obo", hpoInternationalUrl);
+        builder.custom("hp-all.babelon.json", hpoInternationalUrl);
         BioDownloader downloader = builder.build();
         List<File> files = downloader.download();
         return 0;

src/main/java/org/monarchinitiative/phenopacket2prompt/cmd/GbtTranslateBatchCommand.java

@@ -13,6 +13,7 @@
 import picocli.CommandLine;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.Callable;
 
@@ -28,7 +29,7 @@ public class GbtTranslateBatchCommand implements Callable<Integer> {
 
     @CommandLine.Option(names = {"--translations"},
             description = "path to translations file")
-    private String translationsPath = "data/hp-international.obo";
+    private String translationsPath = "data/hp-all.babelon.json";
 
     @CommandLine.Option(names = {"-o", "--outdir"},
             description = "path to outdir")
@@ -47,6 +48,10 @@ public class GbtTranslateBatchCommand implements Callable<Integer> {
             defaultValue = "false")
     private boolean onlyPatient;
 
+    @CommandLine.Option(names = {"-j", "--jsonl-output"},
+            description = "Only output patient description")
+    private boolean jsonOutput;
+
     public boolean getPatientFlag() {
         return onlyPatient;
     }
@@ -55,10 +60,11 @@ public boolean getPatientFlag() {
 
 
     @Override
-    public Integer call() {
+    public Integer call() throws IOException {
         File hpJsonFile = new File(hpoJsonPath);
         Context.getInstance().setFullTranslations(fullTransl);
         Context.getInstance().setOnlyPatient(onlyPatient);
+        Context.getInstance().setJsonOutput(jsonOutput);
 
         boolean useExactMatching = true;
         if (! hpJsonFile.isFile()) {
@@ -142,7 +148,9 @@ public Integer call() {
             pcopy.copyFile(file);
         }
         // output file with correct diagnosis list
-        Utility.outputCorrectPPKt(correctResultList);
+        if(!Context.getInstance().isJsonOutput()) {
+            Utility.outputCorrectPPKt(correctResultList);
+        }
         return 0;
     }
 

src/main/java/org/monarchinitiative/phenopacket2prompt/cmd/TestDriveCommand.java

@@ -306,7 +306,7 @@ private static List<PpktIndividual> getPpktIndividuals() {
     }
 
     @Override
-    public Integer call() {
+    public Integer call() throws IOException {
         java.io.File hpJsonFile = new java.io.File(hpoJsonPath);
         boolean useExactMatching = true;
         if (! hpJsonFile.isFile()) {