working proof of concept

Author: R. S. Doiel

articlefetch.1.md

@@ -1,4 +1,4 @@
-%articlefetch(1) user manual | version 0.0.0 fd34fb1
+%articlefetch(1) user manual | version 0.0.0 e1aa0c9
 % R. S. Doiel
 % 2025-10-23
 

articlefetch.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"
+	"path/filepath"
+	//"strings"
 	"time"
 )
 
@@ -17,34 +18,56 @@ func Run(in io.Reader, out io.Writer, eout io.Writer, appName string, hostname s
 		return 1
 	}
 	tot := len(rdmIds)
+	t0 := time.Now()
+	iTime := time.Now()
+	reportProgress := false
 	retrieved := 0
-	fmt.Printf("retrieving %d records\n", tot)
-	pdfToRetrieve := []string{}
+	waitTime := 5 * time.Second
+	fmt.Printf("processing %d records\n", tot)
 	for i, id := range rdmIds {
+		if i > 0 {
+			if iTime, reportProgress = CheckWaitInterval(iTime, waitTime); reportProgress {
+				fmt.Printf("%s | next id %q\n", ProgressETA(t0, i, tot), id)
+				time.Sleep(waitTime)
+			}
+		}
 		rdmUrl := RdmRecordURL(hostname, id)
-		src, duration, err := RdmFetchJSON(rdmUrl)
+		src, err := RdmFetchJSON(rdmUrl)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s\n", err)
 			continue
 		}
-		time.Sleep(duration)
 
 		pdfUrls, err := RdmPdfURLs(src)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to find a pdfUrls %q, %s\n", rdmUrl, err)
 			continue
 		}
 		if len(pdfUrls) > 0 {
-			fmt.Printf("DEBUG pdfUrls (%d)\n\t%+v\n", len(pdfUrls), strings.Join(pdfUrls, "\n\t"))
-			pdfToRetrieve = append(pdfToRetrieve, pdfUrls...)
-			time.Sleep(10 * time.Second)
+			// Make a directory for {clpid}/{rdmid}
+			saveDir := filepath.Join(clpid, id)
+			if _, err := os.Stat(saveDir); err != nil {
+				os.MkdirAll(saveDir, 0775)
+			}
+			// For each PDF create a directory for the RDM record id
+			for i, pdfUrl := range pdfUrls {
+				fName, err := RdmGetFilenameFromContentURL(pdfUrl)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to extract filename (file no. %d for %s) %s, %s\n", i, id, pdfUrl, err)
+				}
+				// Retrieve and write out the PDF to dir
+				if src, err := RdmRetrieveFile(pdfUrl); err != nil {
+					fmt.Fprintf(os.Stderr, "failed to retrieve file %s, %s\n", pdfUrl, err)
+				} else {
+					fName = filepath.Join(saveDir, fName)
+					if err := os.WriteFile(fName, src, 0664); err != nil {
+						fmt.Fprintf(os.Stderr, "failed to write %s, %s\n", fName, err)
+					}
+				}
+			}
 		}
 		retrieved += 1
-		if (i % 5) == 0{
-			fmt.Printf("%d/%d processed\n", i+1, tot)
-		}
 	}
-	fmt.Printf("%d/%d retrieved\n", retrieved, tot)
-	fmt.Printf("Retrieve the following URL:\n\n%s\n\n", strings.Join(pdfToRetrieve, "\n\t"))
+	fmt.Printf("%d of %d records processed\n", retrieved, tot)
 	return 0
 }

installer.ps1

@@ -1,5 +1,5 @@
 #!/usr/bin/env pwsh
-# generated with CMTools 0.0.0 fd34fb1
+# generated with CMTools 0.0.0 e1aa0c9
 
 #
 # Set the package name and version to install

installer.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# generated with CMTools 0.0.0 fd34fb1
+# generated with CMTools 0.0.0 e1aa0c9
 
 #
 # Set the package name and version to install

progress.go

@@ -0,0 +1,84 @@
+package articlefetch
+
+import (
+    "fmt"
+    "time"
+)
+
+// CheckWaitInterval checks to see if an interval of time has been met or exceeded.
+// It returns the remaining time interval (possibly reset) and a boolean. The
+// boolean is true when the time interval has been met or exceeded, false otherwise.
+//
+// ```
+// tot := len(something) // calculate the total number of items to process
+// t0 := time.Now()
+// iTime := time.Now()
+// reportProgress := false
+//
+// for i, key := range records {
+//     // ... process stuff ...
+//     if iTime, reportProgress = CheckWaitInterval(rptTime, (30 * time.Second)); reportProgress {
+//         log.Printf("%s", ProgressETA(t0, i, tot))
+//     }
+// }
+//
+// ```
+func CheckWaitInterval(iTime time.Time, wait time.Duration) (time.Time, bool) {
+    if time.Since(iTime) >= wait {
+        iTime = time.Now()
+        return iTime, true
+    }
+    return iTime, false
+}
+
+// ProgressETA returns a string with the percentage processed and estimated time remaining.
+// It requires the a counter of records processed, the total count of records and a time zero value.
+//
+// ```
+// tot := len(something) // calculate the total number of items to process
+// t0 := time.Now()
+// iTime := time.Now()
+// reportProgress := false
+//
+// for i, key := range records {
+//     // ... process stuff ...
+//     if iTime, reportProgress = CheckWaitInterval(rptTime, (30 * time.Second)); reportProgress {
+//         log.Printf("%s", ProgressETA(t0, i, tot))
+//     }
+// }
+//
+// ```
+func ProgressETA(t0 time.Time, i int, tot int) string {
+    if i == 0 {
+        return fmt.Sprintf("%.2f%% ETA unknown", 0.0)
+    }
+
+    percent := (float64(i) / float64(tot)) * 100.0
+    elapsedT := float64(time.Since(t0))
+	eta := (time.Duration((float64(tot) * (float64(elapsedT) / float64(i))) - elapsedT))
+    return fmt.Sprintf("%.2f%% ETA %v", percent, eta.Round(time.Second))
+}
+
+// ProgressIPS returns a string with the elapsed time and increments per second.
+// Takes a time zero, a counter and time unit. Returns a string with count, running time and
+// increments per time unit.
+// ```
+// t0 := time.Now()
+// iTime := time.Now()
+// reportProgress := false
+//
+// for i, key := range records {
+//     // ... process stuff ...
+//     if iTime, reportProgress = CheckWaitInterval(iTime, (30 * time.Second)); reportProgress || i = 0 {
+//         log.Printf("%s", ProgressIPS(t0, i, time.Second))
+//     }
+// }
+//
+// ```
+func ProgressIPS(t0 time.Time, i int, timeUnit time.Duration) string {
+    if i == 0 {
+        return fmt.Sprintf("(%d/%s) IPS unknown", i, time.Since(t0).Round(timeUnit))
+    }
+    ips := float64(i) / float64(time.Since(t0).Seconds())
+    return fmt.Sprintf("(%d/%s) IPS %.2f i/sec.", i, time.Since(t0).Round(timeUnit), ips)
+}

query.go

@@ -5,10 +5,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
-	"strconv"
+	"net/url"
+	"path"
 	"strings"
-	"time"
 )
 
 // FeedsURL takes a set of query terms and returns the url with query
@@ -63,48 +62,67 @@ func RdmRecordURL(hostname string, id string) string {
 	return fmt.Sprintf("https://%s/api/records/%s/files", hostname, id)
 }
 
-func RdmFetchJSON(u string) ([]byte, time.Duration, error) {
-	var targetTime time.Time
+func RdmFetchJSON(u string) ([]byte, error) {
 	res, err := http.Get(u)
-	tReset := res.Header.Get("X-RateLimit-Reset")
-	if tReset != "" {
-		unixTime, _ := strconv.ParseInt(tReset, 10, 64)
-		targetTime = time.Unix(unixTime, 0)
-		fmt.Fprintf(os.Stderr, "Reset will happen at %s\n", targetTime.Format(time.RFC822Z))
-	}
 	if err != nil {
-		return nil, time.Until(targetTime), err
-	}
-	duration := time.Until(targetTime)
-	if res.StatusCode != 200 {
-		return nil, duration, fmt.Errorf("failed to retrieve %s, %s", u, res.Status)
+		return nil, err
 	}
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
-		return nil, duration, err
+		return nil, err
 	}
 	res.Body.Close()
-	return body, duration, nil
+	return body, nil
 }
 
 func RdmPdfURLs(src []byte) ([]string, error) {
-	//fmt.Printf("DEBUG obj retrieved -> %s\n", src)
 	contentUrls := []string{}
 	obj := map[string]interface{}{}
 	if err := JSONUnmarshal(src, &obj); err != nil {
 		return nil, fmt.Errorf("failed to unmarhsal object %s\n", err)
 	}
-	if entries, ok := obj["entries"].([]map[string]interface{}); ok {
-		for _, entry := range entries {
+	// DEBUG
+	//src, _ = JSONMarshalIndent(obj, "", "    ") // DEBUG
+	//fmt.Printf("DEBUG obj retrieved -> %s\n", src) // DEBUG
+	if entries, ok := obj["entries"].([]interface{}); ok {
+		//fmt.Printf("DEBUG entries (%T): %+v\n", entries)
+		for _, val := range entries {
+			entry := val.(map[string]interface{})
+			//src, _ = JSONMarshalIndent(entry, "", "    ") // DEBUG
+			//fmt.Printf("DEBUG obj entries -> %s\n", src) // DEBUG
 			if mimetype, ok := entry["mimetype"].(string); ok && mimetype == "application/pdf" {
-				if links, ok := entry["links"].(map[string]string); ok {
-					if contentUrl, ok := links["content"]; ok {
-						fmt.Printf("DEBUG contentUrl: %s\n", contentUrl)
+				//fmt.Printf("DEBUG entry.mimetype -> %s\n", mimetype) // DEBUG
+				if links, ok := entry["links"].(map[string]interface{}); ok {
+					//src, _ = JSONMarshalIndent(links, "", "    ") // DEBUG
+					//fmt.Printf("DEBUG links %s\n", src) // DEBUG
+					if contentUrl, ok := links["content"].(string); ok {
+						//fmt.Printf("DEBUG contentUrl: %s\n", contentUrl)
 						contentUrls = append(contentUrls, contentUrl)
 					}
 				}
 			}
 		}
 	}
 	return contentUrls, nil
+}
+
+func RdmGetFilenameFromContentURL(s string) (string, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return "", err
+	}
+	return path.Base(strings.TrimSuffix(u.Path, "/content")), nil
+}
+
+func RdmRetrieveFile(u string) ([]byte, error) {
+	res, err := http.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	res.Body.Close()
+	return body, nil
 }

version.go

@@ -12,7 +12,7 @@ const (
     ReleaseDate = "2025-10-23"
 
     // ReleaseHash, the Git hash when version.go was generated
-    ReleaseHash = "fd34fb1"
+    ReleaseHash = "e1aa0c9"
     LicenseText = `
 
 Copyright (c) 2025, Caltech