Quick Save

R. S. Doiel · R. S. Doiel · commit e1aa0c9289b3 · 2025-10-23T16:59:42.000-07:00
diff --git a/CITATION.cff b/CITATION.cff
@@ -3,7 +3,8 @@ cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 type: software
 title: articlefetch
-abstract: "A client that will submit an search to an RDM instance and retrieve the articles in the results."
+abstract: "A client that will find the person ID provided in Caltech Library's feed then retrieve the list of articles
+before fetching the PDFs from our RDM instance."
 authors:
   - family-names: Doiel
     given-names: R. S.
diff --git a/README.md b/README.md
@@ -2,8 +2,8 @@
 
 # articlefetch
 
-A client that will submit an search to an RDM instance and retrieve the articles in the results.
-
+A client that will find the person ID provided in Caltech Library's feed then retrieve the list of articles
+before fetching the PDFs from our RDM instance.
 
 
 ### Authors
diff --git a/about.md b/about.md
@@ -1,6 +1,7 @@
 ---
 title: articlefetch
-abstract: "A client that will submit an search to an RDM instance and retrieve the articles in the results."
+abstract: "A client that will find the person ID provided in Caltech Library&#x27;s feed then retrieve the list of articles
+before fetching the PDFs from our RDM instance."
 authors:
   - family_name: Doiel
     given_name: R. S.
@@ -33,7 +34,8 @@ About this software
 
 
 
-A client that will submit an search to an RDM instance and retrieve the articles in the results.
+A client that will find the person ID provided in Caltech Library&#x27;s feed then retrieve the list of articles
+before fetching the PDFs from our RDM instance.
 
 
 - GitHub: <https://github.com/caltechlibrary/articlefetch>
diff --git a/articlefetch.1.md b/articlefetch.1.md
@@ -1,4 +1,4 @@
-%articlefetch(1) user manual | version 0.0.0 0fd37b8
+%articlefetch(1) user manual | version 0.0.0 fd34fb1
 % R. S. Doiel
 % 2025-10-23
 
@@ -8,12 +8,12 @@ articlefetch
 
 # SYNOPSIS
 
-articlefetch [OPTIONS] HOSTNAME QUERY_STRING
+articlefetch [OPTIONS] RDM_HOSTNAME CLPID
 
 # DESCRIPTION
 
-Take the HOSTNAME and QUERY_STRING values, retrieve the results from RDM and then using the results
-retrieve the PDFs.
+Use the CLPID provided to retreive a list of article from feeds, then use the
+RDM_HOSTNAME to retrieve the PDFs for the articles found.
 
 # OPTIONS
 
@@ -29,7 +29,7 @@ retrieve the PDFs.
 # EXAMPLE
 
 ~~~shell
-articlefetch authors.library.caltech.edu "Grubbs, Robert"
+articlefetch authors.library.caltech.edu Grubbs-R-H
 ~~~
 
 
diff --git a/articlefetch.go b/articlefetch.go
@@ -1,9 +1,50 @@
 package articlefetch
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"strings"
+	"time"
 )
 
-func Run(in io.Reader, out io.Writer, eout io.Writer, appName string, hostname string, query string) int {
-	return 1 // DEBUG
+func Run(in io.Reader, out io.Writer, eout io.Writer, appName string, hostname string, clpid string) int {
+	// NOTE: URL encode our query string
+	feedsUrl := FeedsURL(clpid)
+	rdmIds, err := FeedsRdmIds(feedsUrl)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return 1
+	}
+	tot := len(rdmIds)
+	retrieved := 0
+	fmt.Printf("retrieving %d records\n", tot)
+	pdfToRetrieve := []string{}
+	for i, id := range rdmIds {
+		rdmUrl := RdmRecordURL(hostname, id)
+		src, duration, err := RdmFetchJSON(rdmUrl)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s\n", err)
+			continue
+		}
+		time.Sleep(duration)
+
+		pdfUrls, err := RdmPdfURLs(src)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to find a pdfUrls %q, %s\n", rdmUrl, err)
+			continue
+		}
+		if len(pdfUrls) > 0 {
+			fmt.Printf("DEBUG pdfUrls (%d)\n\t%+v\n", len(pdfUrls), strings.Join(pdfUrls, "\n\t"))
+			pdfToRetrieve = append(pdfToRetrieve, pdfUrls...)
+			time.Sleep(10 * time.Second)
+		}
+		retrieved += 1
+		if (i % 5) == 0{
+			fmt.Printf("%d/%d processed\n", i+1, tot)
+		}
+	}
+	fmt.Printf("%d/%d retrieved\n", retrieved, tot)
+	fmt.Printf("Retrieve the following URL:\n\n%s\n\n", strings.Join(pdfToRetrieve, "\n\t"))
+	return 0
 }
diff --git a/cmd/articlefetch/articlefetch.go b/cmd/articlefetch/articlefetch.go
@@ -42,9 +42,9 @@ func main() {
 
 	// handle missing hostname and query
 	if len(args) != 2 {
-		fmt.Fprintf(os.Stderr, "missing hostname or query")
+		fmt.Fprintf(os.Stderr, "missing RDM hostname or clpid")
 		os.Exit(1)
 	}
-	hostname, query := args[0], args[1]
-	os.Exit(articlefetch.Run(os.Stdin, os.Stdout, os.Stderr, appName, hostname, query))
+	hostname, clpid:= args[0], args[1]
+	os.Exit(articlefetch.Run(os.Stdin, os.Stdout, os.Stderr, appName, hostname, clpid))
 }
diff --git a/codemeta.json b/codemeta.json
@@ -14,7 +14,7 @@
   "dateCreated": "2025-10-23",
   "dateModified": "2025-10-23",
   "datePublished": "2025-10-23",
-  "description": "A client that will submit an search to an RDM instance and retrieve the articles in the results.",
+  "description": "A client that will find the person ID provided in Caltech Library's feed then retrieve the list of articles\nbefore fetching the PDFs from our RDM instance.",
   "name": "articlefetch",
   "version": "0.0.0",
   "developmentStatus": "concept",
diff --git a/helptext.go b/helptext.go
@@ -11,12 +11,12 @@ const (
 
 # SYNOPSIS
 
-{app_name} [OPTIONS] HOSTNAME QUERY_STRING
+{app_name} [OPTIONS] RDM_HOSTNAME CLPID
 
 # DESCRIPTION
 
-Take the HOSTNAME and QUERY_STRING values, retrieve the results from RDM and then using the results
-retrieve the PDFs.
+Use the CLPID provided to retreive a list of article from feeds, then use the
+RDM_HOSTNAME to retrieve the PDFs for the articles found.
 
 # OPTIONS
 
@@ -32,7 +32,7 @@ retrieve the PDFs.
 # EXAMPLE
 
 ~~~shell
-{app_name} authors.library.caltech.edu "Grubbs, Robert"
+{app_name} authors.library.caltech.edu Grubbs-R-H
 ~~~
 
 `
diff --git a/installer.ps1 b/installer.ps1
@@ -1,5 +1,5 @@
 #!/usr/bin/env pwsh
-# generated with CMTools 0.0.0 0fd37b8
+# generated with CMTools 0.0.0 fd34fb1
 
 #
 # Set the package name and version to install
diff --git a/installer.sh b/installer.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-# generated with CMTools 0.0.0 0fd37b8
+# generated with CMTools 0.0.0 fd34fb1
 
 #
 # Set the package name and version to install
diff --git a/json_handlers.go b/json_handlers.go
@@ -0,0 +1,55 @@
+package articlefetch
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+)
+
+// JSONUnmarshal is a custom JSON decoder so we can treat numbers easier
+func JSONUnmarshal(src []byte, data interface{}) error {
+	dec := json.NewDecoder(bytes.NewReader(src))
+	dec.UseNumber()
+	err := dec.Decode(&data)
+	if err != nil && err != io.EOF {
+		return err
+	}
+	return nil
+}
+
+// JSONMarshal provides provide a custom json encoder to solve a an issue with
+// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
+func JSONMarshal(data interface{}) ([]byte, error) {
+	buf1 := []byte{}
+	w1 := bytes.NewBuffer(buf1)
+	enc := json.NewEncoder(w1)
+	enc.SetIndent("", "")
+	enc.SetEscapeHTML(false)
+	err := enc.Encode(data)
+	if err != nil {
+		return nil, err
+	}
+	src1 := w1.Bytes()
+
+	// compact the record so it takes up only one line.
+	buf2 := []byte{}
+	w2 := bytes.NewBuffer(buf2)
+	err = json.Compact(w2, src1)
+	src2 := w2.Bytes()
+	return src2, err
+}
+
+// JSONMarshalIndent provides provide a custom json encoder to solve a an issue with
+// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
+func JSONMarshalIndent(data interface{}, prefix string, indent string) ([]byte, error) {
+	buf := []byte{}
+	w := bytes.NewBuffer(buf)
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent(prefix, indent)
+	err := enc.Encode(data)
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), err
+}
diff --git a/query.go b/query.go
@@ -1,12 +1,110 @@
 package articlefetch
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 )
 
-// QueryURL takes a set of query terms and returns the url with query
+// FeedsURL takes a set of query terms and returns the url with query
 // string 
-func QueryURL(hostname string, query string, sort string) string {
-	return fmt.Sprintf("https://%s/search?q=%s&l=list&p=1&s=10&sort=%s", hostname, query, sort)
+//
+//~~~
+//  feedsUrl := feedsURL("authors.library.caltech.edu", "Grubbs, Robert")
+//~~~
+func FeedsURL(clpid string) string {
+	return fmt.Sprintf("https://feeds.library.caltech.edu/people/%s/article.json", clpid)
 }
 
+// FeedsRdmIds retrieves the JSON object from Feeds
+func FeedsRdmIds(u string) ([]string, error) {
+	res, err := http.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to retrieve %s, %s", u, res.Status)
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	res.Body.Close()
+
+	objs := []map[string]interface{}{}
+	if err := json.Unmarshal(body, &objs); err != nil {
+		return nil, err
+	}
+	if len(objs) == 0 {
+		return nil, fmt.Errorf("not records returned")
+	}
+	rdmIds := []string{}
+	for _, obj := range objs {
+		if val, ok := obj["collection_id"].(string); ok {
+			rdmIds = append(rdmIds, val)
+		} else if val, ok := obj["id"].(string); ok {
+			parts := strings.Split(val, ":")
+			if len(parts) == 2 {
+				rdmIds = append(rdmIds, parts[1])
+			} else {
+				rdmIds = append(rdmIds, val)				
+			}
+		}
+	}
+	return rdmIds, nil
+}
+
+func RdmRecordURL(hostname string, id string) string {
+	return fmt.Sprintf("https://%s/api/records/%s/files", hostname, id)
+}
+
+func RdmFetchJSON(u string) ([]byte, time.Duration, error) {
+	var targetTime time.Time
+	res, err := http.Get(u)
+	tReset := res.Header.Get("X-RateLimit-Reset")
+	if tReset != "" {
+		unixTime, _ := strconv.ParseInt(tReset, 10, 64)
+		targetTime = time.Unix(unixTime, 0)
+		fmt.Fprintf(os.Stderr, "Reset will happen at %s\n", targetTime.Format(time.RFC822Z))
+	}
+	if err != nil {
+		return nil, time.Until(targetTime), err
+	}
+	duration := time.Until(targetTime)
+	if res.StatusCode != 200 {
+		return nil, duration, fmt.Errorf("failed to retrieve %s, %s", u, res.Status)
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, duration, err
+	}
+	res.Body.Close()
+	return body, duration, nil
+}
+
+func RdmPdfURLs(src []byte) ([]string, error) {
+	//fmt.Printf("DEBUG obj retrieved -> %s\n", src)
+	contentUrls := []string{}
+	obj := map[string]interface{}{}
+	if err := JSONUnmarshal(src, &obj); err != nil {
+		return nil, fmt.Errorf("failed to unmarhsal object %s\n", err)
+	}
+	if entries, ok := obj["entries"].([]map[string]interface{}); ok {
+		for _, entry := range entries {
+			if mimetype, ok := entry["mimetype"].(string); ok && mimetype == "application/pdf" {
+				if links, ok := entry["links"].(map[string]string); ok {
+					if contentUrl, ok := links["content"]; ok {
+						fmt.Printf("DEBUG contentUrl: %s\n", contentUrl)
+						contentUrls = append(contentUrls, contentUrl)
+					}
+				}
+			}
+		}
+	}
+	return contentUrls, nil
+}
diff --git a/version.go b/version.go
@@ -12,7 +12,7 @@ const (
     ReleaseDate = "2025-10-23"
 
     // ReleaseHash, the Git hash when version.go was generated
-    ReleaseHash = "0fd37b8"
+    ReleaseHash = "fd34fb1"
     LicenseText = `
 
 Copyright (c) 2025, Caltech

Original file line number	Diff line number	Diff line change
`@@ -42,9 +42,9 @@ func main() {`
`42`	`42`
`43`	`43`	`// handle missing hostname and query`
`44`	`44`	`if len(args) != 2 {`
`45`		`- fmt.Fprintf(os.Stderr, "missing hostname or query")`
	`45`	`+ fmt.Fprintf(os.Stderr, "missing RDM hostname or clpid")`
`46`	`46`	`os.Exit(1)`
`47`	`47`	`}`
`48`		`- hostname, query := args[0], args[1]`
`49`		`- os.Exit(articlefetch.Run(os.Stdin, os.Stdout, os.Stderr, appName, hostname, query))`
	`48`	`+ hostname, clpid:= args[0], args[1]`
	`49`	`+ os.Exit(articlefetch.Run(os.Stdin, os.Stdout, os.Stderr, appName, hostname, clpid))`
`50`	`50`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`#!/usr/bin/env pwsh`
`2`		`-# generated with CMTools 0.0.0 0fd37b8`
	`2`	`+# generated with CMTools 0.0.0 fd34fb1`
`3`	`3`
`4`	`4`	`#`
`5`	`5`	`# Set the package name and version to install`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`#!/bin/sh`
`2`		`-# generated with CMTools 0.0.0 0fd37b8`
	`2`	`+# generated with CMTools 0.0.0 fd34fb1`
`3`	`3`
`4`	`4`	`#`
`5`	`5`	`# Set the package name and version to install`