switch to github.com/google/go-github/github

With this change, the GITHUB_USERNAME, GITHUB_PASSWORD and GITHUB_OTP
environment variables are introduced. Authenticating to GitHub is useful to
obtain a higher rate limit.

Furthermore, an in-memory cache is now used to reduce the number of requests
sent to GitHub in the first place.

fixes https://bugs.debian.org/898397

Author: stapelberg

description.go

@@ -2,22 +2,14 @@ package main
 
 import (
 	"bytes"
-	"encoding/base64"
-	"encoding/json"
+	"context"
 	"fmt"
-	"net/http"
 	"os/exec"
 	"strings"
 
 	"github.com/russross/blackfriday"
 )
 
-type readmeReply struct {
-	Content  string `json:"content"`
-	Encoding string `json:"encoding"`
-	Name     string `json:"name"`
-}
-
 func reformatForControl(raw string) string {
 	// Reformat the wrapped description to conform to Debian’s control format.
 	for strings.Contains(raw, "\n\n") {
@@ -30,26 +22,18 @@ func getLongDescriptionForGopkg(gopkg string) (string, error) {
 	if !strings.HasPrefix(gopkg, "github.com/") {
 		return "", nil
 	}
-	resp, err := http.Get("https://api.github.com/repos/" + gopkg[len("github.com/"):] + "/readme")
-	if err != nil {
-		return "", err
+	parts := strings.Split(strings.TrimPrefix(gopkg, "github.com/"), "/")
+	if got, want := len(parts), 2; got != want {
+		return "", fmt.Errorf("invalid GitHub repo: %q does not follow github.com/owner/repo", gopkg)
 	}
-	defer resp.Body.Close()
+	owner, repo := parts[0], parts[1]
 
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("unexpected HTTP status: got %d, want %d", resp.StatusCode, http.StatusOK)
-	}
-
-	var rr readmeReply
-	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
+	rr, _, err := gitHub.Repositories.GetReadme(context.TODO(), owner, repo, nil)
+	if err != nil {
 		return "", err
 	}
 
-	if rr.Encoding != "base64" {
-		return "", fmt.Errorf("unexpected encoding: got %q, want %q", rr.Encoding, "base64")
-	}
-
-	content, err := base64.StdEncoding.DecodeString(rr.Content)
+	content, err := rr.GetContent()
 	if err != nil {
 		return "", err
 	}
@@ -61,14 +45,14 @@ func getLongDescriptionForGopkg(gopkg string) (string, error) {
 	// fairly involved, but it’d be the most correct solution to the problem at
 	// hand. Our current code just knows markdown, which is good enough since
 	// most (Go?) projects in fact use markdown for their README files.
-	if !strings.HasSuffix(rr.Name, "md") &&
-		!strings.HasSuffix(rr.Name, "markdown") &&
-		!strings.HasSuffix(rr.Name, "mdown") &&
-		!strings.HasSuffix(rr.Name, "mkdn") {
+	if !strings.HasSuffix(rr.GetName(), "md") &&
+		!strings.HasSuffix(rr.GetName(), "markdown") &&
+		!strings.HasSuffix(rr.GetName(), "mdown") &&
+		!strings.HasSuffix(rr.GetName(), "mkdn") {
 		return reformatForControl(strings.TrimSpace(string(content))), nil
 	}
 
-	output := blackfriday.Markdown(content, &TextRenderer{}, 0)
+	output := blackfriday.Markdown([]byte(content), &TextRenderer{}, 0)
 	// Shell out to fmt(1) to line-wrap the output.
 	cmd := exec.Command("fmt")
 	cmd.Stdin = bytes.NewBuffer(output)

main.go

@@ -3,6 +3,13 @@ package main
 import (
 	"fmt"
 	"os"
+
+	"github.com/google/go-github/github"
+	"github.com/gregjones/httpcache"
+)
+
+var (
+	gitHub *github.Client
 )
 
 func usage() {
@@ -19,6 +26,14 @@ func usage() {
 }
 
 func main() {
+	transport := github.BasicAuthTransport{
+		Username:  os.Getenv("GITHUB_USERNAME"),
+		Password:  os.Getenv("GITHUB_PASSWORD"),
+		OTP:       os.Getenv("GITHUB_OTP"),
+		Transport: httpcache.NewMemoryCacheTransport(),
+	}
+	gitHub = github.NewClient(transport.Client())
+
 	// Retrieve args and Shift binary name off argument list.
 	args := os.Args[1:]
 

metadata.go

@@ -1,44 +1,17 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"golang.org/x/net/html"
 )
 
-type licensesReply struct {
-	Key      string `json:"key"`
-	Name     string `json:"name"`
-	URL      string `json:"url"`
-	Featured bool   `json:"featured"`
-}
-
-type ownerReply struct {
-	URL string `json:"url"`
-}
-
-type repositoryReply struct {
-	License     licensesReply `json:"license"`
-	CreatedAt   string        `json:"created_at"`
-	Description string        `json:"description"`
-	Owner       ownerReply    `json:"owner"`
-}
-
-type licenseReply struct {
-	Body string `json:"body"`
-}
-
-type usersReply struct {
-	Name string `json:"name"`
-}
-
 // To update, use:
-// curl -s -H 'Accept: application/vnd.github.drax-preview+json' https://api.github.com/licenses | jq '.[].key'
+// curl -s https://api.github.com/licenses | jq '.[].key'
 // then compare with https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-specification
 var githubLicenseToDebianLicense = map[string]string{
 	//"agpl-3.0" (not in debian?)
@@ -85,7 +58,7 @@ var debianLicenseText = map[string]string{
 
 var githubRegexp = regexp.MustCompile(`github\.com/([^/]+/[^/]+)`)
 
-func findGitHubRepo(gopkg string) (string, error) {
+func findGitHubOwnerRepo(gopkg string) (string, error) {
 	if strings.HasPrefix(gopkg, "github.com/") {
 		return strings.TrimPrefix(gopkg, "github.com/"), nil
 	}
@@ -136,27 +109,30 @@ func findGitHubRepo(gopkg string) (string, error) {
 	}
 }
 
-func getLicenseForGopkg(gopkg string) (string, string, error) {
-	repo, err := findGitHubRepo(gopkg)
+func findGitHubRepo(gopkg string) (owner string, repo string, _ error) {
+	ownerrepo, err := findGitHubOwnerRepo(gopkg)
 	if err != nil {
 		return "", "", err
 	}
-	// TODO: cache this reply
-	req, err := http.NewRequest("GET", "https://api.github.com/repos/"+repo, nil)
-	if err != nil {
-		return "", "", err
+	parts := strings.Split(ownerrepo, "/")
+	if got, want := len(parts), 2; got != want {
+		return "", "", fmt.Errorf("invalid GitHub repo: %q does not follow owner/repo", repo)
 	}
-	req.Header.Set("Accept", "application/vnd.github.drax-preview+json")
-	resp, err := http.DefaultClient.Do(req)
+	return parts[0], parts[1], nil
+}
+
+func getLicenseForGopkg(gopkg string) (string, string, error) {
+	owner, repo, err := findGitHubRepo(gopkg)
 	if err != nil {
 		return "", "", err
 	}
-	defer resp.Body.Close()
-	var r repositoryReply
-	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+
+	rl, _, err := gitHub.Repositories.License(context.TODO(), owner, repo)
+	if err != nil {
 		return "", "", err
 	}
-	if deblicense, ok := githubLicenseToDebianLicense[r.License.Key]; ok {
+
+	if deblicense, ok := githubLicenseToDebianLicense[rl.GetLicense().GetKey()]; ok {
 		fulltext := debianLicenseText[deblicense]
 		if fulltext == "" {
 			fulltext = "TODO"
@@ -168,74 +144,53 @@ func getLicenseForGopkg(gopkg string) (string, string, error) {
 }
 
 func getAuthorAndCopyrightForGopkg(gopkg string) (string, string, error) {
-	repo, err := findGitHubRepo(gopkg)
-	if err != nil {
-		return "", "", err
-	}
-	resp, err := http.Get("https://api.github.com/repos/" + repo)
+	owner, repo, err := findGitHubRepo(gopkg)
 	if err != nil {
 		return "", "", err
 	}
-	defer resp.Body.Close()
-
-	var rr repositoryReply
-	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
-		return "", "", err
-	}
 
-	creation, err := time.Parse("2006-01-02T15:04:05Z", rr.CreatedAt)
+	rr, _, err := gitHub.Repositories.Get(context.TODO(), owner, repo)
 	if err != nil {
 		return "", "", err
 	}
 
-	if strings.TrimSpace(rr.Owner.URL) == "" {
+	if strings.TrimSpace(rr.GetOwner().GetURL()) == "" {
 		return "", "", fmt.Errorf("Repository owner URL not present in API response")
 	}
 
-	resp, err = http.Get(rr.Owner.URL)
+	ur, _, err := gitHub.Users.Get(context.TODO(), rr.GetOwner().GetLogin())
 	if err != nil {
 		return "", "", err
 	}
-	defer resp.Body.Close()
 
-	var ur usersReply
-	if err := json.NewDecoder(resp.Body).Decode(&ur); err != nil {
-		return "", "", err
-	}
-
-	copyright := creation.Format("2006") + " " + ur.Name
+	copyright := rr.CreatedAt.Format("2006") + " " + ur.GetName()
 	if strings.HasPrefix(repo, "google/") {
 		// As per https://opensource.google.com/docs/creating/, Google retains
 		// the copyright for repositories underneath github.com/google/.
-		copyright = creation.Format("2006") + " Google Inc."
+		copyright = rr.CreatedAt.Format("2006") + " Google Inc."
 	}
 
-	return ur.Name, copyright, nil
+	return ur.GetName(), copyright, nil
 }
 
 func getDescriptionForGopkg(gopkg string) (string, error) {
-	repo, err := findGitHubRepo(gopkg)
+	owner, repo, err := findGitHubRepo(gopkg)
 	if err != nil {
 		return "", err
 	}
-	resp, err := http.Get("https://api.github.com/repos/" + repo)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
 
-	var rr repositoryReply
-	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
+	rr, _, err := gitHub.Repositories.Get(context.TODO(), owner, repo)
+	if err != nil {
 		return "", err
 	}
 
-	return strings.TrimSpace(rr.Description), nil
+	return strings.TrimSpace(rr.GetDescription()), nil
 }
 
 func getHomepageForGopkg(gopkg string) string {
-	repo, err := findGitHubRepo(gopkg)
+	owner, repo, err := findGitHubRepo(gopkg)
 	if err != nil {
 		return "TODO"
 	}
-	return "https://github.com/" + repo
+	return "https://github.com/" + owner + "/" + repo
 }