diff --git a/api-go/cmd/contributor/main.go b/api-go/cmd/contributor/main.go index d3e25b0..3f3ee9d 100644 --- a/api-go/cmd/contributor/main.go +++ b/api-go/cmd/contributor/main.go @@ -57,12 +57,13 @@ func main() { } func getContributor(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("Authorization") v := r.URL.Query() repo := v.Get("repo") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Access-Control-Allow-Origin", "*") - conList, code, err := contributor.GetContributorList(repo) + conList, code, err := contributor.GetContributorList(repo, token) if err != nil { w.WriteHeader(code) @@ -74,12 +75,13 @@ func getContributor(w http.ResponseWriter, r *http.Request) { } func getMonthlyContributor(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("Authorization") v := r.URL.Query() repo := v.Get("repo") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Access-Control-Allow-Origin", "*") - monthlyConLists, code, err := contributor.GetContributorMonthly(repo) + monthlyConLists, code, err := contributor.GetContributorMonthly(repo, token) if err != nil { w.WriteHeader(code) @@ -172,7 +174,7 @@ func getActivities(w http.ResponseWriter, r *http.Request) { } func refreshAll(w http.ResponseWriter, r *http.Request) { - _, code, err := gcpdb.UpdateDB("") + _, code, err := gcpdb.UpdateDB("", "") if err != nil { w.WriteHeader(code) @@ -182,7 +184,7 @@ func refreshAll(w http.ResponseWriter, r *http.Request) { } func refreshMonthly(w http.ResponseWriter, r *http.Request) { - _, code, err := contributor.GetContributorMonthly("") + _, code, err := contributor.GetContributorMonthly("", "") if err != nil { w.WriteHeader(code) diff --git a/api-go/cmd/setupDB/main.go b/api-go/cmd/setupDB/main.go deleted file mode 100644 index a66fa02..0000000 --- a/api-go/cmd/setupDB/main.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "log" - - "github.com/api7/contributor-graph/api/internal/gcpdb" -) - -func main() { - _, _, err := gcpdb.UpdateDB("") - if err != nil { - log.Fatal(err) - } -} diff --git a/api-go/internal/contributor/contributor.go b/api-go/internal/contributor/contributor.go index 1914454..104efea 100644 --- a/api-go/internal/contributor/contributor.go +++ b/api-go/internal/contributor/contributor.go @@ -19,14 +19,14 @@ import ( "github.com/api7/contributor-graph/api/internal/utils" ) -func GetContributorList(repoName string) ([]utils.ReturnCon, int, error) { +func GetContributorList(repoName string, token string) ([]utils.ReturnCon, int, error) { _, _, err := ghapi.SplitRepo(repoName) if err != nil { return nil, http.StatusNotFound, fmt.Errorf("Repo format error") } fmt.Printf("New request coming with %s\n", repoName) - returnCons, code, err := gcpdb.SingleCon(repoName) + returnCons, code, err := gcpdb.SingleCon(repoName, token) if err != nil { return nil, code, err } @@ -34,7 +34,7 @@ func GetContributorList(repoName string) ([]utils.ReturnCon, int, error) { return returnCons, http.StatusOK, nil } -func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error) { +func GetContributorMonthly(repoInput string, token string) ([]utils.MonthlyConList, int, error) { ctx := context.Background() dbCli, err := datastore.NewClient(ctx, utils.ProjectID) @@ -71,7 +71,14 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error }) if len(monthlyConLists) == 0 || monthlyConLists[len(monthlyConLists)-1].Month.AddDate(0, 2, 0).Before(time.Now()) { - ghCli := ghapi.GetGithubClient(ctx, utils.UpdateToken[i%len(utils.UpdateToken)]) + var ghToken string + if repoInput == "" { + ghToken = utils.UpdateToken[i%len(utils.UpdateToken)] + } else { + ghToken = token + } + + ghCli := ghapi.GetGithubClient(ctx, ghToken) var firstDay time.Time if len(monthlyConLists) > 0 { @@ -79,7 +86,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error } else { // get first commit of the repo and use it as the start var code int - firstDay, code, err = ghapi.GetFirstCommit(ctx, ghCli, repoName) + firstDay, code, err = ghapi.GetFirstCommit(ctx, ghCli, repoName, isSearch) if err != nil { return nil, code, err } @@ -118,7 +125,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error untilDay := sinceDay.AddDate(0, 1, 0) listCommitOpts := &github.CommitsListOptions{Since: sinceDay, Until: untilDay, ListOptions: ghapi.ListOpts} for { - commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { errCh <- utils.ErrorWithCode{err, statusCode} return diff --git a/api-go/internal/gcpdb/gcpdb.go b/api-go/internal/gcpdb/gcpdb.go index 8230be6..19bec15 100644 --- a/api-go/internal/gcpdb/gcpdb.go +++ b/api-go/internal/gcpdb/gcpdb.go @@ -22,7 +22,7 @@ import ( // if repoInput is not empty, fetch single repo and store it in db // else, use repo list to do daily update for all repos -func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { +func UpdateDB(repoInput string, token string) ([]*utils.ConList, int, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -53,7 +53,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { if repoInput == "" { ghToken = utils.UpdateToken[i%len(utils.UpdateToken)] } else { - ghToken = utils.Token + ghToken = token } ghCli := ghapi.GetGithubClient(ctx, ghToken) @@ -84,7 +84,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { // get last page lastPage := 0 listCommitOpts := &github.CommitsListOptions{Since: lastModifiedTimeDB, ListOptions: ghapi.ListOpts} - _, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + _, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } @@ -114,8 +114,8 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { return conLists, http.StatusOK, nil } -func SingleCon(repoInput string) ([]utils.ReturnCon, int, error) { - conLists, code, err := UpdateDB(repoInput) +func SingleCon(repoInput string, token string) ([]utils.ReturnCon, int, error) { + conLists, code, err := UpdateDB(repoInput, token) if err != nil { return nil, code, err } @@ -132,7 +132,7 @@ func MultiCon(repoInput string) ([]utils.ReturnCon, int, error) { conMap := make(map[string]time.Time) for _, r := range repos { - conLists, code, err := UpdateDB(r) + conLists, code, err := UpdateDB(r, "") if err != nil { return nil, code, err } @@ -204,7 +204,7 @@ func updateContributorList( // could not directly pass listCommitOpts since `Page` would be changed in different goroutine optsGoroutine := *listCommitOpts optsGoroutine.Page = i - commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, &optsGoroutine) + commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, &optsGoroutine, isSearch) if err != nil { errCh <- utils.ErrorWithCode{err, statusCode} return @@ -245,10 +245,6 @@ func updateContributorList( return commitLists[i].Date.Before(commitLists[j].Date) }) - for _, c := range commitLists { - fmt.Printf("%s %s\n", c.Date.String(), c.Author) - } - // at most write 500 entities in a single call rangeMax := 500 rangeNeeded := int(math.Ceil(float64(len(commitLists)) / float64(rangeMax))) diff --git a/api-go/internal/ghapi/ghapi.go b/api-go/internal/ghapi/ghapi.go index 6daa9f9..bfdd0d7 100644 --- a/api-go/internal/ghapi/ghapi.go +++ b/api-go/internal/ghapi/ghapi.go @@ -27,7 +27,10 @@ func SplitRepo(repo string) (string, string, error) { } func GetGithubClient(ctx context.Context, token string) *github.Client { - tc := getToken(ctx, token) + var tc *http.Client + if token != "" { + tc = getToken(ctx, token) + } return github.NewClient(tc) } @@ -57,7 +60,7 @@ func FormatCommits(ctx context.Context, comLists []*utils.ConList) ([]utils.Retu return returnCons, http.StatusOK, nil } -func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions) ([]*github.RepositoryCommit, *github.Response, int, error) { +func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions, isSearch bool) ([]*github.RepositoryCommit, *github.Response, int, error) { owner, repo, err := SplitRepo(repoName) if err != nil { return nil, nil, http.StatusBadRequest, err @@ -69,6 +72,9 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list return nil, nil, http.StatusNotFound, fmt.Errorf("Repo not found") } if _, ok := err.(*github.RateLimitError); ok || strings.Contains(err.Error(), "403 API rate limit exceeded") { + if isSearch { + return nil, nil, http.StatusForbidden, fmt.Errorf("Hit rate limit! Need Github Token to increase API quota!") + } // give it another random chance to see if magic happens *ghCli = *GetGithubClient(ctx, utils.UpdateToken[rand.Intn(len(utils.UpdateToken))]) commits, resp, err = ghCli.Repositories.ListCommits(ctx, owner, repo, listCommitOpts) @@ -83,16 +89,16 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list return commits, resp, http.StatusOK, nil } -func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string) (time.Time, int, error) { +func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string, isSearch bool) (time.Time, int, error) { listCommitOpts := &github.CommitsListOptions{} var firstCommitTime *time.Time - commits, resp, statusCode, err := GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err := GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return time.Time{}, statusCode, err } if resp.NextPage != 0 { listCommitOpts.Page = resp.LastPage - commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return time.Time{}, statusCode, err } @@ -116,7 +122,7 @@ func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string) for firstCommitTime == nil { listCommitOpts.Page = resp.PrevPage - commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return time.Time{}, statusCode, err } diff --git a/api-go/internal/graph/graph.go b/api-go/internal/graph/graph.go index d1a2738..eb79312 100644 --- a/api-go/internal/graph/graph.go +++ b/api-go/internal/graph/graph.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "strings" "cloud.google.com/go/storage" "github.com/api7/contributor-graph/api/internal/utils" @@ -32,16 +33,18 @@ func GenerateAndSaveSVG(ctx context.Context, repo string) error { return err } - wc := client.Bucket(bucket).Object(object).NewWriter(ctx) - wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} - wc.CacheControl = "public, max-age=86400" - wc.ContentType = "image/svg+xml;charset=utf-8" + if !strings.Contains(repo, ",") { + wc := client.Bucket(bucket).Object(object).NewWriter(ctx) + wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} + wc.CacheControl = "public, max-age=86400" + wc.ContentType = "image/svg+xml;charset=utf-8" - if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil { - return fmt.Errorf("upload svg failed: io.Copy: %v", err) - } - if err := wc.Close(); err != nil { - return fmt.Errorf("upload svg failed: Writer.Close: %v", err) + if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil { + return fmt.Errorf("upload svg failed: io.Copy: %v", err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("upload svg failed: Writer.Close: %v", err) + } } fmt.Printf("New SVG generated with %s\n", repo) return nil