Skip to content

feat: support token #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat: support token #31

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
68e58cc
feat: support token
Yiyiyimu Apr 20, 2021
b166821
chore: merge master
Yiyiyimu Apr 21, 2021
a7c3f7a
fix: use header to get token, to comply with FE
Yiyiyimu Apr 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions api-go/cmd/contributor/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ func main() {
func getContributor(w http.ResponseWriter, r *http.Request) {
v := r.URL.Query()
repo := v.Get("repo")
token := v.Get("token")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Access-Control-Allow-Origin", "*")

conList, code, err := contributor.GetContributorList(repo)
conList, code, err := contributor.GetContributorList(repo, token)

if err != nil {
w.WriteHeader(code)
Expand Down Expand Up @@ -172,7 +173,7 @@ func getActivities(w http.ResponseWriter, r *http.Request) {
}

func refreshAll(w http.ResponseWriter, r *http.Request) {
_, code, err := gcpdb.UpdateDB("")
_, code, err := gcpdb.UpdateDB("", "")

if err != nil {
w.WriteHeader(code)
Expand Down
14 changes: 0 additions & 14 deletions api-go/cmd/setupDB/main.go
View file
Open in desktop

This file was deleted.

15 changes: 9 additions & 6 deletions api-go/internal/contributor/contributor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ import (
"github.com/api7/contributor-graph/api/internal/utils"
)

func GetContributorList(repoName string) ([]utils.ReturnCon, int, error) {
func GetContributorList(repoName string, token string) ([]utils.ReturnCon, int, error) {
_, _, err := ghapi.SplitRepo(repoName)
if err != nil {
return nil, http.StatusNotFound, fmt.Errorf("Repo format error")
}

fmt.Printf("New request coming with %s\n", repoName)
returnCons, code, err := gcpdb.SingleCon(repoName)
returnCons, code, err := gcpdb.SingleCon(repoName, token)
if err != nil {
return nil, code, err
}
Expand All @@ -42,12 +42,15 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error
defer dbCli.Close()

var repos []string
var isSearch bool
if repoInput == "" {
isSearch = false
repos, err = getUpdateRepoList(ctx, dbCli)
if err != nil {
return nil, http.StatusInternalServerError, err
}
} else {
isSearch = true
repos = []string{strings.ToLower(repoInput)}
}

Expand All @@ -74,13 +77,13 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error
// get first commit of the repo and use it as the start
listCommitOpts := &github.CommitsListOptions{}
var firstCommitTime *time.Time
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
if resp.NextPage != 0 {
listCommitOpts.Page = resp.LastPage
commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
Expand All @@ -104,7 +107,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error

for firstCommitTime == nil {
listCommitOpts.Page = resp.PrevPage
commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
Expand Down Expand Up @@ -136,7 +139,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error
comLists := make(map[string]bool)
listCommitOpts := &github.CommitsListOptions{Since: firstDay, Until: firstDay.AddDate(0, 1, 0), ListOptions: ghapi.ListOpts}
for {
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
Expand Down
20 changes: 12 additions & 8 deletions api-go/internal/gcpdb/gcpdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

// if repoInput is not empty, fetch single repo and store it in db
// else, use repo list to do daily update for all repos
func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
func UpdateDB(repoInput string, token string) ([]*utils.ConList, int, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

Expand All @@ -47,10 +47,13 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
conLists = []*utils.ConList{}

var ghToken string
var isSearch bool
if repoInput == "" {
ghToken = utils.UpdateToken[i%len(utils.UpdateToken)]
isSearch = false
} else {
ghToken = utils.Token
ghToken = token
isSearch = true
}
ghCli := ghapi.GetGithubClient(ctx, ghToken)

Expand Down Expand Up @@ -81,15 +84,15 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
// get last page
lastPage := 0
listCommitOpts := &github.CommitsListOptions{Since: lastModifiedTimeDB, ListOptions: ghapi.ListOpts}
_, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
_, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
if resp.LastPage != 0 {
lastPage = resp.LastPage
}

newConLists, code, err := updateContributorList(ctx, dbCli, ghCli, conMap, repoName, lastPage, listCommitOpts)
newConLists, code, err := updateContributorList(ctx, dbCli, ghCli, conMap, repoName, lastPage, listCommitOpts, isSearch)
if err != nil {
return nil, code, err
}
Expand All @@ -111,8 +114,8 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
return conLists, http.StatusOK, nil
}

func SingleCon(repoInput string) ([]utils.ReturnCon, int, error) {
conLists, code, err := UpdateDB(repoInput)
func SingleCon(repoInput string, token string) ([]utils.ReturnCon, int, error) {
conLists, code, err := UpdateDB(repoInput, token)
if err != nil {
return nil, code, err
}
Expand All @@ -129,7 +132,7 @@ func MultiCon(repoInput string) ([]utils.ReturnCon, int, error) {
conMap := make(map[string]time.Time)

for _, r := range repos {
conLists, code, err := UpdateDB(r)
conLists, code, err := UpdateDB(r, "")
if err != nil {
return nil, code, err
}
Expand Down Expand Up @@ -174,13 +177,14 @@ func updateContributorList(
repoName string,
lastPage int,
listCommitOpts *github.CommitsListOptions,
isSearch bool,
) ([]*utils.ConList, int, error) {
bar := progressbar.Default(int64(lastPage + 1))

var commitLists []*utils.ConList
for i := lastPage; i >= 0; i-- {
listCommitOpts.Page = i
commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
Expand Down
10 changes: 8 additions & 2 deletions api-go/internal/ghapi/ghapi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ func SplitRepo(repo string) (string, string, error) {
}

func GetGithubClient(ctx context.Context, token string) *github.Client {
tc := getToken(ctx, token)
var tc *http.Client
if token != "" {
tc = getToken(ctx, token)
}

return github.NewClient(tc)
}
Expand Down Expand Up @@ -58,7 +61,7 @@ func FormatCommits(ctx context.Context, comLists []*utils.ConList) ([]utils.Retu
}

// TODO: support goroutine
func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions) ([]*github.RepositoryCommit, *github.Response, int, error) {
func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions, isSearch bool) ([]*github.RepositoryCommit, *github.Response, int, error) {
owner, repo, err := SplitRepo(repoName)
if err != nil {
return nil, nil, http.StatusBadRequest, err
Expand All @@ -70,6 +73,9 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list
return nil, nil, http.StatusNotFound, fmt.Errorf("Repo not found")
}
if _, ok := err.(*github.RateLimitError); ok || strings.Contains(err.Error(), "403 API rate limit exceeded") {
if isSearch {
return nil, nil, http.StatusForbidden, fmt.Errorf("Hit rate limit! Need Github Token to increase API quota!")
}
// give it another random chance to see if magic happens
*ghCli = *GetGithubClient(ctx, utils.UpdateToken[rand.Intn(len(utils.UpdateToken))])
commits, resp, err = ghCli.Repositories.ListCommits(ctx, owner, repo, listCommitOpts)
Expand Down
21 changes: 12 additions & 9 deletions api-go/internal/graph/graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"io/ioutil"
"net/http"
"strings"

"cloud.google.com/go/storage"
"github.com/api7/contributor-graph/api/internal/utils"
Expand All @@ -32,16 +33,18 @@ func GenerateAndSaveSVG(ctx context.Context, repo string) error {
return err
}

wc := client.Bucket(bucket).Object(object).NewWriter(ctx)
wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
wc.CacheControl = "public, max-age=86400"
wc.ContentType = "image/svg+xml;charset=utf-8"
if !strings.Contains(repo, ",") {
wc := client.Bucket(bucket).Object(object).NewWriter(ctx)
wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
wc.CacheControl = "public, max-age=86400"
wc.ContentType = "image/svg+xml;charset=utf-8"

if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil {
return fmt.Errorf("upload svg failed: io.Copy: %v", err)
}
if err := wc.Close(); err != nil {
return fmt.Errorf("upload svg failed: Writer.Close: %v", err)
if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil {
return fmt.Errorf("upload svg failed: io.Copy: %v", err)
}
if err := wc.Close(); err != nil {
return fmt.Errorf("upload svg failed: Writer.Close: %v", err)
}
}
fmt.Printf("New SVG generated with %s\n", repo)
return nil
Expand Down