Skip to content

Commit f50b1ab

Browse files
tardyptardyp
committed
use github api to find github tags
svn version is kept if no GH_TOKEN is given in environ, but this method has a lot of issues (proxy support, scalability) Signed-off-by: Pierre Tardy <[email protected]>
1 parent a0a733a commit f50b1ab

12 files changed

+270
-10
lines changed

‎vulnerabilities/package_managers.py‎

Lines changed: 86 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
# for any legal advice.
2020
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
import os
2223
import asyncio
2324
import dataclasses
2425
import xml.etree.ElementTree as ET
@@ -30,6 +31,7 @@
3031
from typing import Set
3132

3233
from aiohttp import ClientSession
34+
import aiohttp
3335
from aiohttp.client_exceptions import ClientResponseError
3436
from aiohttp.client_exceptions import ServerDisconnectedError
3537
from bs4 import BeautifulSoup
@@ -48,6 +50,10 @@ class VersionResponse:
4850
newer_versions: Set[str] = dataclasses.field(default_factory=set)
4951

5052

53+
class GraphQLError(Exception):
54+
pass
55+
56+
5157
class VersionAPI:
5258
def __init__(self, cache: Mapping[str, Set[Version]] = None):
5359
self.cache = cache or {}
@@ -377,21 +383,92 @@ def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
377383
class GitHubTagsAPI(VersionAPI):
378384

379385
package_type = "github"
386+
GQL_QUERY = """
387+
query getTags($name: String!, $owner: String!, $after: String)
388+
{
389+
repository(name: $name, owner: $owner) {
390+
refs(refPrefix: "refs/tags/", first: 100, after: $after) {
391+
totalCount
392+
pageInfo {
393+
endCursor
394+
hasNextPage
395+
}
396+
nodes {
397+
name
398+
target {
399+
... on Commit {
400+
committedDate
401+
}
402+
... on Tag {
403+
target {
404+
... on Commit {
405+
committedDate
406+
}
407+
}
408+
}
409+
}
410+
}
411+
}
412+
}
413+
}"""
380414

381-
async def fetch(self, owner_repo: str, session) -> None:
415+
def __init__(self, cache: Mapping[str, Set[Version]] = None):
416+
self.gh_token = os.getenv("GH_TOKEN")
417+
super().__init__(cache=cache)
418+
419+
async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
382420
"""
383421
owner_repo is a string of format "{repo_owner}/{repo_name}"
384422
Example value of owner_repo = "nexB/scancode-toolkit"
385423
"""
386424
self.cache[owner_repo] = set()
387-
endpoint = f"https://github.com/{owner_repo}"
388-
389-
tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
390-
elements = ET.fromstring(tags_xml)
391-
for entry in elements.iter("entry"):
392-
name = entry.find("name").text
393-
release_date = dateparser.parse(entry.find("commit/date").text)
394-
self.cache[owner_repo].add(Version(value=name, release_date=release_date))
425+
if self.gh_token:
426+
# graphql api cannot work without api token
427+
session.headers["Authorization"] = "token " + self.gh_token
428+
endpoint = f"https://api.github.com/graphql"
429+
owner, name = owner_repo.split("/")
430+
query = {"query": self.GQL_QUERY, "variables": {"name": name, "owner": owner}}
431+
432+
while True:
433+
response = await session.post(endpoint, json=query)
434+
resp_json = await response.json()
435+
436+
if "errors" in resp_json:
437+
raise GraphQLError(resp_json["errors"])
438+
439+
refs = resp_json["data"]["repository"]["refs"]
440+
441+
for entry in refs["nodes"]:
442+
name = entry["name"]
443+
target = entry["target"]
444+
# in case the tag is a signed tag, then the commit info is in target['target']
445+
if "committedDate" not in target:
446+
target = target["target"]
447+
if "committedDate" in target:
448+
release_date = dateparser.parse(target["committedDate"])
449+
else:
450+
# but tags can actually point to tree and not commit, so there is no date
451+
# probably this only happened for linux. Github cannot even properly display it.
452+
# https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/refs/tags/v2.6.11
453+
release_date = None
454+
self.cache[owner_repo].add(Version(value=name, release_date=release_date))
455+
456+
if not refs["pageInfo"]["hasNextPage"]:
457+
break
458+
# to fetch next page, we just set the after variable to endCursor
459+
query["variables"]["after"] = refs["pageInfo"]["endCursor"]
460+
461+
else:
462+
# In case we don't have GH_TOKEN, we use the svn ls method to get the tags
463+
# It allows to get all the information needed in one request without any rate limiting
464+
# this method is however not scalable for larger repo and the api is unresponsive
465+
# for repo with > 50 tags
466+
endpoint = f"https://github.com/{owner_repo}"
467+
tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
468+
elements = ET.fromstring(tags_xml)
469+
for entry in elements.iter("entry"):
470+
name = entry.find("name").text
471+
release_date = dateparser.parse(entry.find("commit/date").text)
395472

396473

397474
class HexVersionAPI(VersionAPI):

‎vulnerabilities/tests/test_data/github_api/release_response.json‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
[
2+
{
3+
"url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782",
4+
"assets_url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782/assets",
5+
"upload_url": "https://uploads.github.com/repos/nexB/vulnerablecode/releases/32748782/assets{?name,label}",
6+
"html_url": "https://github.com/nexB/vulnerablecode/releases/tag/v20.10",
7+
"id": 32748782,
8+
"author": {
9+
"login": "pombredanne",
10+
"id": 675997,
11+
"node_id": "MDQ6VXNlcjY3NTk5Nw==",
12+
"avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
13+
"gravatar_id": "",
14+
"url": "https://api.github.com/users/pombredanne",
15+
"html_url": "https://github.com/pombredanne",
16+
"followers_url": "https://api.github.com/users/pombredanne/followers",
17+
"following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
18+
"gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
19+
"starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
20+
"subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
21+
"organizations_url": "https://api.github.com/users/pombredanne/orgs",
22+
"repos_url": "https://api.github.com/users/pombredanne/repos",
23+
"events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
24+
"received_events_url": "https://api.github.com/users/pombredanne/received_events",
25+
"type": "User",
26+
"site_admin": false
27+
},
28+
"node_id": "MDc6UmVsZWFzZTMyNzQ4Nzgy",
29+
"tag_name": "v20.10",
30+
"target_commitish": "main",
31+
"name": "v20.10",
32+
"draft": false,
33+
"prerelease": false,
34+
"created_at": "2020-09-28T12:31:16Z",
35+
"published_at": "2020-10-19T10:46:17Z",
36+
"assets": [
37+
{
38+
"url": "https://api.github.com/repos/nexB/vulnerablecode/releases/assets/27230021",
39+
"id": 27230021,
40+
"node_id": "MDEyOlJlbGVhc2VBc3NldDI3MjMwMDIx",
41+
"name": "vulnerablecode-2020-10-19.json.xz",
42+
"label": null,
43+
"uploader": {
44+
"login": "pombredanne",
45+
"id": 675997,
46+
"node_id": "MDQ6VXNlcjY3NTk5Nw==",
47+
"avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
48+
"gravatar_id": "",
49+
"url": "https://api.github.com/users/pombredanne",
50+
"html_url": "https://github.com/pombredanne",
51+
"followers_url": "https://api.github.com/users/pombredanne/followers",
52+
"following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
53+
"gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
54+
"starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
55+
"subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
56+
"organizations_url": "https://api.github.com/users/pombredanne/orgs",
57+
"repos_url": "https://api.github.com/users/pombredanne/repos",
58+
"events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
59+
"received_events_url": "https://api.github.com/users/pombredanne/received_events",
60+
"type": "User",
61+
"site_admin": false
62+
},
63+
"content_type": "application/x-xz",
64+
"state": "uploaded",
65+
"size": 13603356,
66+
"download_count": 20,
67+
"created_at": "2020-10-20T09:40:08Z",
68+
"updated_at": "2020-10-20T09:40:25Z",
69+
"browser_download_url": "https://github.com/nexB/vulnerablecode/releases/download/v20.10/vulnerablecode-2020-10-19.json.xz"
70+
}
71+
],
72+
"tarball_url": "https://api.github.com/repos/nexB/vulnerablecode/tarball/v20.10",
73+
"zipball_url": "https://api.github.com/repos/nexB/vulnerablecode/zipball/v20.10",
74+
"body": "This release comes with the new calver versioning scheme and an initial data dump.\r\n\r\nTo load the JSON data attached here:\r\n- extract it with `unxz vulnerablecode-2020-10-19.json.xz`\r\n- run `DJANGO_DEV=1 python manage.py loaddata vulnerablecode-2020-10-19.json`\r\n\r\nThe data import is not optimized yet and takes a long time."
75+
}
76+
]

‎vulnerabilities/tests/test_data/records/nexB_vulnerablecode_1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"data":{"repository":{"refs":{"totalCount":2,"pageInfo":{"endCursor":"Mg","hasNextPage":false},"nodes":[{"name":"v0.1","target":{"committedDate":"2019-12-03T13:48:53Z"}},{"name":"v20.10","target":{"committedDate":"2020-09-28T12:31:16Z"}}]}}}}

‎vulnerabilities/tests/test_data/records/torvalds_linux_1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"data":{"repository":{"refs":{"totalCount":713,"pageInfo":{"endCursor":"MTAw","hasNextPage":true},"nodes":[{"name":"v2.6.11-tree","target":{"target":{}}},{"name":"v2.6.11","target":{"target":{}}},{"name":"v2.6.12-rc2","target":{"target":{"committedDate":"2005-04-16T22:20:36Z"}}},{"name":"v2.6.12-rc3","target":{"target":{"committedDate":"2005-04-20T23:24:21Z"}}},{"name":"v2.6.12-rc4","target":{"target":{"committedDate":"2005-05-07T05:20:31Z"}}},{"name":"v2.6.12-rc5","target":{"target":{"committedDate":"2005-05-25T03:31:20Z"}}},{"name":"v2.6.12-rc6","target":{"target":{"committedDate":"2005-06-06T15:22:29Z"}}},{"name":"v2.6.12","target":{"target":{"committedDate":"2005-06-17T19:48:29Z"}}},{"name":"v2.6.13-rc1","target":{"target":{"committedDate":"2005-06-29T05:57:29Z"}}},{"name":"v2.6.13-rc2","target":{"target":{"committedDate":"2005-07-06T03:46:33Z"}}},{"name":"v2.6.13-rc3","target":{"target":{"committedDate":"2005-07-13T04:46:46Z"}}},{"name":"v2.6.13-rc4","target":{"target":{"committedDate":"2005-07-28T22:44:44Z"}}},{"name":"v2.6.13-rc5","target":{"target":{"committedDate":"2005-08-02T04:45:48Z"}}},{"name":"v2.6.13-rc6","target":{"target":{"committedDate":"2005-08-07T18:18:56Z"}}},{"name":"v2.6.13-rc7","target":{"target":{"committedDate":"2005-08-24T03:39:14Z"}}},{"name":"v2.6.13","target":{"target":{"committedDate":"2005-08-28T23:41:01Z"}}},{"name":"v2.6.14-rc1","target":{"target":{"committedDate":"2005-09-13T03:12:09Z"}}},{"name":"v2.6.14-rc2","target":{"target":{"committedDate":"2005-09-20T03:00:41Z"}}},{"name":"v2.6.14-rc3","target":{"target":{"committedDate":"2005-09-30T21:17:35Z"}}},{"name":"v2.6.14-rc4","target":{"target":{"committedDate":"2005-10-11T01:19:19Z"}}},{"name":"v2.6.14-rc5","target":{"target":{"committedDate":"2005-10-20T06:23:05Z"}}},{"name":"v2.6.14","target":{"target":{"committedDate":"2005-10-28T00:02:08Z"}}},{"name":"v2.6.15-rc1","target":{"target":{"committedDate":"2005-11-12T01:43:36Z"}}},{"name":"v2.6.15-rc2","target":{"target":{"committedDate":"2005-11-20T03:25:03Z"}}},{"name":"v2.6.15-rc3","target":{"target":{"committedDate":"2005-11-29T03:51:27Z"}}},{"name":"v2.6.15-rc4","target":{"target":{"committedDate":"2005-12-01T06:25:15Z"}}},{"name":"v2.6.15-rc5","target":{"target":{"committedDate":"2005-12-04T05:10:42Z"}}},{"name":"v2.6.15-rc6","target":{"target":{"committedDate":"2005-12-19T00:36:54Z"}}},{"name":"v2.6.15-rc7","target":{"target":{"committedDate":"2005-12-24T23:47:48Z"}}},{"name":"v2.6.15","target":{"target":{"committedDate":"2006-01-03T03:21:10Z"}}},{"name":"v2.6.16-rc1","target":{"target":{"committedDate":"2006-01-17T07:44:47Z"}}},{"name":"v2.6.16-rc2","target":{"target":{"committedDate":"2006-02-03T06:03:08Z"}}},{"name":"v2.6.16-rc3","target":{"target":{"committedDate":"2006-02-13T00:27:25Z"}}},{"name":"v2.6.16-rc4","target":{"target":{"committedDate":"2006-02-17T22:23:45Z"}}},{"name":"v2.6.16-rc5","target":{"target":{"committedDate":"2006-02-27T05:09:35Z"}}},{"name":"v2.6.16-rc6","target":{"target":{"committedDate":"2006-03-11T22:12:55Z"}}},{"name":"v2.6.16","target":{"target":{"committedDate":"2006-03-20T05:53:29Z"}}},{"name":"v2.6.17-rc1","target":{"target":{"committedDate":"2006-04-03T03:22:10Z"}}},{"name":"v2.6.17-rc2","target":{"target":{"committedDate":"2006-04-19T03:00:49Z"}}},{"name":"v2.6.17-rc3","target":{"target":{"committedDate":"2006-04-27T02:19:25Z"}}},{"name":"v2.6.17-rc4","target":{"target":{"committedDate":"2006-05-11T23:31:53Z"}}},{"name":"v2.6.17-rc5","target":{"target":{"committedDate":"2006-05-25T01:50:17Z"}}},{"name":"v2.6.17-rc6","target":{"target":{"committedDate":"2006-06-06T00:57:02Z"}}},{"name":"v2.6.17","target":{"target":{"committedDate":"2006-06-18T01:49:35Z"}}},{"name":"v2.6.18-rc1","target":{"target":{"committedDate":"2006-07-06T04:09:49Z"}}},{"name":"v2.6.18-rc2","target":{"target":{"committedDate":"2006-07-15T21:53:08Z"}}},{"name":"v2.6.18-rc3","target":{"target":{"committedDate":"2006-07-30T06:15:36Z"}}},{"name":"v2.6.18-rc4","target":{"target":{"committedDate":"2006-08-06T18:20:11Z"}}},{"name":"v2.6.18-rc5","target":{"target":{"committedDate":"2006-08-28T03:41:48Z"}}},{"name":"v2.6.18-rc6","target":{"target":{"committedDate":"2006-09-04T02:19:48Z"}}},{"name":"v2.6.18-rc7","target":{"target":{"committedDate":"2006-09-13T01:41:36Z"}}},{"name":"v2.6.18","target":{"target":{"committedDate":"2006-09-20T03:42:06Z"}}},{"name":"v2.6.19-rc1","target":{"target":{"committedDate":"2006-10-05T02:57:05Z"}}},{"name":"v2.6.19-rc2","target":{"target":{"committedDate":"2006-10-13T16:25:04Z"}}},{"name":"v2.6.19-rc3","target":{"target":{"committedDate":"2006-10-23T23:02:02Z"}}},{"name":"v2.6.19-rc4","target":{"target":{"committedDate":"2006-10-31T03:37:36Z"}}},{"name":"v2.6.19-rc5","target":{"target":{"committedDate":"2006-11-08T02:24:20Z"}}},{"name":"v2.6.19-rc6","target":{"target":{"committedDate":"2006-11-16T04:03:40Z"}}},{"name":"v2.6.19","target":{"target":{"committedDate":"2006-11-29T21:57:37Z"}}},{"name":"v2.6.20-rc1","target":{"target":{"committedDate":"2006-12-14T01:14:23Z"}}},{"name":"v2.6.20-rc2","target":{"target":{"committedDate":"2006-12-24T04:00:32Z"}}},{"name":"v2.6.20-rc3","target":{"target":{"committedDate":"2007-01-01T00:53:20Z"}}},{"name":"v2.6.20-rc4","target":{"target":{"committedDate":"2007-01-07T05:45:51Z"}}},{"name":"v2.6.20-rc5","target":{"target":{"committedDate":"2007-01-12T18:54:26Z"}}},{"name":"v2.6.20-rc6","target":{"target":{"committedDate":"2007-01-25T02:19:28Z"}}},{"name":"v2.6.20-rc7","target":{"target":{"committedDate":"2007-01-31T03:42:57Z"}}},{"name":"v2.6.20","target":{"target":{"committedDate":"2007-02-04T18:44:54Z"}}},{"name":"v2.6.21-rc1","target":{"target":{"committedDate":"2007-02-21T04:32:30Z"}}},{"name":"v2.6.21-rc2","target":{"target":{"committedDate":"2007-02-28T04:59:12Z"}}},{"name":"v2.6.21-rc3","target":{"target":{"committedDate":"2007-03-07T04:41:20Z"}}},{"name":"v2.6.21-rc4","target":{"target":{"committedDate":"2007-03-16T00:20:01Z"}}},{"name":"v2.6.21-rc5","target":{"target":{"committedDate":"2007-03-25T22:56:23Z"}}},{"name":"v2.6.21-rc6","target":{"target":{"committedDate":"2007-04-06T02:36:56Z"}}},{"name":"v2.6.21-rc7","target":{"target":{"committedDate":"2007-04-15T23:50:57Z"}}},{"name":"v2.6.21","target":{"target":{"committedDate":"2007-04-26T03:08:32Z"}}},{"name":"v2.6.22-rc1","target":{"target":{"committedDate":"2007-05-13T01:45:56Z"}}},{"name":"v2.6.22-rc2","target":{"target":{"committedDate":"2007-05-19T04:06:17Z"}}},{"name":"v2.6.22-rc3","target":{"target":{"committedDate":"2007-05-26T02:55:14Z"}}},{"name":"v2.6.22-rc4","target":{"target":{"committedDate":"2007-06-05T00:57:25Z"}}},{"name":"v2.6.22-rc5","target":{"target":{"committedDate":"2007-06-17T02:09:12Z"}}},{"name":"v2.6.22-rc6","target":{"target":{"committedDate":"2007-06-24T23:21:48Z"}}},{"name":"v2.6.22-rc7","target":{"target":{"committedDate":"2007-07-01T19:54:24Z"}}},{"name":"v2.6.22","target":{"target":{"committedDate":"2007-07-08T23:32:17Z"}}},{"name":"v2.6.23-rc1","target":{"target":{"committedDate":"2007-07-22T20:41:00Z"}}},{"name":"v2.6.23-rc2","target":{"target":{"committedDate":"2007-08-04T02:49:55Z"}}},{"name":"v2.6.23-rc3","target":{"target":{"committedDate":"2007-08-13T04:25:24Z"}}},{"name":"v2.6.23-rc4","target":{"target":{"committedDate":"2007-08-28T01:32:35Z"}}},{"name":"v2.6.23-rc5","target":{"target":{"committedDate":"2007-09-01T06:08:24Z"}}},{"name":"v2.6.23-rc6","target":{"target":{"committedDate":"2007-09-11T02:50:29Z"}}},{"name":"v2.6.23-rc7","target":{"target":{"committedDate":"2007-09-19T23:01:13Z"}}},{"name":"v2.6.23-rc8","target":{"target":{"committedDate":"2007-09-25T00:33:10Z"}}},{"name":"v2.6.23-rc9","target":{"target":{"committedDate":"2007-10-02T03:24:52Z"}}},{"name":"v2.6.23","target":{"target":{"committedDate":"2007-10-09T20:31:38Z"}}},{"name":"v2.6.24-rc1","target":{"target":{"committedDate":"2007-10-24T03:50:57Z"}}},{"name":"v2.6.24-rc2","target":{"target":{"committedDate":"2007-11-06T21:57:46Z"}}},{"name":"v2.6.24-rc3","target":{"target":{"committedDate":"2007-11-17T05:16:36Z"}}},{"name":"v2.6.24-rc4","target":{"target":{"committedDate":"2007-12-04T04:26:10Z"}}},{"name":"v2.6.24-rc5","target":{"target":{"committedDate":"2007-12-11T03:48:43Z"}}},{"name":"v2.6.24-rc6","target":{"target":{"committedDate":"2007-12-21T01:25:48Z"}}},{"name":"v2.6.24-rc7","target":{"target":{"committedDate":"2008-01-06T21:45:38Z"}}}]}}}}

0 commit comments

Comments
 (0)