Skip to content

Commit dcbe42d

Browse files
G-RathG-Rath
authored
feat: support parsing pylock.toml files (#303)
* feat: support parsing `pylock.toml` files * feat: update tests and parser
1 parent 3b22785 commit dcbe42d

File tree

13 files changed

+687
-2
lines changed

13 files changed

+687
-2
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ The detector supports parsing the following lockfiles:
7070
| `poetry.lock` | `PyPI` | `poetry` |
7171
| `uv.lock` | `PyPI` | `uv` |
7272
| `Pipfile.lock` | `PyPI` | `pipenv` |
73+
| `pylock.toml` | `PyPI` | (none) |
7374
| `pdm.lock` | `PyPI` | `pdm` |
7475
| `pubspec.lock` | `Pub` | `pub` |
7576
| `pom.xml`\* | `Maven` | `maven` |

‎__snapshots__/main_test.snap‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ Don't know how to parse files as "my-file" - supported values are:
3939
poetry.lock
4040
pom.xml
4141
pubspec.lock
42+
pylock.toml
4243
renv.lock
4344
requirements.txt
4445
uv.lock

‎pkg/lockfile/ecosystems_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ func TestKnownEcosystems(t *testing.T) {
3535
expectedCount := numberOfLockfileParsers(t)
3636

3737
// - npm, yarn, bun, and pnpm,
38-
// - pip, poetry, uv, pdm and pipenv,
38+
// - pip, poetry, uv, pdm, pylock, and pipenv,
3939
// - maven and gradle,
4040
// all use the same ecosystem so "ignore" those parsers in the count
41-
expectedCount -= 8
41+
expectedCount -= 9
4242

4343
ecosystems := lockfile.KnownEcosystems()
4444

‎pkg/lockfile/parse-pylock.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package lockfile
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/BurntSushi/toml"
8+
)
9+
10+
type pylockVCS struct {
11+
Type string `toml:"type"`
12+
Commit string `toml:"commit-id"`
13+
}
14+
15+
type pylockDirectory struct {
16+
Path string `toml:"path"`
17+
}
18+
19+
type PylockPackage struct {
20+
Name string `toml:"name"`
21+
Version string `toml:"version"`
22+
VCS pylockVCS `toml:"vcs"`
23+
Directory pylockDirectory `toml:"directory"`
24+
}
25+
26+
type PylockLockfile struct {
27+
Version string `toml:"lock-version"`
28+
Packages []PylockPackage `toml:"packages"`
29+
}
30+
31+
const PylockEcosystem = PipEcosystem
32+
33+
func ParsePylock(pathToLockfile string) ([]PackageDetails, error) {
34+
var parsedLockfile *PylockLockfile
35+
36+
lockfileContents, err := os.ReadFile(pathToLockfile)
37+
38+
if err != nil {
39+
return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err)
40+
}
41+
42+
err = toml.Unmarshal(lockfileContents, &parsedLockfile)
43+
44+
if err != nil {
45+
return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err)
46+
}
47+
48+
packages := make([]PackageDetails, 0, len(parsedLockfile.Packages))
49+
50+
for _, pkg := range parsedLockfile.Packages {
51+
// this is likely the root package, which is sometimes included in the lockfile
52+
if pkg.Version == "" && pkg.Directory.Path == "." {
53+
continue
54+
}
55+
56+
pkgDetails := PackageDetails{
57+
Name: pkg.Name,
58+
Version: pkg.Version,
59+
Ecosystem: PylockEcosystem,
60+
CompareAs: PylockEcosystem,
61+
}
62+
63+
if pkg.VCS.Commit != "" {
64+
pkgDetails.Commit = pkg.VCS.Commit
65+
}
66+
67+
packages = append(packages, pkgDetails)
68+
}
69+
70+
return packages, nil
71+
}

‎pkg/lockfile/parse-pylock_test.go‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
package lockfile_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/g-rath/osv-detector/pkg/lockfile"
7+
)
8+
9+
func TestParsePylock_FileDoesNotExist(t *testing.T) {
10+
t.Parallel()
11+
12+
packages, err := lockfile.ParsePylock("testdata/pylock/does-not-exist")
13+
14+
expectErrContaining(t, err, "could not read")
15+
expectPackages(t, packages, []lockfile.PackageDetails{})
16+
}
17+
18+
func TestParsePylock_InvalidToml(t *testing.T) {
19+
t.Parallel()
20+
21+
packages, err := lockfile.ParsePylock("testdata/pylock/not-toml.txt")
22+
23+
expectErrContaining(t, err, "could not parse")
24+
expectPackages(t, packages, []lockfile.PackageDetails{})
25+
}
26+
27+
func TestParsePylock_NoPackages(t *testing.T) {
28+
t.Skip("todo: need a fixture")
29+
30+
t.Parallel()
31+
32+
packages, err := lockfile.ParsePylock("testdata/pylock/empty.toml")
33+
34+
if err != nil {
35+
t.Errorf("Got unexpected error: %v", err)
36+
}
37+
38+
expectPackages(t, packages, []lockfile.PackageDetails{})
39+
}
40+
41+
func TestParsePylock_OnePackage(t *testing.T) {
42+
t.Skip("todo: need a fixture")
43+
44+
t.Parallel()
45+
46+
packages, err := lockfile.ParsePylock("testdata/pylock/one-package.toml")
47+
48+
if err != nil {
49+
t.Errorf("Got unexpected error: %v", err)
50+
}
51+
52+
expectPackages(t, packages, []lockfile.PackageDetails{
53+
// ...
54+
})
55+
}
56+
57+
func TestParsePylock_TwoPackages(t *testing.T) {
58+
t.Skip("todo: need a fixture")
59+
60+
t.Parallel()
61+
62+
packages, err := lockfile.ParsePylock("testdata/pylock/two-packages.toml")
63+
64+
if err != nil {
65+
t.Errorf("Got unexpected error: %v", err)
66+
}
67+
68+
expectPackages(t, packages, []lockfile.PackageDetails{
69+
// ...
70+
})
71+
}
72+
73+
func TestParsePylock_Example(t *testing.T) {
74+
t.Parallel()
75+
76+
// from https://peps.python.org/pep-0751/#example
77+
packages, err := lockfile.ParsePylock("testdata/pylock/example.toml")
78+
79+
if err != nil {
80+
t.Errorf("Got unexpected error: %v", err)
81+
}
82+
83+
expectPackages(t, packages, []lockfile.PackageDetails{
84+
{
85+
Name: "attrs",
86+
Version: "25.1.0",
87+
Ecosystem: lockfile.PylockEcosystem,
88+
CompareAs: lockfile.PylockEcosystem,
89+
},
90+
{
91+
Name: "cattrs",
92+
Version: "24.1.2",
93+
Ecosystem: lockfile.PylockEcosystem,
94+
CompareAs: lockfile.PylockEcosystem,
95+
},
96+
{
97+
Name: "numpy",
98+
Version: "2.2.3",
99+
Ecosystem: lockfile.PylockEcosystem,
100+
CompareAs: lockfile.PylockEcosystem,
101+
},
102+
})
103+
}
104+
105+
func TestParsePylock_PackageWithCommits(t *testing.T) {
106+
t.Parallel()
107+
108+
packages, err := lockfile.ParsePylock("testdata/pylock/commits.toml")
109+
110+
if err != nil {
111+
t.Errorf("Got unexpected error: %v", err)
112+
}
113+
114+
expectPackages(t, packages, []lockfile.PackageDetails{
115+
{
116+
Name: "click",
117+
Version: "8.2.1",
118+
Ecosystem: lockfile.PylockEcosystem,
119+
CompareAs: lockfile.PylockEcosystem,
120+
},
121+
{
122+
Name: "mleroc",
123+
Version: "0.1.0",
124+
Ecosystem: lockfile.PylockEcosystem,
125+
CompareAs: lockfile.PylockEcosystem,
126+
Commit: "735093f03c4d8be70bfaaae44074ac92d7419b6d",
127+
},
128+
{
129+
Name: "packaging",
130+
Version: "24.2",
131+
Ecosystem: lockfile.PylockEcosystem,
132+
CompareAs: lockfile.PylockEcosystem,
133+
},
134+
{
135+
Name: "pathspec",
136+
Version: "0.12.1",
137+
Ecosystem: lockfile.PylockEcosystem,
138+
CompareAs: lockfile.PylockEcosystem,
139+
},
140+
{
141+
Name: "python-dateutil",
142+
Version: "2.9.0.post0",
143+
Ecosystem: lockfile.PylockEcosystem,
144+
CompareAs: lockfile.PylockEcosystem,
145+
},
146+
{
147+
Name: "scikit-learn",
148+
Version: "1.6.1",
149+
Ecosystem: lockfile.PylockEcosystem,
150+
CompareAs: lockfile.PylockEcosystem,
151+
},
152+
{
153+
Name: "tqdm",
154+
Version: "4.67.1",
155+
Ecosystem: lockfile.PylockEcosystem,
156+
CompareAs: lockfile.PylockEcosystem,
157+
},
158+
})
159+
}
160+
161+
func TestParsePylock_CreatedByPipWithJustSelf(t *testing.T) {
162+
t.Parallel()
163+
164+
packages, err := lockfile.ParsePylock("testdata/pylock/pip-just-self.toml")
165+
166+
if err != nil {
167+
t.Errorf("Got unexpected error: %v", err)
168+
}
169+
170+
expectPackages(t, packages, []lockfile.PackageDetails{})
171+
}
172+
173+
func TestParsePylock_CreatedByPip(t *testing.T) {
174+
t.Parallel()
175+
176+
// from https://peps.python.org/pep-0751/#example
177+
packages, err := lockfile.ParsePylock("testdata/pylock/pip-full.toml")
178+
179+
if err != nil {
180+
t.Errorf("Got unexpected error: %v", err)
181+
}
182+
183+
expectPackages(t, packages, []lockfile.PackageDetails{
184+
{
185+
Name: "annotated-types",
186+
Version: "0.7.0",
187+
Ecosystem: lockfile.PylockEcosystem,
188+
CompareAs: lockfile.PylockEcosystem,
189+
},
190+
{
191+
Name: "packaging",
192+
Version: "25.0",
193+
Ecosystem: lockfile.PylockEcosystem,
194+
CompareAs: lockfile.PylockEcosystem,
195+
},
196+
{
197+
Name: "pyproject-toml",
198+
Version: "0.1.0",
199+
Ecosystem: lockfile.PylockEcosystem,
200+
CompareAs: lockfile.PylockEcosystem,
201+
},
202+
{
203+
Name: "setuptools",
204+
Version: "80.9.0",
205+
Ecosystem: lockfile.PylockEcosystem,
206+
CompareAs: lockfile.PylockEcosystem,
207+
},
208+
{
209+
Name: "wheel",
210+
Version: "0.45.1",
211+
Ecosystem: lockfile.PylockEcosystem,
212+
CompareAs: lockfile.PylockEcosystem,
213+
},
214+
})
215+
}
216+
217+
func TestParsePylock_CreatedByPdm(t *testing.T) {
218+
t.Parallel()
219+
220+
// from https://peps.python.org/pep-0751/#example
221+
packages, err := lockfile.ParsePylock("testdata/pylock/pdm-full.toml")
222+
223+
if err != nil {
224+
t.Errorf("Got unexpected error: %v", err)
225+
}
226+
227+
expectPackages(t, packages, []lockfile.PackageDetails{
228+
{
229+
Name: "certifi",
230+
Version: "2025.1.31",
231+
Ecosystem: lockfile.PylockEcosystem,
232+
CompareAs: lockfile.PylockEcosystem,
233+
},
234+
{
235+
Name: "chardet",
236+
Version: "3.0.4",
237+
Ecosystem: lockfile.PylockEcosystem,
238+
CompareAs: lockfile.PylockEcosystem,
239+
},
240+
{
241+
Name: "charset-normalizer",
242+
Version: "2.0.12",
243+
Ecosystem: lockfile.PylockEcosystem,
244+
CompareAs: lockfile.PylockEcosystem,
245+
},
246+
{
247+
Name: "colorama",
248+
Version: "0.3.9",
249+
Ecosystem: lockfile.PylockEcosystem,
250+
CompareAs: lockfile.PylockEcosystem,
251+
},
252+
{
253+
Name: "idna",
254+
Version: "2.7",
255+
Ecosystem: lockfile.PylockEcosystem,
256+
CompareAs: lockfile.PylockEcosystem,
257+
},
258+
{
259+
Name: "py",
260+
Version: "1.4.34",
261+
Ecosystem: lockfile.PylockEcosystem,
262+
CompareAs: lockfile.PylockEcosystem,
263+
},
264+
{
265+
Name: "pytest",
266+
Version: "3.2.5",
267+
Ecosystem: lockfile.PylockEcosystem,
268+
CompareAs: lockfile.PylockEcosystem,
269+
},
270+
{
271+
Name: "requests",
272+
Version: "2.27.1",
273+
Ecosystem: lockfile.PylockEcosystem,
274+
CompareAs: lockfile.PylockEcosystem,
275+
},
276+
{
277+
Name: "setuptools",
278+
Version: "39.2.0",
279+
Ecosystem: lockfile.PylockEcosystem,
280+
CompareAs: lockfile.PylockEcosystem,
281+
},
282+
{
283+
Name: "urllib3",
284+
Version: "1.26.20",
285+
Ecosystem: lockfile.PylockEcosystem,
286+
CompareAs: lockfile.PylockEcosystem,
287+
},
288+
})
289+
}

‎pkg/lockfile/parse.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ var parsers = map[string]PackageDetailsParser{
3030
"pdm.lock": ParsePdmLock,
3131
"pnpm-lock.yaml": ParsePnpmLock,
3232
"poetry.lock": ParsePoetryLock,
33+
"pylock.toml": ParsePylock,
3334
"Pipfile.lock": ParsePipenvLock,
3435
"pom.xml": ParseMavenLock,
3536
"pubspec.lock": ParsePubspecLock,

0 commit comments

Comments
 (0)