Skip to content

Commit 497a8d6

Browse files
8W9aGdanfairs
8W9aG
and
danfairs
authored
fix: Requirements format parser (#2292)
* Correct long line (with continuation) handling Signed-off-by: Dan Fairs <[email protected]> * Remove a redundant line Signed-off-by: Dan Fairs <[email protected]> --------- Signed-off-by: Dan Fairs <[email protected]> Co-authored-by: Dan Fairs <[email protected]>
1 parent d530094 commit 497a8d6

File tree

2 files changed

+139
-40
lines changed

2 files changed

+139
-40
lines changed

‎pkg/requirements/requirements.go‎

Lines changed: 58 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -45,59 +45,77 @@ func ReadRequirements(path string) ([]string, error) {
4545
if err != nil {
4646
return nil, err
4747
}
48+
defer fh.Close()
49+
4850
// Use scanner to handle CRLF endings
4951
scanner := bufio.NewScanner(fh)
5052
scanner.Split(scanLinesWithContinuations)
51-
requirements := []string{}
53+
54+
var requirements []string
55+
5256
for scanner.Scan() {
53-
requirementsText := strings.TrimSpace(scanner.Text())
54-
if len(requirementsText) == 0 {
57+
line := strings.TrimSpace(scanner.Text())
58+
59+
// Skip empty lines and comment lines
60+
if strings.HasPrefix(line, "#") {
5561
continue
5662
}
57-
requirements = append(requirements, requirementsText)
63+
64+
// Remove any trailing comments
65+
if idx := strings.Index(line, "#"); idx >= 0 {
66+
line = line[:idx]
67+
}
68+
69+
if line != "" {
70+
requirements = append(requirements, line)
71+
}
5872
}
59-
return requirements, nil
73+
74+
return requirements, scanner.Err()
6075
}
6176

77+
// scanLinesWithContinuations is a modified version of bufio.ScanLines that
78+
// also handles line continuations (lines ending with a backslash).
6279
func scanLinesWithContinuations(data []byte, atEOF bool) (advance int, token []byte, err error) {
63-
advance = 0
64-
token = nil
65-
err = nil
66-
inHash := false
67-
for {
68-
if atEOF || len(data) == 0 {
69-
break
70-
}
71-
if token == nil {
72-
token = []byte{}
73-
}
74-
if data[advance] == '#' {
75-
inHash = true
76-
}
77-
if data[advance] == '\n' {
78-
shouldAdvance := true
79-
if len(token) > 0 {
80-
if token[len(token)-1] == '\r' && !inHash {
81-
token = token[:len(token)-1]
82-
}
83-
if token[len(token)-1] == '\\' {
84-
if !inHash {
85-
token = token[:len(token)-1]
86-
}
87-
shouldAdvance = false
88-
}
80+
// If we're at EOF and there's no data, return nil
81+
if atEOF && len(data) == 0 {
82+
return 0, nil, nil
83+
}
84+
85+
var line []byte
86+
start := 0
87+
for i := 0; i < len(data); i++ {
88+
if data[i] == '\n' {
89+
end := i
90+
if end > 0 && data[end-1] == '\r' {
91+
end--
8992
}
90-
if shouldAdvance {
91-
advance++
92-
break
93+
// Add this segment to our accumulated line
94+
line = append(line, data[start:end]...)
95+
96+
if len(line) > 0 && line[len(line)-1] == '\\' {
97+
// This is a continuation - remove the backslash and continue
98+
line = line[:len(line)-1]
99+
start = i + 1
100+
continue
93101
}
94-
} else if !inHash {
95-
token = append(token, data[advance])
102+
103+
// Not a continuation, return the accumulated line
104+
return i + 1, line, nil
96105
}
97-
advance++
98-
if advance == len(data) {
99-
break
106+
}
107+
108+
// If we're at EOF, we have a final, non-terminated line
109+
if atEOF {
110+
if len(data) > start {
111+
line = append(line, data[start:]...)
112+
if len(line) > 0 && line[len(line)-1] == '\r' {
113+
line = line[:len(line)-1]
114+
}
100115
}
116+
return len(data), line, nil
101117
}
102-
return advance, token, err
118+
119+
// Need more data
120+
return 0, nil, nil
103121
}

‎pkg/requirements/requirements_test.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"os"
55
"path"
66
"path/filepath"
7+
"strings"
78
"testing"
89

910
"github.com/stretchr/testify/require"
@@ -72,3 +73,83 @@ flask>0.4
7273
require.NoError(t, err)
7374
require.Equal(t, []string{"foo==1.0.0", "fastapi>=0.6,<1", "flask>0.4", "-f http://example.com"}, requirements)
7475
}
76+
77+
func TestReadRequirementsLongLine(t *testing.T) {
78+
srcDir := t.TempDir()
79+
reqFile := path.Join(srcDir, "requirements.txt")
80+
err := os.WriteFile(reqFile, []byte(`
81+
antlr4-python3-runtime==4.9.3 \
82+
--hash=sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b
83+
colorama==0.4.6 ; sys_platform == 'win32' \
84+
--hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
85+
--hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
86+
contourpy==1.3.2 \
87+
--hash=sha256:0475b1f6604896bc7c53bb070e355e9321e1bc0d381735421a2d2068ec56531f \
88+
--hash=sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92 \
89+
--hash=sha256:15ce6ab60957ca74cff444fe66d9045c1fd3e92c8936894ebd1f3eef2fff075f \
90+
--hash=sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f \
91+
--hash=sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7 \
92+
--hash=sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e \
93+
--hash=sha256:3f9e896f447c5c8618f1edb2bafa9a4030f22a575ec418ad70611450720b5b08 \
94+
--hash=sha256:434f0adf84911c924519d2b08fc10491dd282b20bdd3fa8f60fd816ea0b48841 \
95+
--hash=sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5 \
96+
--hash=sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2 \
97+
--hash=sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415 \
98+
--hash=sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878 \
99+
--hash=sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0 \
100+
--hash=sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab \
101+
--hash=sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445 \
102+
--hash=sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43 \
103+
--hash=sha256:71e2bd4a1c4188f5c2b8d274da78faab884b59df20df63c34f74aa1813c4427c \
104+
--hash=sha256:745b57db7758f3ffc05a10254edd3182a2a83402a89c00957a8e8a22f5582823 \
105+
--hash=sha256:78e9253c3de756b3f6a5174d024c4835acd59eb3f8e2ca13e775dbffe1558f69 \
106+
--hash=sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15 \
107+
--hash=sha256:8b7fc0cd78ba2f4695fd0a6ad81a19e7e3ab825c31b577f384aa9d7817dc3bef \
108+
--hash=sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5 \
109+
--hash=sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73 \
110+
--hash=sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912 \
111+
--hash=sha256:970e9173dbd7eba9b4e01aab19215a48ee5dd3f43cef736eebde064a171f89a5 \
112+
--hash=sha256:977e98a0e0480d3fe292246417239d2d45435904afd6d7332d8455981c408b85 \
113+
--hash=sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54 \
114+
--hash=sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773 \
115+
--hash=sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441 \
116+
--hash=sha256:c66c4906cdbc50e9cba65978823e6e00b45682eb09adbb78c9775b74eb222422 \
117+
--hash=sha256:c6c4639a9c22230276b7bffb6a850dfc8258a2521305e1faefe804d006b2e532 \
118+
--hash=sha256:c85bb486e9be652314bb5b9e2e3b0d1b2e643d5eec4992c0fbe8ac71775da739 \
119+
--hash=sha256:cc829960f34ba36aad4302e78eabf3ef16a3a100863f0d4eeddf30e8a485a03b \
120+
--hash=sha256:d0e589ae0d55204991450bb5c23f571c64fe43adaa53f93fc902a84c96f52fe1 \
121+
--hash=sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87 \
122+
--hash=sha256:d32530b534e986374fc19eaa77fcb87e8a99e5431499949b828312bdcd20ac52 \
123+
--hash=sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1 \
124+
--hash=sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd \
125+
--hash=sha256:de39db2604ae755316cb5967728f4bea92685884b1e767b7c24e983ef5f771cb \
126+
--hash=sha256:de425af81b6cea33101ae95ece1f696af39446db9682a0b56daaa48cfc29f38f \
127+
--hash=sha256:e1578f7eafce927b168752ed7e22646dad6cd9bca673c60bff55889fa236ebf9 \
128+
--hash=sha256:e298e7e70cf4eb179cc1077be1c725b5fd131ebc81181bf0c03525c8abc297fd \
129+
--hash=sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83 \
130+
--hash=sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe
131+
cycler==0.12.1 \
132+
--hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \
133+
--hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c`), 0o644)
134+
require.NoError(t, err)
135+
requirements, err := ReadRequirements(reqFile)
136+
require.NoError(t, err)
137+
checkRequirements(t, []string{
138+
"antlr4-python3-runtime==4.9.3 --hash=sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b",
139+
"colorama==0.4.6 ; sys_platform == 'win32' --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
140+
"contourpy==1.3.2 --hash=sha256:0475b1f6604896bc7c53bb070e355e9321e1bc0d381735421a2d2068ec56531f --hash=sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92 --hash=sha256:15ce6ab60957ca74cff444fe66d9045c1fd3e92c8936894ebd1f3eef2fff075f --hash=sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f --hash=sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7 --hash=sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e --hash=sha256:3f9e896f447c5c8618f1edb2bafa9a4030f22a575ec418ad70611450720b5b08 --hash=sha256:434f0adf84911c924519d2b08fc10491dd282b20bdd3fa8f60fd816ea0b48841 --hash=sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5 --hash=sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2 --hash=sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415 --hash=sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878 --hash=sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0 --hash=sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab --hash=sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445 --hash=sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43 --hash=sha256:71e2bd4a1c4188f5c2b8d274da78faab884b59df20df63c34f74aa1813c4427c --hash=sha256:745b57db7758f3ffc05a10254edd3182a2a83402a89c00957a8e8a22f5582823 --hash=sha256:78e9253c3de756b3f6a5174d024c4835acd59eb3f8e2ca13e775dbffe1558f69 --hash=sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15 --hash=sha256:8b7fc0cd78ba2f4695fd0a6ad81a19e7e3ab825c31b577f384aa9d7817dc3bef --hash=sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5 --hash=sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73 --hash=sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912 --hash=sha256:970e9173dbd7eba9b4e01aab19215a48ee5dd3f43cef736eebde064a171f89a5 --hash=sha256:977e98a0e0480d3fe292246417239d2d45435904afd6d7332d8455981c408b85 --hash=sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54 --hash=sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773 --hash=sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441 --hash=sha256:c66c4906cdbc50e9cba65978823e6e00b45682eb09adbb78c9775b74eb222422 --hash=sha256:c6c4639a9c22230276b7bffb6a850dfc8258a2521305e1faefe804d006b2e532 --hash=sha256:c85bb486e9be652314bb5b9e2e3b0d1b2e643d5eec4992c0fbe8ac71775da739 --hash=sha256:cc829960f34ba36aad4302e78eabf3ef16a3a100863f0d4eeddf30e8a485a03b --hash=sha256:d0e589ae0d55204991450bb5c23f571c64fe43adaa53f93fc902a84c96f52fe1 --hash=sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87 --hash=sha256:d32530b534e986374fc19eaa77fcb87e8a99e5431499949b828312bdcd20ac52 --hash=sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1 --hash=sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd --hash=sha256:de39db2604ae755316cb5967728f4bea92685884b1e767b7c24e983ef5f771cb --hash=sha256:de425af81b6cea33101ae95ece1f696af39446db9682a0b56daaa48cfc29f38f --hash=sha256:e1578f7eafce927b168752ed7e22646dad6cd9bca673c60bff55889fa236ebf9 --hash=sha256:e298e7e70cf4eb179cc1077be1c725b5fd131ebc81181bf0c03525c8abc297fd --hash=sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83 --hash=sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe",
141+
"cycler==0.12.1 --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c",
142+
}, requirements)
143+
}
144+
145+
func checkRequirements(t *testing.T, expected []string, actual []string) {
146+
t.Helper()
147+
for n, expectLine := range expected {
148+
actualLine := actual[n]
149+
// collapse any multiple-space runs with single spaces in the actual line - the generator may output these
150+
// but we don't care about them for comparison purposes
151+
actualLine = strings.Join(strings.Fields(actualLine), " ")
152+
require.Equal(t, expectLine, actualLine)
153+
}
154+
require.Equal(t, len(expected), len(actual))
155+
}

0 commit comments

Comments
 (0)