@@ -11,11 +11,12 @@ import (
1111 "sync"
1212 "time"
1313
14- "github.com/htcat/htcat"
1514 "github.com/inconshreveable/log15"
1615 "github.com/spf13/viper"
1716 "github.com/ulikunitz/xz"
1817 "golang.org/x/xerrors"
18+
19+ "github.com/vulsio/goval-dictionary/config"
1920)
2021
2122// MIMEType :
@@ -68,7 +69,6 @@ type FetchRequest struct {
6869 Target string
6970 URL string
7071 MIMEType MIMEType
71- Concurrently bool
7272 LogSuppressed bool
7373}
7474
@@ -122,12 +122,7 @@ func FetchFeedFiles(reqs []FetchRequest) (results []FetchResult, err error) {
122122 tasks <- func () {
123123 select {
124124 case req := <- reqChan :
125- var body []byte
126- if req .Concurrently {
127- body , err = fetchFileConcurrently (req , 20 / len (reqs ))
128- } else {
129- body , err = fetchFileWithUA (req )
130- }
125+ body , err := fetchFileWithUA (req )
131126 wg .Done ()
132127 if err != nil {
133128 errChan <- err
@@ -163,57 +158,6 @@ func FetchFeedFiles(reqs []FetchRequest) (results []FetchResult, err error) {
163158 return results , nil
164159}
165160
166- func fetchFileConcurrently (req FetchRequest , concurrency int ) (body []byte , err error ) {
167- var proxyURL * url.URL
168- httpClient := & http.Client {}
169- httpProxy := viper .GetString ("http-proxy" )
170- if httpProxy != "" {
171- if proxyURL , err = url .Parse (httpProxy ); err != nil {
172- return nil , xerrors .Errorf ("Failed to parse proxy url. err: %w" , err )
173- }
174- httpClient = & http.Client {Transport : & http.Transport {Proxy : http .ProxyURL (proxyURL )}}
175- }
176-
177- u , err := url .Parse (req .URL )
178- if err != nil {
179- return nil , xerrors .Errorf ("Failed to parse given URL: %w" , err )
180- }
181-
182- buf := bytes.Buffer {}
183- htc := htcat .New (httpClient , u , concurrency )
184- if _ , err := htc .WriteTo (& buf ); err != nil {
185- return nil , xerrors .Errorf ("Failed to write to output stream: %w" , err )
186- }
187-
188- var b bytes.Buffer
189- switch req .MIMEType {
190- case MIMETypeXML , MIMETypeTxt , MIMETypeJSON , MIMETypeYml , MIMETypeHTML :
191- b = buf
192- case MIMETypeBzip2 :
193- if _ , err := b .ReadFrom (bzip2 .NewReader (bytes .NewReader (buf .Bytes ()))); err != nil {
194- return nil , xerrors .Errorf ("Failed to open bzip2 file. err: %w" , err )
195- }
196- case MIMETypeXz :
197- r , err := xz .NewReader (bytes .NewReader (buf .Bytes ()))
198- if err != nil {
199- return nil , xerrors .Errorf ("Failed to open xz file. err: %w" , err )
200- }
201- if _ , err := b .ReadFrom (r ); err != nil {
202- return nil , xerrors .Errorf ("Failed to read xz file. err: %w" , err )
203- }
204- case MIMETypeGzip :
205- r , err := gzip .NewReader (bytes .NewReader (buf .Bytes ()))
206- if err != nil {
207- return nil , xerrors .Errorf ("Failed to open gzip file. err: %w" , err )
208- }
209- if _ , err := b .ReadFrom (r ); err != nil {
210- return nil , xerrors .Errorf ("Failed to read gzip file. err: %w" , err )
211- }
212- }
213-
214- return b .Bytes (), nil
215- }
216-
217161func fetchFileWithUA (req FetchRequest ) (body []byte , err error ) {
218162 var proxyURL * url.URL
219163 var resp * http.Response
@@ -232,7 +176,7 @@ func fetchFileWithUA(req FetchRequest) (body []byte, err error) {
232176 return nil , xerrors .Errorf ("Failed to download. err: %w" , err )
233177 }
234178
235- httpreq .Header .Set ("User-Agent" , "curl/7.37.0" )
179+ httpreq .Header .Set ("User-Agent" , fmt . Sprintf ( "goval-dictionary/%s.%s" , config . Version , config . Revision ) )
236180 resp , err = httpClient .Do (httpreq )
237181 if err != nil {
238182 return nil , xerrors .Errorf ("Failed to download. err: %w" , err )
0 commit comments