22package main
33
44import (
5+ "context"
6+ "crypto/sha256"
57 "encoding/csv"
8+ "encoding/hex"
69 "encoding/json"
10+ "errors"
711 "flag"
812 "fmt"
913 "log/slog"
10-
1114 "net/http"
1215 "os"
1316 "path"
1417 "sort"
1518 "strconv"
1619 "strings"
20+ "sync"
1721 "time"
1822
23+ "cloud.google.com/go/storage"
1924 "github.com/google/osv/vulnfeeds/cves"
2025 "github.com/google/osv/vulnfeeds/faulttolerant"
2126 "github.com/google/osv/vulnfeeds/models"
@@ -29,12 +34,16 @@ const (
2934 debianOutputPathDefault = "debian-cve-osv"
3035 debianDistroInfoURL = "https://debian.pages.debian.net/distro-info-data/debian.csv"
3136 debianSecurityTrackerURL = "https://security-tracker.debian.org/tracker/data/json"
37+ outputBucketDefault = "debian-osv"
38+ hashMetadataKey = "sha256-hash"
3239)
3340
3441func main () {
3542 logger .InitGlobalLogger ()
3643
3744 debianOutputPath := flag .String ("output_path" , debianOutputPathDefault , "Path to output OSV files." )
45+ outputBucketName := flag .String ("output_bucket" , outputBucketDefault , "The GCS bucket to write to." )
46+ numWorkers := flag .String ("num_workers" , "64" , "Number of workers to process records" )
3847 flag .Parse ()
3948
4049 err := os .MkdirAll (* debianOutputPath , 0755 )
@@ -53,15 +62,106 @@ func main() {
5362 }
5463
5564 allCVEs := vulns .LoadAllCVEs (defaultCvePath )
56- osvCves := generateOSVFromDebianTracker (debianData , debianReleaseMap , allCVEs )
5765
58- if err = writeToOutput (osvCves , * debianOutputPath ); err != nil {
59- logger .Fatal ("Failed to write OSV output file" , slog .Any ("err" , err ))
66+ ctx := context .Background ()
67+ storageClient , err := storage .NewClient (ctx )
68+ if err != nil {
69+ logger .Fatal ("Failed to create storage client" , slog .Any ("err" , err ))
70+ }
71+ bkt := storageClient .Bucket (* outputBucketName )
72+
73+ var wg sync.WaitGroup
74+ vulnChan := make (chan * vulns.Vulnerability )
75+
76+ for range * numWorkers {
77+ wg .Add (1 )
78+ go func () {
79+ defer wg .Done ()
80+ worker (ctx , vulnChan , bkt , * debianOutputPath )
81+ }()
82+ }
83+
84+ osvCVEs := generateOSVFromDebianTracker (debianData , debianReleaseMap , allCVEs )
85+
86+ for _ , v := range osvCVEs {
87+ if len (v .Affected ) == 0 {
88+ logger .Warn (fmt .Sprintf ("Skipping %s as no affected versions found." , v .ID ), slog .String ("id" , v .ID ))
89+ continue
90+ }
91+ vulnChan <- v
6092 }
93+ close (vulnChan )
94+ wg .Wait ()
6195
6296 logger .Info ("Debian CVE conversion succeeded." )
6397}
6498
99+ func worker (ctx context.Context , vulnChan <- chan * vulns.Vulnerability , bkt * storage.BucketHandle , outputDir string ) {
100+ for v := range vulnChan {
101+ debianID := v .ID
102+ if len (v .Affected ) == 0 {
103+ logger .Warn (fmt .Sprintf ("Skipping %s as no affected versions found." , debianID ), slog .String ("id" , debianID ))
104+ continue
105+ }
106+
107+ // Marshal before setting modified time to generate hash.
108+ buf , err := json .MarshalIndent (v , "" , " " )
109+ if err != nil {
110+ logger .Error ("failed to marshal vulnerability" , slog .String ("id" , debianID ), slog .Any ("err" , err ))
111+ continue
112+ }
113+
114+ hash := sha256 .Sum256 (buf )
115+ hexHash := hex .EncodeToString (hash [:])
116+
117+ objName := path .Join (outputDir , debianID + ".json" )
118+ obj := bkt .Object (objName )
119+
120+ // Check if object exists and if hash matches.
121+ attrs , err := obj .Attrs (ctx )
122+ if err == nil {
123+ // Object exists, check hash.
124+ if attrs .Metadata != nil && attrs .Metadata [hashMetadataKey ] == hexHash {
125+ logger .Info ("Skipping upload, hash matches" , slog .String ("id" , debianID ))
126+ continue
127+ }
128+ } else if ! errors .Is (err , storage .ErrObjectNotExist ) {
129+ logger .Error ("failed to get object attributes" , slog .String ("id" , debianID ), slog .Any ("err" , err ))
130+ continue
131+ }
132+
133+ // Object does not exist or hash differs, upload.
134+ v .Modified = time .Now ().UTC ()
135+ buf , err = json .MarshalIndent (v , "" , " " )
136+ if err != nil {
137+ logger .Error ("failed to marshal vulnerability with modified time" , slog .String ("id" , debianID ), slog .Any ("err" , err ))
138+ continue
139+ }
140+
141+ logger .Info ("Uploading" , slog .String ("id" , debianID ))
142+ wc := obj .NewWriter (ctx )
143+ wc .Metadata = map [string ]string {
144+ hashMetadataKey : hexHash ,
145+ }
146+ wc .ContentType = "application/json"
147+
148+ if _ , err := wc .Write (buf ); err != nil {
149+ logger .Error ("failed to write to GCS object" , slog .String ("id" , debianID ), slog .Any ("err" , err ))
150+ // Try to close writer even if write failed.
151+ if closeErr := wc .Close (); closeErr != nil {
152+ logger .Error ("failed to close GCS writer after write error" , slog .String ("id" , debianID ), slog .Any ("err" , closeErr ))
153+ }
154+
155+ continue
156+ }
157+
158+ if err := wc .Close (); err != nil {
159+ logger .Error ("failed to close GCS writer" , slog .String ("id" , debianID ), slog .Any ("err" , err ))
160+ continue
161+ }
162+ }
163+ }
164+
65165// generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format.
66166func generateOSVFromDebianTracker (debianData DebianSecurityTrackerData , debianReleaseMap map [string ]string , allCVEs map [cves.CVEID ]cves.Vulnerability ) map [string ]* vulns.Vulnerability {
67167 logger .Info ("Converting Debian Security Tracker data to OSV." )
@@ -100,7 +200,6 @@ func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianRe
100200 Vulnerability : osvschema.Vulnerability {
101201 ID : "DEBIAN-" + cveID ,
102202 Upstream : []string {cveID },
103- Modified : time .Now ().UTC (),
104203 Published : allCVEs [cves .CVEID (cveID )].CVE .Published .Time ,
105204 Details : cveData .Description ,
106205 References : []osvschema.Reference {
@@ -198,34 +297,6 @@ func getDebianReleaseMap() (map[string]string, error) {
198297 return releaseMap , err
199298}
200299
201- func writeToOutput (osvCves map [string ]* vulns.Vulnerability , debianOutputPath string ) error {
202- logger .Info ("Writing OSV files to the output." )
203- for cveID , osv := range osvCves {
204- debianID := "DEBIAN-" + cveID
205- if len (osv .Affected ) == 0 {
206- logger .Warn (fmt .Sprintf ("Skipping %s as no affected versions found." , debianID ), slog .String ("id" , debianID ))
207- continue
208- }
209- file , err := os .OpenFile (path .Join (debianOutputPath , debianID + ".json" ), os .O_CREATE | os .O_RDWR | os .O_TRUNC , 0644 )
210- if err != nil {
211- return err
212- }
213-
214- encoder := json .NewEncoder (file )
215- encoder .SetIndent ("" , " " )
216- err = encoder .Encode (osv )
217- closeErr := file .Close ()
218- if err != nil {
219- return err
220- }
221- if closeErr != nil {
222- return closeErr
223- }
224- }
225-
226- return nil
227- }
228-
229300// downloadDebianSecurityTracker download Debian json file
230301func downloadDebianSecurityTracker () (DebianSecurityTrackerData , error ) {
231302 res , err := faulttolerant .Get (debianSecurityTrackerURL )
0 commit comments