Add support for extracting values.yaml from charts for chart-repo-sync (#421)

* add support for extracting values.yaml from chart

* merge readme and values into files collection

Author: sozercan

src/api/cmd/chart-repo-sync/sync.go

@@ -27,8 +27,8 @@ import (
 )
 
 const (
-	chartCollection       = "charts"
-	chartReadmeCollection = "readmes"
+	chartCollection      = "charts"
+	chartFilesCollection = "files"
 )
 
 type importChartFilesJob struct {
@@ -84,8 +84,8 @@ func main() {
 // Syncing is performed in the following steps:
 // 1. Update database to match chart metadata from index
 // 2. Concurrently process icons for charts (concurrently)
-// 3. Concurrently process the README for the latest chart version of each chart
-// 4. Concurrently process READMEs for historic chart versions
+// 3. Concurrently process the README and values.yaml for the latest chart version of each chart
+// 4. Concurrently process READMEs and values.yaml for historic chart versions
 //
 // These steps are processed in this way to ensure relevant chart data is
 // imported into the database as fast as possible. E.g. we want all icons for
@@ -231,7 +231,7 @@ func importCharts(charts []chart) error {
 	// Upsert pairs of selectors, charts
 	bulk.Upsert(pairs...)
 
-	// Remove	charts no longer existing in index
+	// Remove charts no longer existing in index
 	bulk.RemoveAll(bson.M{
 		"_id": bson.M{
 			"$nin": chartIDs,
@@ -252,9 +252,9 @@ func importWorker(wg *sync.WaitGroup, icons <-chan chart, chartFiles <-chan impo
 		}
 	}
 	for j := range chartFiles {
-		log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).Debug("importing readme")
-		if err := fetchAndImportReadme(j.Name, j.Repo, j.ChartVersion); err != nil {
-			log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).WithError(err).Error("failed to import readme")
+		log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).Debug("importing readme and values")
+		if err := fetchAndImportFiles(j.Name, j.Repo, j.ChartVersion); err != nil {
+			log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).WithError(err).Error("failed to import files")
 		}
 	}
 }
@@ -297,15 +297,15 @@ func fetchAndImportIcon(c chart) error {
 	return db.C(chartCollection).UpdateId(c.ID, bson.M{"$set": bson.M{"raw_icon": b.Bytes()}})
 }
 
-func fetchAndImportReadme(name string, r repo, cv chartVersion) error {
-	chartReadmeID := fmt.Sprintf("%s/%s-%s", r.Name, name, cv.Version)
+func fetchAndImportFiles(name string, r repo, cv chartVersion) error {
+	chartFilesID := fmt.Sprintf("%s/%s-%s", r.Name, name, cv.Version)
 	db, closer := dbSession.DB()
 	defer closer()
-	if err := db.C(chartReadmeCollection).FindId(chartReadmeID).One(&chartReadme{}); err == nil {
-		log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("skipping existing readme")
+	if err := db.C(chartFilesCollection).FindId(chartFilesID).One(&chartFiles{}); err == nil {
+		log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("skipping existing files")
 		return nil
 	}
-	log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("fetching readme")
+	log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("fetching files")
 
 	url := chartTarballURL(r, cv)
 	req, err := http.NewRequest("GET", url, nil)
@@ -333,22 +333,37 @@ func fetchAndImportReadme(name string, r repo, cv chartVersion) error {
 	tarf := tar.NewReader(gzf)
 
 	readmeFileName := name + "/README.md"
-	readme, err := extractFileFromTarball(readmeFileName, tarf)
-	if err != nil && !strings.Contains(err.Error(), "file not found") {
-		return err
-	}
+	valuesFileName := name + "/values.yaml"
+	fileNames := []string{valuesFileName, readmeFileName}
 
-	// Even if the readme doesn't exist, we create an empty entry to avoid
-	// refetching for this chart version in the future
-	if readme == "" {
-		log.WithFields(log.Fields{"name": name, "version": cv.Version}).Info("readme not found")
-	}
+	files, err := getFiles(cv, name, fileNames, tarf)
+	values := files[0]
+	readme := files[1]
 
-	db.C(chartReadmeCollection).Insert(chartReadme{chartReadmeID, readme})
+	db.C(chartFilesCollection).Insert(chartFiles{chartFilesID, readme, values})
 
 	return nil
 }
 
+func getFiles(cv chartVersion, name string, filenames []string, tarf *tar.Reader) ([]string, error) {
+	var files []string
+
+	for _, filename := range filenames {
+		file, err := extractFileFromTarball(filename, tarf)
+		if err != nil && !strings.Contains(err.Error(), "file not found") {
+			return nil, err
+		}
+
+		if file == "" {
+			log.WithFields(log.Fields{"name": name, "version": cv.Version}).Info(filename + " not found")
+		}
+
+		files = append(files, file)
+	}
+
+	return files, nil
+}
+
 func chartTarballURL(r repo, cv chartVersion) string {
 	source := cv.URLs[0]
 	if _, err := url.ParseRequestURI(source); err != nil {

src/api/cmd/chart-repo-sync/sync_test.go

@@ -85,14 +85,19 @@ func (h *goodIconClient) Do(req *http.Request) (*http.Response, error) {
 type goodTarballClient struct {
 	c          chart
 	skipReadme bool
+	skipValues bool
 }
 
 var testChartReadme = "# readme for chart\n\nBest chart in town"
+var testChartValues = "image: test"
 
 func (h *goodTarballClient) Do(req *http.Request) (*http.Response, error) {
 	w := httptest.NewRecorder()
 	gzw := gzip.NewWriter(w)
 	files := []tarballFile{{h.c.Name + "/Chart.yaml", "should be a Chart.yaml here..."}}
+	if !h.skipValues {
+		files = append(files, tarballFile{h.c.Name + "/values.yaml", testChartValues})
+	}
 	if !h.skipReadme {
 		files = append(files, tarballFile{h.c.Name + "/README.md", testChartReadme})
 	}
@@ -247,7 +252,7 @@ func Test_fetchAndImportIcon(t *testing.T) {
 	})
 }
 
-func Test_fetchAndImportReadme(t *testing.T) {
+func Test_fetchAndImportFiles(t *testing.T) {
 	index, _ := parseRepoIndex([]byte(validRepoIndexYAML))
 	charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"})
 	cv := charts[0].ChartVersions[0]
@@ -257,37 +262,37 @@ func Test_fetchAndImportReadme(t *testing.T) {
 		m.On("One", mock.Anything).Return(errors.New("return an error when checking if readme already exists to force fetching"))
 		dbSession = mockstore.NewMockSession(&m)
 		netClient = &badHTTPClient{}
-		assert.Err(t, io.EOF, fetchAndImportReadme(charts[0].Name, charts[0].Repo, cv))
+		assert.Err(t, io.EOF, fetchAndImportFiles(charts[0].Name, charts[0].Repo, cv))
 	})
 
-	t.Run("readme not found", func(t *testing.T) {
-		netClient = &goodTarballClient{c: charts[0], skipReadme: true}
+	t.Run("file not found", func(t *testing.T) {
+		netClient = &goodTarballClient{c: charts[0], skipValues: true, skipReadme: true}
 		m := mock.Mock{}
-		m.On("One", mock.Anything).Return(errors.New("return an error when checking if readme already exists to force fetching"))
-		m.On("Insert", chartReadme{fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version), ""})
+		m.On("One", mock.Anything).Return(errors.New("return an error when checking if files already exists to force fetching"))
+		m.On("Insert", chartFiles{fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version), "", ""})
 		dbSession = mockstore.NewMockSession(&m)
-		err := fetchAndImportReadme(charts[0].Name, charts[0].Repo, cv)
+		err := fetchAndImportFiles(charts[0].Name, charts[0].Repo, cv)
 		assert.NoErr(t, err)
 		m.AssertExpectations(t)
 	})
 
 	t.Run("valid tarball", func(t *testing.T) {
 		netClient = &goodTarballClient{c: charts[0]}
 		m := mock.Mock{}
-		m.On("One", mock.Anything).Return(errors.New("return an error when checking if readme already exists to force fetching"))
-		m.On("Insert", chartReadme{fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version), testChartReadme})
+		m.On("One", mock.Anything).Return(errors.New("return an error when checking if files already exists to force fetching"))
+		m.On("Insert", chartFiles{fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version), testChartReadme, testChartValues})
 		dbSession = mockstore.NewMockSession(&m)
-		err := fetchAndImportReadme(charts[0].Name, charts[0].Repo, cv)
+		err := fetchAndImportFiles(charts[0].Name, charts[0].Repo, cv)
 		assert.NoErr(t, err)
 		m.AssertExpectations(t)
 	})
 
-	t.Run("readme exists", func(t *testing.T) {
+	t.Run("file exists", func(t *testing.T) {
 		m := mock.Mock{}
-		// don't return an error when checking if readme already exists
+		// don't return an error when checking if files already exists
 		m.On("One", mock.Anything).Return(nil)
 		dbSession = mockstore.NewMockSession(&m)
-		err := fetchAndImportReadme(charts[0].Name, charts[0].Repo, cv)
+		err := fetchAndImportFiles(charts[0].Name, charts[0].Repo, cv)
 		assert.NoErr(t, err)
 		m.AssertNotCalled(t, "Insert", mock.Anything)
 	})

src/api/cmd/chart-repo-sync/types.go

@@ -35,7 +35,8 @@ type chartVersion struct {
 	URLs       []string
 }
 
-type chartReadme struct {
+type chartFiles struct {
 	ID     string `bson:"_id"`
 	Readme string
+	Values string
 }