Merge pull request #174 from influxdata/dockerlib

Move update.go logic to a command line program

Author: jsternberg

Jenkinsfile

@@ -51,7 +51,7 @@ pipeline {
       }
     }
 
-    stage('Update official images') {
+    stage('Checkout official images') {
       steps {
         dir('official-images') {
           checkout(
@@ -69,11 +69,19 @@ pipeline {
           // Reset master to the value at origin.
           sh "git checkout master && git reset --hard origin/master"
         }
+      }
+    }
 
-        withDockerContainer(image: "golang:1.9.1-stretch") {
-          sh 'cd influxdata-docker; go run update.go -n'
+    stage('Update official images') {
+      agent {
+        dockerfile {
+          dir 'dockerlib'
         }
       }
+
+      steps {
+        sh "dockerlib update"
+      }
     }
 
     stage('Push changes to GitHub') {

chronograf/manifest.json

@@ -7,6 +7,6 @@
     "arm64v8"
   ],
   "variants": [
-    "alpine"
+    {"name": "alpine"}
   ]
 }

circle-test.sh

@@ -32,7 +32,7 @@ tags=()
 
 # Gather directories with a Dockerfile and sanitize the path to remove leading
 # a leading ./ and multiple slashes into a single slash.
-dockerfiles=$(find "$dir" -name nightly -prune -o -name Dockerfile -print0 | xargs -0 -I{} dirname {} | sed 's@^./@@' | sed 's@//*@/@g')
+dockerfiles=$(find "$dir" -name nightly -prune -o -name Dockerfile -print0 | grep -v dockerlib | xargs -0 -I{} dirname {} | sed 's@^./@@' | sed 's@//*@/@g')
 for path in $dockerfiles; do
   # Generate a tag by replacing the first slash with a colon and all remaining slashes with a dash.
   tag=$(echo $path | sed 's@/@:@' | sed 's@/@-@g')

dockerlib/Dockerfile

@@ -0,0 +1,9 @@
+FROM golang:1.9.2-stretch as builder
+COPY . /go/src/github.com/influxdata/influxdata-docker/dockerlib
+RUN set -xe && \
+    go get -d github.com/influxdata/influxdata-docker/dockerlib && \
+    go install github.com/influxdata/influxdata-docker/dockerlib
+
+FROM debian:stretch
+COPY --from=builder /go/bin/dockerlib /usr/bin/dockerlib
+ENTRYPOINT ["/usr/bin/dockerlib"]

dockerlib/commands.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"os"
+
+	"path/filepath"
+
+	"fmt"
+
+	"github.com/influxdata/influxdata-docker/dockerlib/dockerlib"
+	"github.com/spf13/cobra"
+)
+
+var RootCmd = &cobra.Command{
+	Use:   "dockerlib",
+	Short: "Manage docker official images",
+}
+
+var UpdateCmd = &cobra.Command{
+	Use:   "update",
+	Short: "Update the official-images repository",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := Update(); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func Update() error {
+	// Locate all of the manifests within this repository.
+	manifests, err := dockerlib.FindImageManifests()
+	if err != nil {
+		return err
+	}
+
+	for _, m := range manifests {
+		header := dockerlib.Header{}
+		header.Add("Maintainers", getDefaultMaintainer())
+		header.Add("GitRepo", remoteRepo)
+
+		rev, err := latestRev(m.BaseDir)
+		if err != nil {
+			return err
+		}
+		header.Add("GitCommit", rev)
+
+		if err := func() error {
+			f, err := os.Create(filepath.Join("../official-images/library", m.Name))
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			if err := m.Write(f, &header); err != nil {
+				return err
+			}
+			return f.Close()
+		}(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func init() {
+	RootCmd.AddCommand(UpdateCmd)
+}

dockerlib/dockerlib/header.go

@@ -0,0 +1,54 @@
+package dockerlib
+
+import (
+	"container/list"
+	"fmt"
+	"io"
+	"strings"
+)
+
+type keyValuePair struct {
+	key    string
+	values []string
+}
+
+type Header struct {
+	values *list.List
+	index  map[string]*list.Element
+}
+
+func (h *Header) Add(key, value string) {
+	e, ok := h.index[key]
+	if !ok {
+		h.Set(key, value)
+		return
+	}
+
+	kv := e.Value.(*keyValuePair)
+	kv.values = append(kv.values, value)
+}
+
+func (h *Header) Set(key, value string) {
+	if h.index == nil {
+		h.index = make(map[string]*list.Element)
+	}
+	if h.values == nil {
+		h.values = list.New()
+	}
+	values := make([]string, 1)
+	values[0] = value
+	h.index[key] = h.values.PushBack(&keyValuePair{
+		key:    key,
+		values: values,
+	})
+}
+
+func (h *Header) Write(w io.Writer) error {
+	for front := h.values.Front(); front != nil; front = front.Next() {
+		kv := front.Value.(*keyValuePair)
+		if _, err := fmt.Fprintf(w, "%s: %s\n", kv.key, strings.Join(kv.values, ", ")); err != nil {
+			return err
+		}
+	}
+	return nil
+}

dockerlib/dockerlib/manifest.go

@@ -0,0 +1,167 @@
+package dockerlib
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+type ImageManifest struct {
+	Name          string     `json:"name"`
+	BaseDir       string     `json:"-"`
+	Versions      []*Version `json:"versions"`
+	Architectures []string   `json:"architectures"`
+	Variants      []*Variant `json:"variants"`
+	Latest        string     `json:"-"`
+}
+
+type Variant struct {
+	Name          string     `json:"name"`
+	Versions      []*Version `json:"versions"`
+	Architectures []string   `json:"architectures"`
+	Latest        string     `json:"-"`
+}
+
+func FindImageManifests() ([]*ImageManifest, error) {
+	var manifests []*ImageManifest
+	if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		} else if info.IsDir() && info.Name() == ".git" {
+			return filepath.SkipDir
+		}
+
+		// Check if there is a manifest.json in this folder.
+		if info.Name() != "manifest.json" {
+			return nil
+		}
+
+		// We found a manifest.json in this folder. Read and parse it into a manifest.
+		manifest, err := ReadImageManifest(path)
+		if err != nil {
+			return err
+		}
+		manifests = append(manifests, manifest)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return manifests, nil
+}
+
+func ReadImageManifest(fpath string) (*ImageManifest, error) {
+	in, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+
+	var manifest ImageManifest
+	if err := json.Unmarshal(in, &manifest); err != nil {
+		return nil, err
+	}
+	manifest.BaseDir = filepath.Dir(fpath)
+
+	// Look through each of the versions to determine which is the latest.
+	if len(manifest.Versions) > 0 {
+		manifest.Latest = Versions(manifest.Versions).Latest().String()
+	}
+
+	// Propagate the image values to the variants and determine the base directory
+	// and the latest version.
+	for _, variant := range manifest.Variants {
+		if len(variant.Versions) == 0 {
+			variant.Versions = manifest.Versions
+			variant.Latest = manifest.Latest
+		} else {
+			variant.Latest = Versions(variant.Versions).Latest().String()
+		}
+	}
+	return &manifest, nil
+}
+
+func (m *ImageManifest) Write(w io.Writer, header *Header) error {
+	// Output the initial header passed in to write.
+	if header != nil {
+		header.Write(w)
+	}
+
+	// Determine all of the versions.
+	for _, v := range m.Versions {
+		dir := filepath.Join(m.BaseDir, v.String())
+
+		header := &Header{}
+		version, err := DockerfileVersion(dir)
+		if err != nil {
+			return err
+		}
+		for i := 2; i <= len(version.Segments()); i++ {
+			segments := version.Segments()
+			parts := make([]string, i)
+			for j, s := range segments[:i] {
+				parts[j] = strconv.Itoa(s)
+			}
+			header.Add("Tags", strings.Join(parts, "."))
+		}
+		if m.Latest == v.String() {
+			header.Add("Tags", "latest")
+		}
+		for _, arch := range m.Architectures {
+			header.Add("Architectures", arch)
+		}
+		header.Add("Directory", dir)
+
+		fmt.Fprintln(w)
+		if err := header.Write(w); err != nil {
+			return err
+		}
+
+		for _, variant := range m.Variants {
+			// Check if this variant is included in the list of versions.
+			found := false
+			for _, vv := range variant.Versions {
+				if v.String() == vv.String() {
+					found = true
+					break
+				}
+			}
+
+			// Skip this variant if the version isn't found in the variant.
+			if !found {
+				continue
+			}
+
+			header := &Header{}
+			version, err := DockerfileVersion(dir)
+			if err != nil {
+				return err
+			}
+			for i := 2; i <= len(version.Segments()); i++ {
+				segments := version.Segments()
+				parts := make([]string, i)
+				for j, s := range segments[:i] {
+					parts[j] = strconv.Itoa(s)
+				}
+				header.Add("Tags", strings.Join(parts, ".")+"-"+variant.Name)
+			}
+			if m.Latest == v.String() {
+				header.Add("Tags", variant.Name)
+			}
+			for _, arch := range variant.Architectures {
+				header.Add("Architectures", arch)
+			}
+			header.Add("Directory", filepath.Join(dir, variant.Name))
+			//header.Add("Tags", variant.Name)
+			fmt.Fprintln(w)
+			if err := header.Write(w); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}