Merge pull request #213 from crawford/digitalocean

digitalocean: Add support for DigitalOcean

Author: Alex Crawford

coreos-cloudinit.go

@@ -11,6 +11,7 @@ import (
 	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
 	"github.com/coreos/coreos-cloudinit/datasource/file"
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
 	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
 	"github.com/coreos/coreos-cloudinit/datasource/url"
@@ -30,13 +31,14 @@ var (
 	printVersion  bool
 	ignoreFailure bool
 	sources       struct {
-		file                      string
-		configDrive               string
-		metadataService           bool
-		ec2MetadataService        string
-		cloudSigmaMetadataService bool
-		url                       string
-		procCmdLine               bool
+		file                        string
+		configDrive                 string
+		metadataService             bool
+		ec2MetadataService          string
+		cloudSigmaMetadataService   bool
+		digitalOceanMetadataService string
+		url                         string
+		procCmdLine                 bool
 	}
 	convertNetconf string
 	workspace      string
@@ -49,11 +51,12 @@ func init() {
 	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
 	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
 	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
-	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download data from the provided metadata service")
+	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
 	flag.BoolVar(&sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
+	flag.StringVar(&sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
 	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
 	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
-	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
+	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
 	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
 	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
 }
@@ -73,16 +76,12 @@ func main() {
 		os.Exit(0)
 	}
 
-	if convertNetconf != "" && sources.configDrive == "" {
-		fmt.Println("-convert-netconf flag requires -from-configdrive")
-		os.Exit(1)
-	}
-
 	switch convertNetconf {
 	case "":
 	case "debian":
+	case "digitalocean":
 	default:
-		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian'\n", convertNetconf)
+		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", convertNetconf)
 		os.Exit(1)
 	}
 
@@ -209,6 +208,13 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
 			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
 		}
 	}
+	if mdcc.NetworkConfig != "" {
+		if udcc.NetworkConfig != "" {
+			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
+		} else {
+			udcc.NetworkConfig = mdcc.NetworkConfig
+		}
+	}
 	return udcc
 }
 
@@ -234,6 +240,9 @@ func getDatasources() []datasource.Datasource {
 	if sources.cloudSigmaMetadataService {
 		dss = append(dss, cloudsigma.NewServerContextService())
 	}
+	if sources.digitalOceanMetadataService != "" {
+		dss = append(dss, digitalocean.NewDatasource(sources.digitalOceanMetadataService))
+	}
 	if sources.procCmdLine {
 		dss = append(dss, proc_cmdline.NewDatasource())
 	}

coreos-cloudinit_test.go

@@ -12,6 +12,7 @@ func TestMergeCloudConfig(t *testing.T) {
 		SSHAuthorizedKeys: []string{"abc", "def"},
 		Hostname:          "foobar",
 		NetworkConfigPath: "/path/somewhere",
+		NetworkConfig:     `{}`,
 	}
 	for i, tt := range []struct {
 		udcc initialize.CloudConfig
@@ -36,6 +37,7 @@ func TestMergeCloudConfig(t *testing.T) {
 			initialize.CloudConfig{
 				Hostname:          "meta-hostname",
 				NetworkConfigPath: "/path/meta",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 			simplecc,
 		},
@@ -45,6 +47,7 @@ func TestMergeCloudConfig(t *testing.T) {
 				SSHAuthorizedKeys: []string{"abc", "def"},
 				Hostname:          "user-hostname",
 				NetworkConfigPath: "/path/somewhere",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 			initialize.CloudConfig{
 				SSHAuthorizedKeys: []string{"woof", "qux"},
@@ -54,6 +57,7 @@ func TestMergeCloudConfig(t *testing.T) {
 				SSHAuthorizedKeys: []string{"abc", "def", "woof", "qux"},
 				Hostname:          "user-hostname",
 				NetworkConfigPath: "/path/somewhere",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 		},
 		{
@@ -64,11 +68,13 @@ func TestMergeCloudConfig(t *testing.T) {
 			initialize.CloudConfig{
 				SSHAuthorizedKeys: []string{"zaphod", "beeblebrox"},
 				NetworkConfigPath: "/dev/fun",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 			initialize.CloudConfig{
 				Hostname:          "supercool",
 				SSHAuthorizedKeys: []string{"zaphod", "beeblebrox"},
 				NetworkConfigPath: "/dev/fun",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 		},
 		{
@@ -80,11 +86,13 @@ func TestMergeCloudConfig(t *testing.T) {
 			initialize.CloudConfig{
 				Hostname:          "youyouyou",
 				NetworkConfigPath: "meta-meta-yo",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 			initialize.CloudConfig{
 				Hostname:          "mememe",
 				ManageEtcHosts:    initialize.EtcHosts("lolz"),
 				NetworkConfigPath: "meta-meta-yo",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 		},
 		{
@@ -95,10 +103,12 @@ func TestMergeCloudConfig(t *testing.T) {
 			initialize.CloudConfig{
 				ManageEtcHosts:    initialize.EtcHosts("lolz"),
 				NetworkConfigPath: "meta-meta-yo",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 			initialize.CloudConfig{
 				Hostname:          "mememe",
 				NetworkConfigPath: "meta-meta-yo",
+				NetworkConfig:     `{"hostname":"test"}`,
 			},
 		},
 	} {

datasource/metadata/digitalocean/metadata.go

@@ -0,0 +1,107 @@
+package digitalocean
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/coreos/coreos-cloudinit/datasource/metadata"
+)
+
+const (
+	DefaultAddress = "http://169.254.169.254/"
+	apiVersion     = "metadata/v1"
+	userdataUrl    = apiVersion + "/user-data"
+	metadataPath   = apiVersion + ".json"
+)
+
+type Address struct {
+	IPAddress string `json:"ip_address"`
+	Netmask   string `json:"netmask"`
+	Cidr      int    `json:"cidr"`
+	Gateway   string `json:"gateway"`
+}
+
+type Interface struct {
+	IPv4 *Address `json:"ipv4"`
+	IPv6 *Address `json:"ipv6"`
+	MAC  string   `json:"mac"`
+	Type string   `json:"type"`
+}
+
+type Interfaces struct {
+	Public  []Interface `json:"public"`
+	Private []Interface `json:"private"`
+}
+
+type DNS struct {
+	Nameservers []string `json:"nameservers"`
+}
+
+type Metadata struct {
+	Hostname   string     `json:"hostname"`
+	Interfaces Interfaces `json:"interfaces"`
+	PublicKeys []string   `json:"public_keys"`
+	DNS        DNS        `json:"dns"`
+}
+
+type metadataService struct {
+	interfaces Interfaces
+	dns        DNS
+	metadata.MetadataService
+}
+
+func NewDatasource(root string) *metadataService {
+	return &metadataService{MetadataService: metadata.NewDatasource(root, apiVersion, userdataUrl, metadataPath)}
+}
+
+func (ms *metadataService) FetchMetadata() ([]byte, error) {
+	data, err := ms.FetchData(ms.MetadataUrl())
+	if err != nil || len(data) == 0 {
+		return []byte{}, err
+	}
+
+	var metadata Metadata
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		return []byte{}, err
+	}
+
+	ms.interfaces = metadata.Interfaces
+	ms.dns = metadata.DNS
+
+	attrs := make(map[string]interface{})
+	if len(metadata.Interfaces.Public) > 0 {
+		if metadata.Interfaces.Public[0].IPv4 != nil {
+			attrs["public-ipv4"] = metadata.Interfaces.Public[0].IPv4.IPAddress
+		}
+		if metadata.Interfaces.Public[0].IPv6 != nil {
+			attrs["public-ipv6"] = metadata.Interfaces.Public[0].IPv6.IPAddress
+		}
+	}
+	if len(metadata.Interfaces.Private) > 0 {
+		if metadata.Interfaces.Private[0].IPv4 != nil {
+			attrs["local-ipv4"] = metadata.Interfaces.Private[0].IPv4.IPAddress
+		}
+		if metadata.Interfaces.Private[0].IPv6 != nil {
+			attrs["local-ipv6"] = metadata.Interfaces.Private[0].IPv6.IPAddress
+		}
+	}
+	attrs["hostname"] = metadata.Hostname
+	keys := make(map[string]string)
+	for i, key := range metadata.PublicKeys {
+		keys[strconv.Itoa(i)] = key
+	}
+	attrs["public_keys"] = keys
+
+	return json.Marshal(attrs)
+}
+
+func (ms metadataService) FetchNetworkConfig(filename string) ([]byte, error) {
+	return json.Marshal(Metadata{
+		Interfaces: ms.interfaces,
+		DNS:        ms.dns,
+	})
+}
+
+func (ms metadataService) Type() string {
+	return "digitalocean-metadata-service"
+}

datasource/metadata/digitalocean/metadata_test.go

@@ -0,0 +1,99 @@
+package digitalocean
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/coreos/coreos-cloudinit/datasource/metadata"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/test"
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+func TestType(t *testing.T) {
+	want := "digitalocean-metadata-service"
+	if kind := (metadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       []byte
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root:         "/",
+			metadataPath: "v1.json",
+			resources: map[string]string{
+				"/v1.json": "bad",
+			},
+			expectErr: fmt.Errorf("invalid character 'b' looking for beginning of value"),
+		},
+		{
+			root:         "/",
+			metadataPath: "v1.json",
+			resources: map[string]string{
+				"/v1.json": `{
+  "droplet_id": 1,
+  "user_data": "hello",
+  "vendor_data": "hello",
+  "public_keys": [
+    "publickey1",
+    "publickey2"
+  ],
+  "region": "nyc2",
+  "interfaces": {
+    "public": [
+      {
+        "ipv4": {
+          "ip_address": "192.168.1.2",
+          "netmask": "255.255.255.0",
+          "gateway": "192.168.1.1"
+        },
+        "ipv6": {
+          "ip_address": "fe00::",
+          "cidr": 126,
+          "gateway": "fe00::"
+        },
+        "mac": "ab:cd:ef:gh:ij",
+        "type": "public"
+      }
+    ]
+  }
+}`,
+			},
+			expect: []byte(`{"hostname":"","public-ipv4":"192.168.1.2","public-ipv6":"fe00::","public_keys":{"0":"publickey1","1":"publickey2"}}`),
+		},
+		{
+			clientErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+		},
+	} {
+		service := &metadataService{
+			MetadataService: metadata.MetadataService{
+				Root:         tt.root,
+				Client:       &test.HttpClient{tt.resources, tt.clientErr},
+				MetadataPath: tt.metadataPath,
+			},
+		}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(metadata, tt.expect) {
+			t.Fatalf("bad fetch (%q): want %q, got %q", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

initialize/config.go

@@ -263,6 +263,8 @@ func Apply(cfg CloudConfig, env *Environment) error {
 		switch env.NetconfType() {
 		case "debian":
 			interfaces, err = network.ProcessDebianNetconf(cfg.NetworkConfig)
+		case "digitalocean":
+			interfaces, err = network.ProcessDigitalOceanNetconf(cfg.NetworkConfig)
 		default:
 			return fmt.Errorf("Unsupported network config format %q", env.NetconfType())
 		}