Merge pull request #339 from vdemeester/v2-volumes

 Add support to volume v2 🐡

Author: vdemeester

config/marshal_config_test.go

@@ -46,14 +46,37 @@ func newTestConfig() TestConfig {
 					"io.rancher.os.createonly": "true",
 					"io.rancher.os.scope":      "system",
 				},
-				Volumes: []string{
-					"/dev:/host/dev",
-					"/var/lib/rancher/conf:/var/lib/rancher/conf",
-					"/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher",
-					"/lib/modules:/lib/modules",
-					"/lib/firmware:/lib/firmware",
-					"/var/run:/var/run",
-					"/var/log:/var/log",
+				Volumes: &yamlTypes.Volumes{
+					Volumes: []*yamlTypes.Volume{
+						{
+							Source:      "/dev",
+							Destination: "/host/dev",
+						},
+						{
+							Source:      "/var/lib/rancher/conf",
+							Destination: "/var/lib/rancher/conf",
+						},
+						{
+							Source:      "/etc/ssl/certs/ca-certificates.crt",
+							Destination: "/etc/ssl/certs/ca-certificates.crt.rancher",
+						},
+						{
+							Source:      "/lib/modules",
+							Destination: "lib/modules",
+						},
+						{
+							Source:      "/lib/firmware",
+							Destination: "/lib/firmware",
+						},
+						{
+							Source:      "/var/run",
+							Destination: "/var/run",
+						},
+						{
+							Source:      "/var/log",
+							Destination: "/var/log",
+						},
+					},
 				},
 				Logging: Log{
 					Driver: "json-file",

config/types.go

@@ -116,7 +116,7 @@ type ServiceConfig struct {
 	ShmSize       yaml.StringorInt     `yaml:"shm_size,omitempty"`
 	StopSignal    string               `yaml:"stop_signal,omitempty"`
 	VolumeDriver  string               `yaml:"volume_driver,omitempty"`
-	Volumes       []string             `yaml:"volumes,omitempty"`
+	Volumes       *yaml.Volumes        `yaml:"volumes,omitempty"`
 	VolumesFrom   []string             `yaml:"volumes_from,omitempty"`
 	Uts           string               `yaml:"uts,omitempty"`
 	Restart       string               `yaml:"restart,omitempty"`

docker/convert.go

@@ -16,6 +16,7 @@ import (
 	composeclient "github.com/docker/libcompose/docker/client"
 	"github.com/docker/libcompose/project"
 	"github.com/docker/libcompose/utils"
+	// "github.com/docker/libcompose/yaml"
 )
 
 // ConfigWrapper wraps Config, HostConfig and NetworkingConfig for a container.
@@ -36,6 +37,16 @@ func Filter(vs []string, f func(string) bool) []string {
 	return r
 }
 
+func toMap(vs []string) map[string]struct{} {
+	m := map[string]struct{}{}
+	for _, v := range vs {
+		if v != "" {
+			m[v] = struct{}{}
+		}
+	}
+	return m
+}
+
 func isBind(s string) bool {
 	return strings.ContainsRune(s, ':')
 }
@@ -58,21 +69,18 @@ func ConvertToAPI(serviceConfig *config.ServiceConfig, ctx project.Context, clie
 	return &result, nil
 }
 
-func isNamedVolume(volume string) bool {
-	return !strings.HasPrefix(volume, ".") && !strings.HasPrefix(volume, "/") && !strings.HasPrefix(volume, "~")
-}
-
-func volumes(c *config.ServiceConfig, ctx project.Context) map[string]struct{} {
-	volumes := make(map[string]struct{}, len(c.Volumes))
-	for k, v := range c.Volumes {
-		if len(ctx.ComposeFiles) > 0 && !isNamedVolume(v) {
-			v = ctx.ResourceLookup.ResolvePath(v, ctx.ComposeFiles[0])
-		}
-
-		c.Volumes[k] = v
-		if isVolume(v) {
-			volumes[v] = struct{}{}
+func volumes(c *config.ServiceConfig, ctx project.Context) []string {
+	if c.Volumes == nil {
+		return []string{}
+	}
+	volumes := make([]string, len(c.Volumes.Volumes))
+	for _, v := range c.Volumes.Volumes {
+		vol := v
+		if len(ctx.ComposeFiles) > 0 && !project.IsNamedVolume(v.Source) {
+			sourceVol := ctx.ResourceLookup.ResolvePath(v.String(), ctx.ComposeFiles[0])
+			vol.Source = strings.SplitN(sourceVol, ":", 2)[0]
 		}
+		volumes = append(volumes, vol.String())
 	}
 	return volumes
 }
@@ -141,6 +149,8 @@ func Convert(c *config.ServiceConfig, ctx project.Context, clientFactory compose
 		}
 	}
 
+	vols := volumes(c, ctx)
+
 	config := &container.Config{
 		Entrypoint:   strslice.StrSlice(utils.CopySlice(c.Entrypoint)),
 		Hostname:     c.Hostname,
@@ -154,7 +164,7 @@ func Convert(c *config.ServiceConfig, ctx project.Context, clientFactory compose
 		Tty:          c.Tty,
 		OpenStdin:    c.StdinOpen,
 		WorkingDir:   c.WorkingDir,
-		Volumes:      volumes(c, ctx),
+		Volumes:      toMap(Filter(vols, isVolume)),
 		MacAddress:   c.MacAddress,
 	}
 
@@ -228,7 +238,7 @@ func Convert(c *config.ServiceConfig, ctx project.Context, clientFactory compose
 		CapDrop:     strslice.StrSlice(utils.CopySlice(c.CapDrop)),
 		ExtraHosts:  utils.CopySlice(c.ExtraHosts),
 		Privileged:  c.Privileged,
-		Binds:       Filter(c.Volumes, isBind),
+		Binds:       Filter(vols, isBind),
 		DNS:         utils.CopySlice(c.DNS),
 		DNSSearch:   utils.CopySlice(c.DNSSearch),
 		LogConfig: container.LogConfig{

docker/convert_test.go

@@ -26,7 +26,29 @@ func TestParseBindsAndVolumes(t *testing.T) {
 	abs, err := filepath.Abs(".")
 	assert.Nil(t, err)
 	cfg, hostCfg, err := Convert(&config.ServiceConfig{
-		Volumes: []string{"/foo", "/home:/home", "/bar/baz", ".:/home", "/usr/lib:/usr/lib:ro"},
+		Volumes: &yaml.Volumes{
+			Volumes: []*yaml.Volume{
+				{
+					Destination: "/foo",
+				},
+				{
+					Source:      "/home",
+					Destination: "/home",
+				},
+				{
+					Destination: "/bar/baz",
+				},
+				{
+					Source:      ".",
+					Destination: "/home",
+				},
+				{
+					Source:      "/usr/lib",
+					Destination: "/usr/lib",
+					AccessMode:  "ro",
+				},
+			},
+		},
 	}, ctx.Context, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, map[string]struct{}{"/foo": {}, "/bar/baz": {}}, cfg.Volumes)

docker/project.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/docker/client"
 	"github.com/docker/libcompose/docker/network"
+	"github.com/docker/libcompose/docker/volume"
 	"github.com/docker/libcompose/labels"
 	"github.com/docker/libcompose/lookup"
 	"github.com/docker/libcompose/project"
@@ -66,6 +67,13 @@ func NewProject(context *Context, parseOptions *config.ParseOptions) (project.AP
 		context.NetworksFactory = networksFactory
 	}
 
+	if context.VolumesFactory == nil {
+		volumesFactory := &volume.DockerFactory{
+			ClientFactory: context.ClientFactory,
+		}
+		context.VolumesFactory = volumesFactory
+	}
+
 	// FIXME(vdemeester) Remove the context duplication ?
 	runtime := &Project{
 		clientFactory: context.ClientFactory,

docker/volume/volume.go

@@ -0,0 +1,154 @@
+package volume
+
+import (
+	"fmt"
+
+	"golang.org/x/net/context"
+
+	"github.com/docker/engine-api/client"
+	"github.com/docker/engine-api/types"
+	"github.com/docker/libcompose/config"
+
+	composeclient "github.com/docker/libcompose/docker/client"
+	"github.com/docker/libcompose/project"
+)
+
+// Volume holds attributes and method for a volume definition in compose
+type Volume struct {
+	client        client.VolumeAPIClient
+	projectName   string
+	name          string
+	driver        string
+	driverOptions map[string]string
+	external      bool
+	// TODO (shouze) missing labels
+}
+
+func (v *Volume) fullName() string {
+	name := v.projectName + "_" + v.name
+	if v.external {
+		name = v.name
+	}
+	return name
+}
+
+// Inspect inspect the current volume
+func (v *Volume) Inspect(ctx context.Context) (types.Volume, error) {
+	return v.client.VolumeInspect(ctx, v.fullName())
+}
+
+// Remove removes the current volume (from docker engine)
+func (v *Volume) Remove(ctx context.Context) error {
+	if v.external {
+		fmt.Printf("Volume %s is external, skipping", v.fullName())
+		return nil
+	}
+	fmt.Printf("Removing volume %q\n", v.fullName())
+	return v.client.VolumeRemove(ctx, v.fullName())
+}
+
+// EnsureItExists make sure the volume exists and return an error if it does not exists
+// and cannot be created.
+func (v *Volume) EnsureItExists(ctx context.Context) error {
+	volumeResource, err := v.Inspect(ctx)
+	if v.external {
+		if client.IsErrVolumeNotFound(err) {
+			// FIXME(shouze) introduce some libcompose error type
+			return fmt.Errorf("Volume %s declared as external, but could not be found. Please create the volume manually using docker volume create %s and try again", v.name, v.name)
+		}
+		return err
+	}
+	if err != nil && client.IsErrVolumeNotFound(err) {
+		return v.create(ctx)
+	}
+	if volumeResource.Driver != v.driver {
+		return fmt.Errorf("Volume %q needs to be recreated - driver has changed", v.name)
+	}
+	return err
+}
+
+func (v *Volume) create(ctx context.Context) error {
+	fmt.Printf("Creating volume %q with driver %q\n", v.fullName(), v.driver)
+	_, err := v.client.VolumeCreate(ctx, types.VolumeCreateRequest{
+		Name:       v.fullName(),
+		Driver:     v.driver,
+		DriverOpts: v.driverOptions,
+		// TODO (shouze) missing labels
+	})
+
+	return err
+}
+
+// NewVolume creates a new volume from the specified name and config.
+func NewVolume(projectName, name string, config *config.VolumeConfig, client client.VolumeAPIClient) *Volume {
+	return &Volume{
+		client:        client,
+		projectName:   projectName,
+		name:          name,
+		driver:        config.Driver,
+		driverOptions: config.DriverOpts,
+		external:      config.External.External,
+	}
+}
+
+// Volumes holds a list of volume
+type Volumes struct {
+	volumes       []*Volume
+	volumeEnabled bool
+}
+
+// Initialize make sure volume exists if volume is enabled
+func (v *Volumes) Initialize(ctx context.Context) error {
+	if !v.volumeEnabled {
+		return nil
+	}
+	for _, volume := range v.volumes {
+		err := volume.EnsureItExists(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Remove removes volumes (clean-up)
+func (v *Volumes) Remove(ctx context.Context) error {
+	if !v.volumeEnabled {
+		return nil
+	}
+	for _, volume := range v.volumes {
+		err := volume.Remove(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// VolumesFromServices creates a new Volumes struct based on volumes configurations and
+// services configuration. If a volume is defined but not used by any service, it will return
+// an error along the Volumes.
+func VolumesFromServices(cli client.VolumeAPIClient, projectName string, volumeConfigs map[string]*config.VolumeConfig, services *config.ServiceConfigs, volumeEnabled bool) (*Volumes, error) {
+	var err error
+	volumes := make([]*Volume, 0, len(volumeConfigs))
+	for name, config := range volumeConfigs {
+		volume := NewVolume(projectName, name, config, cli)
+		volumes = append(volumes, volume)
+	}
+	return &Volumes{
+		volumes:       volumes,
+		volumeEnabled: volumeEnabled,
+	}, err
+}
+
+// DockerFactory implements project.VolumesFactory
+type DockerFactory struct {
+	ClientFactory composeclient.Factory
+}
+
+// Create implements project.VolumesFactory Create method.
+// It creates a Volumes (that implements project.Volumes) from specified configurations.
+func (f *DockerFactory) Create(projectName string, volumeConfigs map[string]*config.VolumeConfig, serviceConfigs *config.ServiceConfigs, volumeEnabled bool) (project.Volumes, error) {
+	cli := f.ClientFactory.Create(nil)
+	return VolumesFromServices(cli, projectName, volumeConfigs, serviceConfigs, volumeEnabled)
+}