Some changes to calls to Docker API for podman compatibility (#463)

* Some changes to calls to Docker API for podman compatibility

- Podman expects the network name specified when launching the container
to match the name used when creating it, the spec is unclear about that
- Adding labels when commiting a container needs to use "changes" query
parameter instead of a POST json, which is unspecified
- Add a config to specify the hostname where Complement on the host can
be reached when inside a container. Podman uses "host.containers.internal"

* Use quotes for labels in changes

* Remove debug + makes AS labels on one line

* Remove hack, not needed anymore

* a bit too far...

* Add doc

* Convert from labels to changes

* Replace networkID by networkName

* Add/fix some comments

* Add comments

Author: Mathieu Velten

README.md

@@ -54,6 +54,17 @@ If you are using [ufw](https://code.launchpad.net/ufw), this can be done with:
 sudo ufw allow in on br-+
 ```
 
+### Running using Podman
+
+It is possible to run the test suite using Podman and the compatibility layer for Docker API.
+Rootless mode is also supported.
+
+To do so you should:
+- `systemctl --user start podman.service` to start the rootless API daemon (can also be enabled).
+- `DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock BUILDAH_FORMAT=docker COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal ...`
+
+Docker image format is needed because OCI format doesn't support the HEALTHCHECK directive unfortunately.
+
 ### Running against Dendrite
 
 For instance, for Dendrite:

internal/config/config.go

@@ -78,6 +78,13 @@ type Complement struct {
 	CAPrivateKey  *rsa.PrivateKey
 
 	BestEffort bool
+
+	// Name: COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT
+	// Default: host.docker.internal
+	// Description: The hostname of Complement from the perspective of a Homeserver running inside a container.
+	// This can be useful for container runtimes using another hostname to access the host from a container,
+	// like Podman that uses `host.containers.internal` instead.
+	HostnameRunningComplement string
 }
 
 var hsRegex = regexp.MustCompile(`COMPLEMENT_BASE_IMAGE_(.+)=(.+)$`)
@@ -129,6 +136,13 @@ func NewConfigFromEnvVars(pkgNamespace, baseImageURI string) *Complement {
 		panic("package namespace must be set")
 	}
 
+	HostnameRunningComplement := os.Getenv("COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT")
+	if HostnameRunningComplement != "" {
+		cfg.HostnameRunningComplement = HostnameRunningComplement
+	} else {
+		cfg.HostnameRunningComplement = "host.docker.internal"
+	}
+
 	return cfg
 }
 

internal/docker/builder.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/go-connections/nat"
@@ -32,8 +31,6 @@ import (
 )
 
 var (
-	// HostnameRunningComplement is the hostname of Complement from the perspective of a Homeserver.
-	HostnameRunningComplement = "host.docker.internal"
 	// HostnameRunningDocker is the hostname of the docker daemon from the perspective of Complement.
 	HostnameRunningDocker = "localhost"
 )
@@ -240,15 +237,15 @@ func (d *Builder) ConstructBlueprint(bprint b.Blueprint) error {
 func (d *Builder) construct(bprint b.Blueprint) (errs []error) {
 	d.log("Constructing blueprint '%s'", bprint.Name)
 
-	networkID, err := createNetworkIfNotExists(d.Docker, d.Config.PackageNamespace, bprint.Name)
+	networkName, err := createNetworkIfNotExists(d.Docker, d.Config.PackageNamespace, bprint.Name)
 	if err != nil {
 		return []error{err}
 	}
 
 	runner := instruction.NewRunner(bprint.Name, d.Config.BestEffort, d.Config.DebugLoggingEnabled)
 	results := make([]result, len(bprint.Homeservers))
 	for i, hs := range bprint.Homeservers {
-		res := d.constructHomeserver(bprint.Name, runner, hs, networkID)
+		res := d.constructHomeserver(bprint.Name, runner, hs, networkName)
 		if res.err != nil {
 			errs = append(errs, res.err)
 			if res.containerID != "" {
@@ -336,9 +333,7 @@ func (d *Builder) construct(bprint b.Blueprint) (errs []error) {
 			Author:    "Complement",
 			Pause:     true,
 			Reference: "localhost/complement:" + res.contextStr,
-			Config: &container.Config{
-				Labels: labels,
-			},
+			Changes:   toChanges(labels),
 		})
 		if err != nil {
 			d.log("%s : failed to ContainerCommit: %s\n", res.contextStr, err)
@@ -351,11 +346,22 @@ func (d *Builder) construct(bprint b.Blueprint) (errs []error) {
 	return errs
 }
 
+// Convert a map of labels to a list of changes directive in Dockerfile format.
+// Labels keys and values can't be multiline (eg. can't contain `\n` character)
+// neither can they contain unescaped `"` character.
+func toChanges(labels map[string]string) []string {
+	var changes []string
+	for k, v := range labels {
+		changes = append(changes, fmt.Sprintf("LABEL \"%s\"=\"%s\"", k, v))
+	}
+	return changes
+}
+
 // construct this homeserver and execute its instructions, keeping the container alive.
-func (d *Builder) constructHomeserver(blueprintName string, runner *instruction.Runner, hs b.Homeserver, networkID string) result {
+func (d *Builder) constructHomeserver(blueprintName string, runner *instruction.Runner, hs b.Homeserver, networkName string) result {
 	contextStr := fmt.Sprintf("%s.%s.%s", d.Config.PackageNamespace, blueprintName, hs.Name)
 	d.log("%s : constructing homeserver...\n", contextStr)
-	dep, err := d.deployBaseImage(blueprintName, hs, contextStr, networkID)
+	dep, err := d.deployBaseImage(blueprintName, hs, contextStr, networkName)
 	if err != nil {
 		log.Printf("%s : failed to deployBaseImage: %s\n", contextStr, err)
 		containerID := ""
@@ -383,7 +389,7 @@ func (d *Builder) constructHomeserver(blueprintName string, runner *instruction.
 }
 
 // deployBaseImage runs the base image and returns the baseURL, containerID or an error.
-func (d *Builder) deployBaseImage(blueprintName string, hs b.Homeserver, contextStr, networkID string) (*HomeserverDeployment, error) {
+func (d *Builder) deployBaseImage(blueprintName string, hs b.Homeserver, contextStr, networkName string) (*HomeserverDeployment, error) {
 	asIDToRegistrationMap := asIDToRegistrationFromLabels(labelsForApplicationServices(hs))
 	var baseImageURI string
 	if hs.BaseImageURI == nil {
@@ -399,28 +405,29 @@ func (d *Builder) deployBaseImage(blueprintName string, hs b.Homeserver, context
 	return deployImage(
 		d.Docker, baseImageURI, fmt.Sprintf("complement_%s", contextStr),
 		d.Config.PackageNamespace, blueprintName, hs.Name, asIDToRegistrationMap, contextStr,
-		networkID, d.Config,
+		networkName, d.Config,
 	)
 }
 
+// Multilines label using Dockerfile syntax is unsupported, let's inline \n instead
 func generateASRegistrationYaml(as b.ApplicationService) string {
-	return fmt.Sprintf("id: %s\n", as.ID) +
-		fmt.Sprintf("hs_token: %s\n", as.HSToken) +
-		fmt.Sprintf("as_token: %s\n", as.ASToken) +
-		fmt.Sprintf("url: '%s'\n", as.URL) +
-		fmt.Sprintf("sender_localpart: %s\n", as.SenderLocalpart) +
-		fmt.Sprintf("rate_limited: %v\n", as.RateLimited) +
-		"namespaces:\n" +
-		"  users:\n" +
-		"    - exclusive: false\n" +
-		"      regex: .*\n" +
-		"  rooms: []\n" +
-		"  aliases: []\n"
+	return fmt.Sprintf("id: %s\\n", as.ID) +
+		fmt.Sprintf("hs_token: %s\\n", as.HSToken) +
+		fmt.Sprintf("as_token: %s\\n", as.ASToken) +
+		fmt.Sprintf("url: '%s'\\n", as.URL) +
+		fmt.Sprintf("sender_localpart: %s\\n", as.SenderLocalpart) +
+		fmt.Sprintf("rate_limited: %v\\n", as.RateLimited) +
+		"namespaces:\\n" +
+		"  users:\\n" +
+		"    - exclusive: false\\n" +
+		"      regex: .*\\n" +
+		"  rooms: []\\n" +
+		"  aliases: []\\n"
 }
 
-// createNetworkIfNotExists creates a docker network and returns its id.
-// ID is guaranteed not to be empty when err == nil
-func createNetworkIfNotExists(docker *client.Client, pkgNamespace, blueprintName string) (networkID string, err error) {
+// createNetworkIfNotExists creates a docker network and returns its name.
+// Name is guaranteed not to be empty when err == nil
+func createNetworkIfNotExists(docker *client.Client, pkgNamespace, blueprintName string) (networkName string, err error) {
 	// check if a network already exists for this blueprint
 	nws, err := docker.NetworkList(context.Background(), types.NetworkListOptions{
 		Filters: label(
@@ -436,10 +443,11 @@ func createNetworkIfNotExists(docker *client.Client, pkgNamespace, blueprintName
 		if len(nws) > 1 {
 			log.Printf("WARNING: createNetworkIfNotExists got %d networks for pkg=%s blueprint=%s", len(nws), pkgNamespace, blueprintName)
 		}
-		return nws[0].ID, nil
+		return nws[0].Name, nil
 	}
+	networkName = "complement_" + pkgNamespace + "_" + blueprintName
 	// make a user-defined network so we get DNS based on the container name
-	nw, err := docker.NetworkCreate(context.Background(), "complement_"+pkgNamespace+"_"+blueprintName, types.NetworkCreate{
+	nw, err := docker.NetworkCreate(context.Background(), networkName, types.NetworkCreate{
 		Labels: map[string]string{
 			complementLabel:        blueprintName,
 			"complement_blueprint": blueprintName,
@@ -458,7 +466,7 @@ func createNetworkIfNotExists(docker *client.Client, pkgNamespace, blueprintName
 	if nw.ID == "" {
 		return "", fmt.Errorf("%s: unexpected empty ID while creating networkID", blueprintName)
 	}
-	return nw.ID, nil
+	return networkName, nil
 }
 
 func printLogs(docker *client.Client, containerID, contextStr string) {

internal/docker/deployer.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -49,7 +49,6 @@ type Deployer struct {
 	DeployNamespace string
 	Docker          *client.Client
 	Counter         int
-	networkID       string
 	debugLogging    bool
 	config          *config.Complement
 }
@@ -93,11 +92,10 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen
 	if len(images) == 0 {
 		return nil, fmt.Errorf("Deploy: No images have been built for blueprint %s", blueprintName)
 	}
-	networkID, err := createNetworkIfNotExists(d.Docker, d.config.PackageNamespace, blueprintName)
+	networkName, err := createNetworkIfNotExists(d.Docker, d.config.PackageNamespace, blueprintName)
 	if err != nil {
 		return nil, fmt.Errorf("Deploy: %w", err)
 	}
-	d.networkID = networkID
 
 	// deploy images in parallel
 	var mu sync.Mutex // protects mutable values like the counter and errors
@@ -116,7 +114,7 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen
 		// TODO: Make CSAPI port configurable
 		deployment, err := deployImage(
 			d.Docker, img.ID, fmt.Sprintf("complement_%s_%s_%s_%d", d.config.PackageNamespace, d.DeployNamespace, contextStr, counter),
-			d.config.PackageNamespace, blueprintName, hsName, asIDToRegistrationMap, contextStr, networkID, d.config,
+			d.config.PackageNamespace, blueprintName, hsName, asIDToRegistrationMap, contextStr, networkName, d.config,
 		)
 		if err != nil {
 			if deployment != nil && deployment.ContainerID != "" {
@@ -184,14 +182,6 @@ func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement)
 		return fmt.Errorf("Restart: Failed to stop container %s: %s", hsDep.ContainerID, err)
 	}
 
-	// Remove the container from the network. If we don't do this,
-	// (re)starting the container fails with an error like
-	// "Error response from daemon: endpoint with name complement_fed_1_fed.alice.hs1_1 already exists in network complement_fed_alice".
-	err = d.Docker.NetworkDisconnect(ctx, d.networkID, hsDep.ContainerID, false)
-	if err != nil {
-		return fmt.Errorf("Restart: Failed to disconnect container %s: %s", hsDep.ContainerID, err)
-	}
-
 	err = d.Docker.ContainerStart(ctx, hsDep.ContainerID, types.ContainerStartOptions{})
 	if err != nil {
 		return fmt.Errorf("Restart: Failed to start container %s: %s", hsDep.ContainerID, err)
@@ -216,7 +206,7 @@ func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement)
 // nolint
 func deployImage(
 	docker *client.Client, imageID string, containerName, pkgNamespace, blueprintName, hsName string,
-	asIDToRegistrationMap map[string]string, contextStr, networkID string, cfg *config.Complement,
+	asIDToRegistrationMap map[string]string, contextStr, networkName string, cfg *config.Complement,
 ) (*HomeserverDeployment, error) {
 	ctx := context.Background()
 	var extraHosts []string
@@ -283,9 +273,8 @@ func deployImage(
 		Mounts:     mounts,
 	}, &network.NetworkingConfig{
 		EndpointsConfig: map[string]*network.EndpointSettings{
-			contextStr: {
-				NetworkID: networkID,
-				Aliases:   []string{hsName},
+			networkName: {
+				Aliases: []string{hsName},
 			},
 		},
 	}, nil, containerName)
@@ -298,7 +287,7 @@ func deployImage(
 
 	containerID := body.ID
 	if cfg.DebugLoggingEnabled {
-		log.Printf("%s: Created container '%s' using image '%s' on network '%s'", contextStr, containerID, imageID, networkID)
+		log.Printf("%s: Created container '%s' using image '%s' on network '%s'", contextStr, containerID, imageID, networkName)
 	}
 	stubDeployment := &HomeserverDeployment{
 		ContainerID: containerID,

internal/docker/labels.go

@@ -33,7 +33,8 @@ func asIDToRegistrationFromLabels(labels map[string]string) map[string]string {
 	asMap := make(map[string]string)
 	for k, v := range labels {
 		if strings.HasPrefix(k, "application_service_") {
-			asMap[strings.TrimPrefix(k, "application_service_")] = v
+			// cf comment of generateASRegistrationYaml for ReplaceAll explanation
+			asMap[strings.TrimPrefix(k, "application_service_")] = strings.ReplaceAll(v, "\\n", "\n")
 		}
 	}
 	return asMap

internal/federation/server.go

@@ -69,7 +69,7 @@ func NewServer(t *testing.T, deployment *docker.Deployment, opts ...func(*Server
 		mux:   mux.NewRouter(),
 		// The server name will be updated when the caller calls Listen() to include the port number
 		// of the HTTP server e.g "host.docker.internal:56353"
-		serverName:                  docker.HostnameRunningComplement,
+		serverName:                  deployment.Config.HostnameRunningComplement,
 		rooms:                       make(map[string]*ServerRoom),
 		aliases:                     make(map[string]string),
 		UnexpectedRequestsAreErrors: true,
@@ -476,10 +476,10 @@ func federationServer(cfg *config.Complement, h http.Handler) (*http.Server, str
 			Locality:      []string{"London"},
 			StreetAddress: []string{"123 Street"},
 			PostalCode:    []string{"12345"},
-			CommonName:    docker.HostnameRunningComplement,
+			CommonName:    cfg.HostnameRunningComplement,
 		},
 	}
-	host := docker.HostnameRunningComplement
+	host := cfg.HostnameRunningComplement
 	if ip := net.ParseIP(host); ip != nil {
 		template.IPAddresses = append(template.IPAddresses, ip)
 	} else {

internal/federation/server_test.go

@@ -11,8 +11,8 @@ import (
 )
 
 func TestComplementServerIsSigned(t *testing.T) {
-	docker.HostnameRunningComplement = "localhost"
 	cfg := config.NewConfigFromEnvVars("test", "unimportant")
+	cfg.HostnameRunningComplement = "localhost"
 	srv := NewServer(t, &docker.Deployment{
 		Config: cfg,
 	})

tests/federation_room_join_test.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/matrix-org/complement/internal/b"
 	"github.com/matrix-org/complement/internal/client"
-	"github.com/matrix-org/complement/internal/docker"
 	"github.com/matrix-org/complement/internal/federation"
 	"github.com/matrix-org/complement/internal/match"
 	"github.com/matrix-org/complement/internal/must"
@@ -157,7 +156,7 @@ func TestJoinFederatedRoomWithUnverifiableEvents(t *testing.T) {
 			},
 		})
 		newSignaturesBlock := map[string]interface{}{
-			docker.HostnameRunningComplement: map[string]string{
+			deployment.Config.HostnameRunningComplement: map[string]string{
 				string(srv.KeyID): "/3z+pJjiJXWhwfqIEzmNksvBHCoXTktK/y0rRuWJXw6i1+ygRG/suDCKhFuuz6gPapRmEMPVILi2mJqHHXPKAg",
 			},
 		}
@@ -186,7 +185,7 @@ func TestJoinFederatedRoomWithUnverifiableEvents(t *testing.T) {
 			},
 		})
 		newSignaturesBlock := map[string]interface{}{
-			docker.HostnameRunningComplement: map[string]string{
+			deployment.Config.HostnameRunningComplement: map[string]string{
 				string(srv.KeyID) + "bogus": "/3z+pJjiJXWhwfqIEzmNksvBHCoXTktK/y0rRuWJXw6i1+ygRG/suDCKhFuuz6gPapRmEMPVILi2mJqHHXPKAg",
 			},
 		}
@@ -216,7 +215,7 @@ func TestJoinFederatedRoomWithUnverifiableEvents(t *testing.T) {
 			},
 		}).JSON()
 		rawSig, err := json.Marshal(map[string]interface{}{
-			docker.HostnameRunningComplement: map[string]string{
+			deployment.Config.HostnameRunningComplement: map[string]string{
 				string(srv.KeyID): "/3z+pJjiJXWhwfqIEzmNksvBHCoXTktK/y0rRuWJXw6i1+ygRG/suDCKhFuuz6gPapRmEMPVILi2mJqHHXPKAg",
 			},
 		})

tests/federation_room_send_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/complement/internal/b"
-	"github.com/matrix-org/complement/internal/docker"
 	"github.com/matrix-org/complement/internal/federation"
 )
 
@@ -53,7 +52,7 @@ func TestOutboundFederationSend(t *testing.T) {
 	roomAlias := srv.MakeAliasMapping("flibble", serverRoom.RoomID)
 
 	// the local homeserver joins the room
-	alice.JoinRoom(t, roomAlias, []string{docker.HostnameRunningComplement})
+	alice.JoinRoom(t, roomAlias, []string{deployment.Config.HostnameRunningComplement})
 
 	// the local homeserver sends an event into the room
 	alice.SendEventSynced(t, serverRoom.RoomID, b.Event{