Introduce the concept of a package namespace (#169)

* Introduce the concept of a package namespace

This is the beginning of the work for #119.

 - Add the concept of `PackageNamespace` to `config.Complement`. Set to `""` for now.
 - Update all filters to look for a matching package namespace when creating blueprints
   and containers.
 - Cleanup `docker.Builder` to read from the config more and from copies of config fields less.

This change should be invisible to any existing Complement users.

Background:

Previously, Complement assumed that each container could be uniquely
identified by the combination of `deployment_namespace + blueprint_name + hs_name`.
For example, if you were running `BlueprintAlice` then a unique string might look like
`5_alice_hs1` where `5` is an atomic, monotonically increasing integer incremented when
`Deploy()` is called (see #113 for more info).

In a parallel by default world this is no longer true because the `deployment_namespace`
is not shared between different test processes. This means we cannot co-ordinate non-clashing
namespaces like before. Instead, we will bring in another namespace for the test process
(which in #119 will be on a per-package basis, hence the name `PackageNamespace`).

As of this PR, literally everything Complement makes (images, containers, networks, etc) are prefixed with
this package namespace, which allows multiple complement instances to share the same underlying
docker daemon, with caveats:
 - Creating CA certificates will race and needs a lockfile to prevent 2 processes trying to create
  the certificate at the same time.
 - Complement federation servers cannot run together due to trying to bind to `:8448` at the same time.

That being said, this PR should enable the parallelisation of a number of CS API only tests,
which will come in another PR.

* Set to pkg to keep the ref format of committed containers valid

Author: kegsay

README.md

@@ -131,7 +131,8 @@ update-ca-certificates
 ## Sytest parity
 
 ```
-$ go run sytest_coverage.go -v
+$ go build ./cmd/sytest-coverage
+$ ./sytest-coverage -v
 10apidoc/01register 3/9 tests
     × GET /register yields a set of flows
     ✓ POST /register can create a user

sytest_coverage.go renamed to cmd/sytest-coverage/main.go

@@ -14,6 +14,8 @@ type Complement struct {
 	BestEffort             bool
 	VersionCheckIterations int
 	KeepBlueprints         []string
+	// The namespace for all complement created blueprints and deployments
+	PackageNamespace string
 }
 
 func NewConfigFromEnvVars() *Complement {
@@ -27,6 +29,7 @@ func NewConfigFromEnvVars() *Complement {
 	if cfg.BaseImageURI == "" {
 		panic("COMPLEMENT_BASE_IMAGE must be set")
 	}
+	cfg.PackageNamespace = "pkg"
 	return cfg
 }
 

internal/config/config.go

@@ -63,15 +63,10 @@ func init() {
 const complementLabel = "complement_context"
 
 type Builder struct {
-	BaseImage      string
-	ImageArgs      []string
-	KeepBlueprints []string
+	Config         *config.Complement
 	CSAPIPort      int
 	FederationPort int
 	Docker         *client.Client
-	debugLogging   bool
-	bestEffort     bool
-	config         *config.Complement
 }
 
 func NewBuilder(cfg *config.Complement) (*Builder, error) {
@@ -81,19 +76,14 @@ func NewBuilder(cfg *config.Complement) (*Builder, error) {
 	}
 	return &Builder{
 		Docker:         cli,
-		BaseImage:      cfg.BaseImageURI,
-		ImageArgs:      cfg.BaseImageArgs,
-		KeepBlueprints: cfg.KeepBlueprints,
+		Config:         cfg,
 		CSAPIPort:      8008,
 		FederationPort: 8448,
-		debugLogging:   cfg.DebugLoggingEnabled,
-		bestEffort:     cfg.BestEffort,
-		config:         cfg,
 	}, nil
 }
 
 func (d *Builder) log(str string, args ...interface{}) {
-	if !d.debugLogging {
+	if !d.Config.DebugLoggingEnabled {
 		return
 	}
 	log.Printf(str, args...)
@@ -117,7 +107,10 @@ func (d *Builder) Cleanup() {
 // removeImages removes all images with `complementLabel`.
 func (d *Builder) removeNetworks() error {
 	networks, err := d.Docker.NetworkList(context.Background(), types.NetworkListOptions{
-		Filters: label(complementLabel),
+		Filters: label(
+			complementLabel,
+			"complement_pkg="+d.Config.PackageNamespace,
+		),
 	})
 	if err != nil {
 		return err
@@ -134,7 +127,10 @@ func (d *Builder) removeNetworks() error {
 // removeImages removes all images with `complementLabel`.
 func (d *Builder) removeImages() error {
 	images, err := d.Docker.ImageList(context.Background(), types.ImageListOptions{
-		Filters: label(complementLabel),
+		Filters: label(
+			complementLabel,
+			"complement_pkg="+d.Config.PackageNamespace,
+		),
 	})
 	if err != nil {
 		return err
@@ -156,7 +152,7 @@ func (d *Builder) removeImages() error {
 		}
 		bprintName := img.Labels["complement_blueprint"]
 		keep := false
-		for _, keepBprint := range d.KeepBlueprints {
+		for _, keepBprint := range d.Config.KeepBlueprints {
 			if bprintName == keepBprint {
 				keep = true
 				break
@@ -180,8 +176,11 @@ func (d *Builder) removeImages() error {
 // removeContainers removes all containers with `complementLabel`.
 func (d *Builder) removeContainers() error {
 	containers, err := d.Docker.ContainerList(context.Background(), types.ContainerListOptions{
-		All:     true,
-		Filters: label(complementLabel),
+		All: true,
+		Filters: label(
+			complementLabel,
+			"complement_pkg="+d.Config.PackageNamespace,
+		),
 	})
 	if err != nil {
 		return err
@@ -201,7 +200,10 @@ func (d *Builder) ConstructBlueprintsIfNotExist(bs []b.Blueprint) error {
 	var blueprintsToBuild []b.Blueprint
 	for _, bprint := range bs {
 		images, err := d.Docker.ImageList(context.Background(), types.ImageListOptions{
-			Filters: label("complement_blueprint=" + bprint.Name),
+			Filters: label(
+				"complement_blueprint="+bprint.Name,
+				"complement_pkg="+d.Config.PackageNamespace,
+			),
 		})
 		if err != nil {
 			return fmt.Errorf("ConstructBlueprintsIfNotExist: failed to ImageList: %w", err)
@@ -242,7 +244,10 @@ func (d *Builder) ConstructBlueprints(bs []b.Blueprint) error {
 	foundImages := false
 	for i := 0; i < 50; i++ { // max 5s
 		images, err := d.Docker.ImageList(context.Background(), types.ImageListOptions{
-			Filters: label(complementLabel),
+			Filters: label(
+				complementLabel,
+				"complement_pkg="+d.Config.PackageNamespace,
+			),
 		})
 		if err != nil {
 			return err
@@ -265,12 +270,12 @@ func (d *Builder) ConstructBlueprints(bs []b.Blueprint) error {
 
 // construct all Homeservers sequentially then commits them
 func (d *Builder) construct(bprint b.Blueprint) (errs []error) {
-	networkID, err := CreateNetworkIfNotExists(d.Docker, bprint.Name)
+	networkID, err := createNetworkIfNotExists(d.Docker, d.Config.PackageNamespace, bprint.Name)
 	if err != nil {
 		return []error{err}
 	}
 
-	runner := instruction.NewRunner(bprint.Name, d.bestEffort, d.debugLogging)
+	runner := instruction.NewRunner(bprint.Name, d.Config.BestEffort, d.Config.DebugLoggingEnabled)
 	results := make([]result, len(bprint.Homeservers))
 	for i, hs := range bprint.Homeservers {
 		res := d.constructHomeserver(bprint.Name, runner, hs, networkID)
@@ -342,7 +347,7 @@ func (d *Builder) construct(bprint b.Blueprint) (errs []error) {
 
 // construct this homeserver and execute its instructions, keeping the container alive.
 func (d *Builder) constructHomeserver(blueprintName string, runner *instruction.Runner, hs b.Homeserver, networkID string) result {
-	contextStr := fmt.Sprintf("%s.%s", blueprintName, hs.Name)
+	contextStr := fmt.Sprintf("%s.%s.%s", d.Config.PackageNamespace, blueprintName, hs.Name)
 	d.log("%s : constructing homeserver...\n", contextStr)
 	dep, err := d.deployBaseImage(blueprintName, hs, contextStr, networkID)
 	if err != nil {
@@ -376,15 +381,17 @@ func (d *Builder) deployBaseImage(blueprintName string, hs b.Homeserver, context
 	asIDToRegistrationMap := asIDToRegistrationFromLabels(labelsForApplicationServices(hs))
 
 	return deployImage(
-		d.Docker, d.BaseImage, d.CSAPIPort, fmt.Sprintf("complement_%s", contextStr), blueprintName, hs.Name, asIDToRegistrationMap, contextStr,
-		networkID, d.config.VersionCheckIterations,
+		d.Docker, d.Config.BaseImageURI, d.CSAPIPort, fmt.Sprintf("complement_%s", contextStr),
+		d.Config.PackageNamespace, blueprintName, hs.Name, asIDToRegistrationMap, contextStr,
+		networkID, d.Config.VersionCheckIterations,
 	)
 }
 
 // getCaVolume returns the correct volume mount for providing a CA to homeserver containers.
 // If running CI, returns an error if it's unable to find a volume that has /ca
 // Otherwise, returns an error if we're unable to find the <cwd>/ca directory on the local host
 func getCaVolume(ctx context.Context, docker *client.Client) (caMount mount.Mount, err error) {
+	// TODO: wrap in a lockfile
 	if os.Getenv("CI") == "true" {
 		// When in CI, Complement itself is a container with the CA volume mounted at /ca.
 		// We need to mount this volume to all homeserver containers to synchronize the CA cert.
@@ -484,7 +491,7 @@ func generateASRegistrationYaml(as b.ApplicationService) string {
 }
 
 func deployImage(
-	docker *client.Client, imageID string, csPort int, containerName, blueprintName, hsName string, asIDToRegistrationMap map[string]string, contextStr, networkID string, versionCheckIterations int,
+	docker *client.Client, imageID string, csPort int, containerName, pkgNamespace, blueprintName, hsName string, asIDToRegistrationMap map[string]string, contextStr, networkID string, versionCheckIterations int,
 ) (*HomeserverDeployment, error) {
 	ctx := context.Background()
 	var extraHosts []string
@@ -526,6 +533,7 @@ func deployImage(
 		Labels: map[string]string{
 			complementLabel:        contextStr,
 			"complement_blueprint": blueprintName,
+			"complement_pkg":       pkgNamespace,
 			"complement_hs_name":   hsName,
 		},
 	}, &container.HostConfig{
@@ -616,12 +624,15 @@ func deployImage(
 	return d, nil
 }
 
-// CreateNetworkIfNotExists creates a docker network and returns its id.
+// createNetworkIfNotExists creates a docker network and returns its id.
 // ID is guaranteed not to be empty when err == nil
-func CreateNetworkIfNotExists(docker *client.Client, blueprintName string) (networkID string, err error) {
+func createNetworkIfNotExists(docker *client.Client, pkgNamespace, blueprintName string) (networkID string, err error) {
 	// check if a network already exists for this blueprint
 	nws, err := docker.NetworkList(context.Background(), types.NetworkListOptions{
-		Filters: label("complement_blueprint=" + blueprintName),
+		Filters: label(
+			"complement_pkg="+pkgNamespace,
+			"complement_blueprint="+blueprintName,
+		),
 	})
 	if err != nil {
 		return "", fmt.Errorf("%s: failed to list networks. %w", blueprintName, err)
@@ -631,10 +642,11 @@ func CreateNetworkIfNotExists(docker *client.Client, blueprintName string) (netw
 		return nws[0].ID, nil
 	}
 	// make a user-defined network so we get DNS based on the container name
-	nw, err := docker.NetworkCreate(context.Background(), "complement_"+blueprintName, types.NetworkCreate{
+	nw, err := docker.NetworkCreate(context.Background(), "complement_"+pkgNamespace+"_"+blueprintName, types.NetworkCreate{
 		Labels: map[string]string{
 			complementLabel:        blueprintName,
 			"complement_blueprint": blueprintName,
+			"complement_pkg":       pkgNamespace,
 		},
 	})
 	if err != nil {
@@ -668,9 +680,14 @@ func printLogs(docker *client.Client, containerID, contextStr string) {
 	log.Printf("============== %s : END LOGS ==============\n\n\n", contextStr)
 }
 
-func label(in string) filters.Args {
+// label returns a filter for the presence of certain labels ("complement_context") or a match of
+// labels ("complement_blueprint=foo").
+func label(labelFilters ...string) filters.Args {
 	f := filters.NewArgs()
-	f.Add("label", in)
+	// label=<key> or label=<key>=<value>
+	for _, in := range labelFilters {
+		f.Add("label", in)
+	}
 	return f
 }
 

internal/docker/builder.go

@@ -29,24 +29,24 @@ import (
 )
 
 type Deployer struct {
-	Namespace    string
-	Docker       *client.Client
-	Counter      int
-	networkID    string
-	debugLogging bool
-	config       *config.Complement
+	DeployNamespace string
+	Docker          *client.Client
+	Counter         int
+	networkID       string
+	debugLogging    bool
+	config          *config.Complement
 }
 
-func NewDeployer(namespace string, cfg *config.Complement) (*Deployer, error) {
+func NewDeployer(deployNamespace string, cfg *config.Complement) (*Deployer, error) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		return nil, err
 	}
 	return &Deployer{
-		Namespace:    namespace,
-		Docker:       cli,
-		debugLogging: cfg.DebugLoggingEnabled,
-		config:       cfg,
+		DeployNamespace: deployNamespace,
+		Docker:          cli,
+		debugLogging:    cfg.DebugLoggingEnabled,
+		config:          cfg,
 	}, nil
 }
 
@@ -64,15 +64,18 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen
 		HS:            make(map[string]HomeserverDeployment),
 	}
 	images, err := d.Docker.ImageList(ctx, types.ImageListOptions{
-		Filters: label("complement_blueprint=" + blueprintName),
+		Filters: label(
+			"complement_pkg="+d.config.PackageNamespace,
+			"complement_blueprint="+blueprintName,
+		),
 	})
 	if err != nil {
 		return nil, fmt.Errorf("Deploy: failed to ImageList: %w", err)
 	}
 	if len(images) == 0 {
 		return nil, fmt.Errorf("Deploy: No images have been built for blueprint %s", blueprintName)
 	}
-	networkID, err := CreateNetworkIfNotExists(d.Docker, blueprintName)
+	networkID, err := createNetworkIfNotExists(d.Docker, d.config.PackageNamespace, blueprintName)
 	if err != nil {
 		return nil, fmt.Errorf("Deploy: %w", err)
 	}
@@ -85,8 +88,8 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen
 
 		// TODO: Make CSAPI port configurable
 		deployment, err := deployImage(
-			d.Docker, img.ID, 8008, fmt.Sprintf("complement_%s_%s_%d", d.Namespace, contextStr, d.Counter),
-			blueprintName, hsName, asIDToRegistrationMap, contextStr, networkID, d.config.VersionCheckIterations)
+			d.Docker, img.ID, 8008, fmt.Sprintf("complement_%s_%s_%s_%d", d.config.PackageNamespace, d.DeployNamespace, contextStr, d.Counter),
+			d.config.PackageNamespace, blueprintName, hsName, asIDToRegistrationMap, contextStr, networkID, d.config.VersionCheckIterations)
 		if err != nil {
 			if deployment != nil && deployment.ContainerID != "" {
 				// print logs to help debug

internal/docker/deployer.go

@@ -20,6 +20,9 @@ import (
 
 var namespaceCounter uint64
 
+// persist the complement builder which is set when the tests start via TestMain
+var complementBuilder *docker.Builder
+
 // TestMain is the main entry point for Complement.
 //
 // It will clean up any old containers/images/networks from the previous run, then run the tests, then clean up
@@ -32,6 +35,7 @@ func TestMain(m *testing.M) {
 		fmt.Printf("Error: %s", err)
 		os.Exit(1)
 	}
+	complementBuilder = builder
 	// remove any old images/containers/networks in case we died horribly before
 	builder.Cleanup()
 
@@ -60,16 +64,14 @@ func TestMain(m *testing.M) {
 func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
 	t.Helper()
 	timeStartBlueprint := time.Now()
-	cfg := config.NewConfigFromEnvVars()
-	builder, err := docker.NewBuilder(cfg)
-	if err != nil {
-		t.Fatalf("Deploy: docker.NewBuilder returned error: %s", err)
+	if complementBuilder == nil {
+		t.Fatalf("complementBuilder not set, did you forget to call TestMain?")
 	}
-	if err = builder.ConstructBlueprintsIfNotExist([]b.Blueprint{blueprint}); err != nil {
+	if err := complementBuilder.ConstructBlueprintsIfNotExist([]b.Blueprint{blueprint}); err != nil {
 		t.Fatalf("Deploy: Failed to construct blueprint: %s", err)
 	}
 	namespace := fmt.Sprintf("%d", atomic.AddUint64(&namespaceCounter, 1))
-	d, err := docker.NewDeployer(namespace, cfg)
+	d, err := docker.NewDeployer(namespace, complementBuilder.Config)
 	if err != nil {
 		t.Fatalf("Deploy: NewDeployer returned error %s", err)
 	}