Convert Couchbase server deployment to use Docker images (#316)

This avoids the overhead of RPM install which has to validate every single file.  That validation basically is done ahead of time in the process of creating the image that will be used.  This saves about 25 seconds of runtime in the case of stopping and restarting.

* Add CreatedBy tag to see who created instances
* Switch yum to dnf

Author: borrrden

environment/aws/common/docker.py

@@ -61,6 +61,7 @@ def start_container(
     host: str,
     docker_args: list[str] | None = None,
     container_args: list[str] | None = None,
+    replace_existing: bool = False,
 ) -> None:
     context_result = subprocess.run(
         ["docker", "context", "ls", "--format", "{{.Name}}"],
@@ -108,17 +109,23 @@ def start_container(
 
     if container_check.stdout.strip() != "":
         if container_check.stdout.startswith("Up"):
-            click.echo(f"{name} already running, returning...")
-            return
+            if not replace_existing:
+                click.echo(f"{name} already running, returning...")
+                return
 
-        click.echo(f"Restarting existing {name} container...")
-        subprocess.run(["docker", "start", name], check=False, env=env)
-        return
+            click.echo(f"Stopping existing {name} container...")
+            subprocess.run(["docker", "stop", name], check=True, env=env)
+
+        if not replace_existing:
+            click.echo(f"Restarting existing {name} container...")
+            subprocess.run(["docker", "start", name], check=False, env=env)
+            return
 
     click.echo(f"Starting new {name} container...")
     args = [
         "docker",
         "run",
+        "--rm",
         "-d",
         "--name",
         name,

environment/aws/lb_setup/configure-system.sh

@@ -3,7 +3,7 @@
 set -x
 
 if ! command -v docker &> /dev/null; then
-  sudo yum install -y docker
+  sudo dnf install -y docker
   sudo systemctl start docker
 fi
 

environment/aws/logslurp_setup/configure-system.sh

@@ -3,7 +3,7 @@
 set -x
 
 if ! command -v docker &> /dev/null; then
-  sudo yum install -y docker
+  sudo dnf install -y docker
   sudo systemctl start docker
 fi
 

environment/aws/main.tf

@@ -13,6 +13,9 @@ provider "aws" {
     region = "us-east-1"
 }
 
+# Who is running Terraform
+data "aws_caller_identity" "current" {}
+
 # Read the subnet in which to insert our EC2 instances.  
 # The mobile-e2e subnet exists in the mobile-e2e VPC and
 # has domain over the IP addresses 10.0.1.1 - 10.0.1.255
@@ -56,6 +59,7 @@ resource "aws_instance" "couchbaseserver" {
         Name = "cbs"
         Type = "couchbaseserver"
         ExpireAt = local.expire_at
+        CreatedBy = local.created_by
     }
 
     lifecycle {
@@ -83,6 +87,7 @@ resource "aws_instance" "sync_gateway" {
         Name = "sg"
         Type = "syncgateway"
         ExpireAt = local.expire_at
+        CreatedBy = local.created_by
     }
 
     lifecycle {
@@ -105,6 +110,7 @@ resource "aws_instance" "edge_server" {
         Name = "es"
         Type = "edgeserver"
         ExpireAt = local.expire_at
+        CreatedBy = local.created_by
     }
 
     lifecycle {
@@ -127,6 +133,7 @@ resource "aws_instance" "load_balancer" {
         Name = "lb"
         Type = "loadbalancer"
         ExpireAt = local.expire_at
+        CreatedBy = local.created_by
     }
 
     lifecycle {
@@ -157,6 +164,8 @@ resource "aws_instance" "log_slurp" {
 }
 
 locals {
+  arn_path_parts = split("/", data.aws_caller_identity.current.arn)
+  created_by     = local.arn_path_parts[length(local.arn_path_parts)-1]
   expire_at = formatdate(
     "YYYY-MM-DD'T'hh:mm:ss'Z'",
     timeadd(timestamp(), format("%dh", 3 * 24))

environment/aws/server_setup/configure-node.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 mkdir -p /home/ec2-user/logs
 
@@ -7,6 +7,7 @@ config_done() {
   touch ${CONFIG_DONE_FILE}
   echo "Couchbase Admin UI: http://localhost:8091" \
      "\nLogin credentials: Administrator / password"
+     sleep infinity
 }
 
 if [ -e ${CONFIG_DONE_FILE} ]; then
@@ -65,14 +66,14 @@ curl_check() {
 wait_for_uri 200 http://localhost:8091/ui/index.html
 echo "Couchbase Server up!"
 
-if [[ ! -z $1 ]]; then
+if [[ ! -z $E2E_PARENT_CLUSTER ]]; then
   my_ip=$(ifconfig | grep "inet 10" | awk '{print $2}')
-  echo "Adding node to cluster"
-  couchbase_cli_check server-add -c $1 -u Administrator -p password --server-add  $my_ip \
+  echo "Adding node to cluster $E2E_PARENT_CLUSTER"
+  couchbase_cli_check server-add -c $E2E_PARENT_CLUSTER -u Administrator -p password --server-add  $my_ip \
     --server-add-username Administrator --server-add-password password --services data,index,query
   echo
   echo "Rebalancing cluster"
-  couchbase_cli_check rebalance -c $1 -u Administrator -p password 
+  couchbase_cli_check rebalance -c $E2E_PARENT_CLUSTER -u Administrator -p password 
   echo
 else 
   echo "Set up the cluster"

environment/aws/server_setup/configure-system.sh

@@ -1,37 +1,14 @@
 #!/bin/bash
 
-# Check if the script is run as root
-if [ "$EUID" -ne 0 ]; then
-  echo "Please run as root"
-  exit 1
-fi
-
-echo "Disabling Transparent Huge Pages (THP)..."
-
-cp /tmp/disable-thp.service /etc/systemd/system/disable-thp.service
-systemctl daemon-reload
-systemctl start disable-thp
-systemctl enable disable-thp
-
-tmp=$(cat /sys/kernel/mm/transparent_hugepage/enabled)
-if [[ $tmp != *"[never]"* ]]; then
-  echo "Failed to disable Transparent Huge Pages!"
-  exit 1
-fi
-
-tmp=$(cat /sys/kernel/mm/transparent_hugepage/defrag)
-if [[ $tmp != *"[never]"* ]]; then
-  echo "Failed to disable THP defrag"
-  exit 1
-fi
-
-echo "Setting swappiness to 1..."
+set -x
 
-echo 1 > /proc/sys/vm/swappiness
-tmp=$(cat /proc/sys/vm/swappiness)
-if [[ $tmp != "1" ]]; then
-  echo "Failed to set swappiness to 1"
-  exit 1
+if ! command -v docker &> /dev/null; then
+  echo "Docker not found, installing and starting..."
+  sudo dnf install -y docker
+  sudo systemctl start docker
 fi
 
-rm -f /home/ec2-user/container-configured
+if ! groups $USER | grep -q "\bdocker\b"; then
+  echo "Adding $USER to docker group"
+  sudo usermod -aG docker $USER
+fi

environment/aws/server_setup/setup_server.py

@@ -21,7 +21,8 @@
 import click
 import paramiko
 
-from environment.aws.common.io import LIGHT_GRAY, sftp_progress_bar
+from environment.aws.common.docker import start_container
+from environment.aws.common.io import LIGHT_GRAY, get_ec2_hostname, sftp_progress_bar
 from environment.aws.common.output import header
 from environment.aws.topology_setup.setup_topology import TopologyConfig
 
@@ -63,6 +64,7 @@ def setup_node(
     hostname: str,
     pkey: paramiko.Ed25519Key | None,
     version: str,
+    tag: str,
     cluster: str | None = None,
 ) -> None:
     """
@@ -74,11 +76,6 @@ def setup_node(
         version (str): The version of Couchbase Server to install.
         cluster (Optional[str]): The cluster to join, if any.
     """
-    couchbase_filename = f"couchbase-server-enterprise-{version}-linux.x86_64.rpm"
-    couchbase_url = (
-        f"http://packages.couchbase.com/releases/{version}/{couchbase_filename}"
-    )
-
     header(f"Setting up server {hostname} with version {version}")
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
@@ -90,45 +87,42 @@ def setup_node(
     sftp_progress_bar(
         sftp, SCRIPT_DIR / "configure-system.sh", "/tmp/configure-system.sh"
     )
-    sftp_progress_bar(
-        sftp, SCRIPT_DIR / "disable-thp.service", "/tmp/disable-thp.service"
-    )
     sftp.close()
 
     global current_ssh
     current_ssh = hostname
 
-    remote_exec(ssh, "sudo bash /tmp/configure-system.sh", "Setting up machine")
     remote_exec(
         ssh,
-        f"wget -nc -O /tmp/{couchbase_filename} {couchbase_url} 2>&1",
-        f"Downloading Couchbase Server {version}",
-        fail_on_error=False,
-    )
-    remote_exec(
-        ssh,
-        "sudo rpm -e couchbase-server",
-        "Uninstalling Couchbase Server",
-        fail_on_error=False,
-    )
-    remote_exec(
-        ssh,
-        f"sudo rpm -i /tmp/{couchbase_filename}",
-        f"Installing Couchbase Server {version}",
-    )
-    remote_exec(
-        ssh, "sudo systemctl start couchbase-server", "Starting Couchbase Server"
+        "chmod +x /tmp/configure-node.sh && bash /tmp/configure-system.sh",
+        "Setting up machine",
     )
+    ssh.close()
 
-    # Some magic here that might be overlooked.  If we pass in a cluster
-    # address to the configure node script, it will join an existing cluster
-    # rather than using itself to create a new one
-    config_command = "bash /tmp/configure-node.sh"
-    if cluster is not None:
-        config_command += f" {cluster}"
-    remote_exec(ssh, config_command, "Setting up node")
+    ec2_hostname = get_ec2_hostname(hostname)
+    docker_args = [
+        "--network",
+        "host",
+        "-v",
+        "/tmp/configure-node.sh:/etc/service/couchbase-config/run",
+    ]
 
-    ssh.close()
+    if cluster is not None:
+        docker_args.extend(
+            [
+                "-e",
+                f"E2E_PARENT_CLUSTER={cluster}",
+            ]
+        )
+    context_name = "cbs" if tag == "" else f"cbs-{tag}"
+    start_container(
+        "cbs-e2e",
+        context_name,
+        f"couchbase/server:enterprise-{version}",
+        ec2_hostname,
+        docker_args,
+        replace_existing=True,
+    )
 
 
 def setup_topology(pkey: paramiko.Ed25519Key | None, topology: TopologyConfig) -> None:
@@ -144,10 +138,19 @@ def setup_topology(pkey: paramiko.Ed25519Key | None, topology: TopologyConfig) -
         return
 
     for cluster_config in topology.clusters:
-        setup_node(cluster_config.public_hostnames[0], pkey, cluster_config.version)
+        setup_node(
+            cluster_config.public_hostnames[0],
+            pkey,
+            cluster_config.version,
+            topology.tag,
+        )
         for server in cluster_config.public_hostnames[1:]:
             setup_node(
-                server, pkey, cluster_config.version, cluster_config.public_hostnames[0]
+                server,
+                pkey,
+                cluster_config.version,
+                topology.tag,
+                cluster_config.internal_hostnames[0],
             )