feat: remove ABFS data disk from the server module

With the introduction of the new GCS/Spanner backend, the 10TB data disk on the ABFS server is now obsolete. This change will delete and recreate the ABFS server. Please make sure to update to v0.5.0 first.

PiperOrigin-RevId: 778541579

Author: kunzese

files/cloud-init/scripts/abfs_container.env.tftpl

@@ -16,20 +16,17 @@
 set -e
 
 source "$(dirname "$0")/abfs_base.sh"
-# TODO: If running as a non-root user add to docker group
-#usermod -a -G docker abfs
-#chown -R -c abfs:abfs /home/abfs-server
+
 # Defend against docker issues potentially causing the boot disk to fill up
 if [[ "$(df /mnt/stateful_partition --output=pcent | egrep -o '[0-9]+')" -ge 90 ]]; then
-  docker system prune -a -f || echo "docker system prune failed, continuing with GCR warmup"
+  docker system prune -a -f \
+    || echo "docker system prune failed, continuing with GCR warmup"
 fi
 AR_HOSTS=("us-docker.pkg.dev","europe-docker.pkg.dev","europe-north1-docker.pkg.dev","europe-west3-docker.pkg.dev")
 docker-credential-gcr configure-docker --registries "${AR_HOSTS[*]}"
-# TODO: Support pulling multiple images
-# TODO: Check VM region in metadata and pull from nearest repo
 docker pull "${ABFS_DOCKER_IMAGE_URI}"
 
-if [[ "$(docker image ls -q "${ABFS_DOCKER_IMAGE_URI}")" == "" ]]; then
+if [[ -z "$(docker image ls -q "${ABFS_DOCKER_IMAGE_URI}")" ]]; then
   while ! 'docker image inspect -f true "${ABFS_DOCKER_IMAGE_URI}" &> /dev/null'; do
     echo "waiting for image to be fully pulled"
     sleep 1

files/cloud-init/scripts/abfs_docker_warmup.sh renamed to files/scripts/abfs_docker_warmup.sh

@@ -15,16 +15,17 @@
 
 source "$(dirname "$0")/abfs_base.sh"
 
-# TODO: think about which additional Docker run args might be needed...
-# --memory, --memory-swap, --ulimit, --oom-score-adj
 DOCKER_RUN_ARGS=(--privileged \
     --cap-add=all \
     --pid=host \
     --env-file /var/run/abfs/abfs_container.env \
-    --publish 50051:50051 \
-    --volume ${DATADISK_MOUNTPOINT}:/abfs-storage
+    --publish 50051:50051
 )
-# TODO: pass abfs args to the VM dynmaically
+
+if [[ -n "${DATADISK_MOUNTPOINT}" ]]; then
+  DOCKER_RUN_ARGS+=(--volume ${DATADISK_MOUNTPOINT}:/abfs-storage)
+fi
+
 docker run --name=abfs-server --log-driver=journald \
   "${DOCKER_RUN_ARGS[@]}" "${ABFS_DOCKER_IMAGE_URI}" \
   ${ABFS_CMD} 

files/cloud-init/scripts/abfs_server.sh renamed to files/scripts/abfs_server.sh

@@ -14,8 +14,10 @@
 
 [Unit]
 Description=Start an ABFS server
+%{ if type == "uploader" ~}
 Wants=abfs-datadisk-ready.target
 After=abfs-datadisk-ready.target
+%{ endif ~}
 Wants=abfs-docker-warmup.service
 After=abfs-docker-warmup.service
 

files/cloud-init/abfs-docker-warmup.service renamed to files/systemd/abfs-docker-warmup.service

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-declare -r DATADISK_MOUNTPOINT="${abfs_datadisk_mountpoint}"
-declare -r ABFS_DOCKER_IMAGE_URI="${abfs_docker_image_uri}"
-declare -r ABFS_CMD="${abfs_command}"
+%{ for env_key, env_value in envs ~}
+declare -r ${env_key}="${env_value}"
+%{ endfor ~}

files/cloud-init/abfs-server.service renamed to files/templates/abfs-server.service.tftpl

@@ -0,0 +1,18 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Environment variables passed to the running ABFS server container
+%{ for env_key, env_value in envs ~}
+${env_key}=${env_value}
+%{ endfor ~}

files/cloud-init/scripts/abfs_base.sh.tftpl renamed to files/templates/abfs_base.sh.tftpl

@@ -11,12 +11,10 @@ For example usage, please check the following [example](../../examples/simple/ma
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| abfs\_bootdisk\_size\_gb | Size in GB for the ABFS bootdisk that will be attached to the VM | `number` | `100` | no |
+| abfs\_bootdisk\_type | The PD regional disk type to use for the ABFS bootdisk | `string` | `"pd-ssd"` | no |
 | abfs\_bucket\_location | The location of the ABFS bucket (https://cloud.google.com/storage/docs/locations). | `string` | n/a | yes |
 | abfs\_bucket\_name | The name of the ABFS bucket. | `string` | `"abfs"` | no |
-| abfs\_datadisk\_mountpoint | Location for mounting the ABFS datadisk on the host VM | `string` | `"/mnt/disks/abfs-data"` | no |
-| abfs\_datadisk\_name | A name for the ABFS datadisk that will be attached to the VM. Note, this does not affect the mounting of the disk - the device name is always set to "abfs-server-storage" | `string` | `"abfs-datadisk"` | no |
-| abfs\_datadisk\_size\_gb | Size in GB for the ABFS datadisk that will be attached to the VM | `number` | `10000` | no |
-| abfs\_datadisk\_type | The PD regional disk type to use for the ABFS datadisk | `string` | `"pd-ssd"` | no |
 | abfs\_docker\_image\_uri | Docker image URI for the ABFS server | `string` | n/a | yes |
 | abfs\_license | ABFS license (JSON) | `string` | n/a | yes |
 | abfs\_server\_cos\_image\_ref | Reference to the COS boot image to use for the ABFS server | `string` | `"projects/cos-cloud/global/images/family/cos-109-lts"` | no |

files/templates/abfs_container.env.tftpl

@@ -13,41 +13,60 @@
 # limitations under the License.
 
 locals {
-  cloud_init_path           = "${path.module}/../../files/cloud-init"
-  goog_cm_deployment_name   = var.goog_cm_deployment_name == "" ? "" : "${var.goog_cm_deployment_name}-"
-  abfs_datadisk_device_name = "abfs-server-storage"
-  static_script_files = [for filename in fileset("${local.cloud_init_path}/scripts/", "*.sh") :
-    {
-      path        = "/var/lib/abfs/bin/${filename}"
-      permissions = "0755"
-      owner       = "root"
-      encoding    = "gzip+base64"
-      content     = base64gzip(file("${local.cloud_init_path}/scripts/${filename}"))
-  }]
-  systemd_files = [for filename in fileset(local.cloud_init_path, "abfs*") :
-    {
-      path        = "/etc/systemd/system/${filename}"
-      permissions = "0644"
-      owner       = "root"
-      encoding    = "gzip+base64"
-      content     = base64gzip(file("${local.cloud_init_path}/${filename}"))
-  }]
+  common_files_root       = "${path.module}/../../files"
+  goog_cm_deployment_name = var.goog_cm_deployment_name == "" ? "" : "${var.goog_cm_deployment_name}-"
+  abfs_server_name        = "${local.goog_cm_deployment_name}${var.abfs_server_name}"
+
+  static_script_files = flatten([
+    for folder in ["${local.common_files_root}/scripts", "${path.module}/files/scripts"] : [
+      for filename in fileset(folder, "*.sh") :
+      {
+        path        = "/var/lib/abfs/bin/${filename}"
+        permissions = "0755"
+        owner       = "root"
+        encoding    = "gzip+base64"
+        content     = base64gzip(file("${folder}/${filename}"))
+      }
+    ]
+  ])
+
+  systemd_files = flatten([
+    for folder in ["${local.common_files_root}/systemd", "${path.module}/files/systemd"] : [
+      for filename in fileset(folder, "*") :
+      {
+        path        = "/etc/systemd/system/${filename}"
+        permissions = "0644"
+        owner       = "root"
+        encoding    = "gzip+base64"
+        content     = base64gzip(file("${folder}/${filename}"))
+      }
+    ]
+  ])
+
   runcmd = [
     "systemctl daemon-reload",
-    "systemctl enable abfs-datadisk.path",
     "systemctl restart abfs-docker-warmup.service",
     "systemctl is-active -q abfs-server.service || systemctl start --no-block abfs-server.service",
     "systemctl is-active -q fluent-bit.service || systemctl start --no-block fluent-bit.service",
     "systemctl is-active -q node-problem-detector.service || systemctl start --no-block node-problem-detector.service"
   ]
+
   bootcmd = [
     "echo 'cloud-init bootcmd'"
   ]
 }
 
+resource "google_compute_disk" "abfs_server_bootdisk" {
+  name  = "${local.abfs_server_name}-bootdisk"
+  zone  = var.zone
+  image = var.abfs_server_cos_image_ref
+  size  = var.abfs_bootdisk_size_gb
+  type  = var.abfs_bootdisk_type
+}
+
 resource "google_compute_instance" "abfs_server" {
   project      = var.project_id
-  name         = "${local.goog_cm_deployment_name}${var.abfs_server_name}"
+  name         = local.abfs_server_name
   machine_type = var.abfs_server_machine_type
   zone         = var.zone
 
@@ -58,9 +77,7 @@ resource "google_compute_instance" "abfs_server" {
   }
 
   boot_disk {
-    initialize_params {
-      image = var.abfs_server_cos_image_ref
-    }
+    source = google_compute_disk.abfs_server_bootdisk.name
   }
 
   network_interface {
@@ -76,29 +93,6 @@ resource "google_compute_instance" "abfs_server" {
     abfs-license = base64encode(var.abfs_license)
     user-data    = data.cloudinit_config.abfs_server.rendered
   }
-
-  lifecycle {
-    ignore_changes = [attached_disk]
-  }
-}
-
-resource "google_compute_disk" "abfs_datadisk" {
-  project = var.project_id
-  name    = "${local.goog_cm_deployment_name}${var.abfs_datadisk_name}"
-  size    = var.abfs_datadisk_size_gb
-  zone    = var.zone
-  type    = var.abfs_datadisk_type
-
-  lifecycle {
-    # prevent_destroy = true
-  }
-}
-
-resource "google_compute_attached_disk" "abfs_datadisk_attachment" {
-  project     = var.project_id
-  disk        = google_compute_disk.abfs_datadisk.id
-  instance    = google_compute_instance.abfs_server.id
-  device_name = local.abfs_datadisk_device_name
 }
 
 data "cloudinit_config" "abfs_server" {
@@ -118,9 +112,11 @@ data "cloudinit_config" "abfs_server" {
             permissions = "0644"
             owner       = "root"
             encoding    = "gzip+base64"
-            content = base64gzip(templatefile("${local.cloud_init_path}/scripts/abfs_container.env.tftpl",
+            content = base64gzip(templatefile("${local.common_files_root}/templates/abfs_container.env.tftpl",
               {
-                needs_git = "false"
+                envs = {
+                  "NEEDS_GIT" = false
+                }
             }))
           }
         ],
@@ -133,16 +129,29 @@ data "cloudinit_config" "abfs_server" {
             permissions = "0755"
             owner       = "root"
             encoding    = "gzip+base64"
-            content = base64gzip(templatefile("${local.cloud_init_path}/scripts/abfs_base.sh.tftpl",
+            content = base64gzip(templatefile("${local.common_files_root}/templates/abfs_base.sh.tftpl",
+              {
+                envs = {
+                  "ABFS_CMD"              = <<-EOT
+                    --project ${google_spanner_instance.abfs.project} \
+                    --bucket ${google_storage_bucket.abfs.name} \
+                    --instance ${google_spanner_instance.abfs.name} \
+                    --db ${google_spanner_database.abfs.name}
+                  EOT
+                  "ABFS_DOCKER_IMAGE_URI" = var.abfs_docker_image_uri,
+                }
+            }))
+          }
+        ],
+        [
+          {
+            path        = "/etc/systemd/system/abfs-server.service"
+            permissions = "0644"
+            owner       = "root"
+            encoding    = "gzip+base64"
+            content = base64gzip(templatefile("${local.common_files_root}/templates/abfs-server.service.tftpl",
               {
-                abfs_docker_image_uri    = var.abfs_docker_image_uri
-                abfs_datadisk_mountpoint = var.abfs_datadisk_mountpoint
-                abfs_command             = <<-EOT
-                  --project ${google_spanner_instance.abfs.project} \
-                  --bucket ${google_storage_bucket.abfs.name} \
-                  --instance ${google_spanner_instance.abfs.name} \
-                  --db ${google_spanner_database.abfs.name}
-                EOT
+                type = "server"
             }))
           }
         ],

modules/server/README.md

@@ -68,20 +68,12 @@ spec:
         description: Docker image URI for the ABFS server
         varType: string
         required: true
-      - name: abfs_datadisk_mountpoint
-        description: Location for mounting the ABFS datadisk on the host VM
-        varType: string
-        defaultValue: /mnt/disks/abfs-data
-      - name: abfs_datadisk_name
-        description: A name for the ABFS datadisk that will be attached to the VM. Note, this does not affect the mounting of the disk - the device name is always set to "abfs-server-storage"
-        varType: string
-        defaultValue: abfs-datadisk
-      - name: abfs_datadisk_size_gb
-        description: Size in GB for the ABFS datadisk that will be attached to the VM
+      - name: abfs_bootdisk_size_gb
+        description: Size in GB for the ABFS bootdisk that will be attached to the VM
         varType: number
-        defaultValue: 10000
-      - name: abfs_datadisk_type
-        description: The PD regional disk type to use for the ABFS datadisk
+        defaultValue: 100
+      - name: abfs_bootdisk_type
+        description: The PD regional disk type to use for the ABFS bootdisk
         varType: string
         defaultValue: pd-ssd
       - name: abfs_bucket_name