Merge pull request kubernetes#92286 from wojtek-t/migrate_if_needed_golang

Merge migrate-if-needed etcd bash script with golang binary

Author: k8s-ci-robot

build/dependencies.yaml

@@ -65,8 +65,7 @@ dependencies:
     refPaths:
     - path: cluster/images/etcd/Makefile
       match: BUNDLED_ETCD_VERSIONS\?|LATEST_ETCD_VERSION\?
-    - path: cluster/images/etcd/migrate-if-needed.sh
-      match: BUNDLED_VERSIONS=
+    - path: cluster/images/etcd/migrate/options.go
 
   - name: "golang"
     version: 1.14.4

cluster/images/etcd/migrate-if-needed.sh

@@ -14,95 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# NOTES
-# This script performs etcd upgrade based on the following environmental
-# variables:
-# TARGET_STORAGE - API of etcd to be used (supported: 'etcd3')
-# TARGET_VERSION - etcd release to be used (supported: '3.0.17', '3.1.12',
-# '3.2.24', "3.3.17", "3.4.9")
-# DATA_DIRECTORY - directory with etcd data
-#
-# The current etcd version and storage format is detected based on the
-# contents of "${DATA_DIRECTORY}/version.txt" file (if the file doesn't
-# exist, we default it to "3.0.17/etcd2".
-#
-# The update workflow support the following upgrade steps:
-# - 3.0.17/etcd3 -> 3.1.12/etcd3
-# - 3.1.12/etcd3 -> 3.2.24/etcd3
-# - 3.2.24/etcd3 -> 3.3.17/etcd3
-# - 3.3.17/etcd3 -> 3.4.9/etcd3
-#
-# NOTE: The releases supported in this script has to match release binaries
-# present in the etcd image (to make this script work correctly).
-#
-# Based on the current etcd version and storage format we detect what
-# upgrade step from this should be done to get reach target configuration
+
+# DEPRECATED:
+# The functionality has been moved to migrate binary and this script
+# if left for backward compatibility with previous manifests. It will be
+# removed in the future.
 
 set -o errexit
 set -o nounset
 
-# NOTE: BUNDLED_VERSION has to match release binaries present in the
-# etcd image (to make this script work correctly).
-BUNDLED_VERSIONS="3.0.17, 3.1.12, 3.2.24, 3.3.17, 3.4.9"
-
-# shellcheck disable=SC2039
-# NOTE: Make sure the resulted ETCD_NAME agrees with --name in etcd.manifest: https://github.com/kubernetes/kubernetes/blob/e7ca64fbe16d0c4b6c7b36aecde9cd75042b2828/cluster/gce/manifests/etcd.manifest#L27
-ETCD_NAME="${ETCD_NAME:-etcd-${ETCD_HOSTNAME:-$HOSTNAME}}"
-if [ -z "${DATA_DIRECTORY:-}" ]; then
-  echo "DATA_DIRECTORY variable unset - unexpected failure"
-  exit 1
-fi
-
-case "${DATA_DIRECTORY}" in
-  *event*)
-    ETCD_PEER_PORT=2381
-    ETCD_CLIENT_PORT=18631
-    ;;
-  *)
-    ETCD_PEER_PORT=2380
-    ETCD_CLIENT_PORT=18629
-    ;;
-esac
-
-if [ -z "${INITIAL_CLUSTER:-}" ]; then
-  echo "Warn: INITIAL_CLUSTER variable unset - defaulting to ${ETCD_NAME}=http://localhost:${ETCD_PEER_PORT}"
-  INITIAL_CLUSTER="${ETCD_NAME}=http://localhost:${ETCD_PEER_PORT}"
-fi
-if [ -z "${LISTEN_PEER_URLS:-}" ]; then
-  echo "Warn: LISTEN_PEER_URLS variable unset - defaulting to http://localhost:${ETCD_PEER_PORT}"
-  LISTEN_PEER_URLS="http://localhost:${ETCD_PEER_PORT}"
-fi
-if [ -z "${INITIAL_ADVERTISE_PEER_URLS:-}" ]; then
-  echo "Warn: INITIAL_ADVERTISE_PEER_URLS variable unset - defaulting to http://localhost:${ETCD_PEER_PORT}"
-  INITIAL_ADVERTISE_PEER_URLS="http://localhost:${ETCD_PEER_PORT}"
-fi
-if [ -z "${TARGET_VERSION:-}" ]; then
-  echo "TARGET_VERSION variable unset - unexpected failure"
-  exit 1
-fi
-if [ -z "${TARGET_STORAGE:-}" ]; then
-  echo "TARGET_STORAGE variable unset - unexpected failure"
-  exit 1
-fi
-ETCD_DATA_PREFIX="${ETCD_DATA_PREFIX:-/registry}"
-ETCD_CREDS="${ETCD_CREDS:-}"
-
-# Correctly support upgrade and rollback to non-default version.
-if [ "${DO_NOT_MOVE_BINARIES:-}" != "true" ]; then
-  /bin/cp "/usr/local/bin/etcd-${TARGET_VERSION}" "/usr/local/bin/etcd"
-  /bin/cp "/usr/local/bin/etcdctl-${TARGET_VERSION}" "/usr/local/bin/etcdctl"
-fi
-
-/usr/local/bin/migrate \
-  --name "${ETCD_NAME}" \
-  --port "${ETCD_CLIENT_PORT}" \
-  --listen-peer-urls "${LISTEN_PEER_URLS}" \
-  --initial-advertise-peer-urls "${INITIAL_ADVERTISE_PEER_URLS}" \
-  --data-dir "${DATA_DIRECTORY}" \
-  --bundled-versions "${BUNDLED_VERSIONS}" \
-  --initial-cluster "${INITIAL_CLUSTER}" \
-  --target-version "${TARGET_VERSION}" \
-  --target-storage "${TARGET_STORAGE}" \
-  --etcd-data-prefix "${ETCD_DATA_PREFIX}" \
-  --ttl-keys-directory "${TTL_KEYS_DIRECTORY:-${ETCD_DATA_PREFIX}/events}" \
-  --etcd-server-extra-args "${ETCD_CREDS}"
+/usr/local/bin/migrate

cluster/images/etcd/migrate/BUILD

@@ -3,11 +3,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "copy_file.go",
         "data_dir.go",
         "migrate.go",
         "migrate_client.go",
         "migrate_server.go",
         "migrator.go",
+        "options.go",
         "util_others.go",
         "utils_windows.go",
         "versions.go",
@@ -17,6 +19,7 @@ go_library(
     deps = [
         "//vendor/github.com/blang/semver:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/go.etcd.io/etcd/client:go_default_library",
         "//vendor/go.etcd.io/etcd/clientv3:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
@@ -69,6 +72,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "data_dir_test.go",
+        "options_test.go",
         "versions_test.go",
     ],
     data = glob(["testdata/**"]),

cluster/images/etcd/migrate/copy_file.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func copyFile(source, dest string) error {
+	sf, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("unable to open source file [%s]: %q", source, err)
+	}
+	defer sf.Close()
+	fi, err := sf.Stat()
+	if err != nil {
+		return fmt.Errorf("unable to stat source file [%s]: %q", source, err)
+	}
+
+	dir := filepath.Dir(dest)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("unable to create directory [%s]: %q", dir, err)
+	}
+	df, err := os.Create(dest)
+	if err != nil {
+		return fmt.Errorf("unable to create destination file [%s]: %q", dest, err)
+	}
+	defer df.Close()
+
+	_, err = io.Copy(df, sf)
+	if err != nil {
+		return fmt.Errorf("unable to copy [%s] to [%s]: %q", source, dest, err)
+	}
+
+	if err := os.Chmod(dest, fi.Mode()); err != nil {
+		return fmt.Errorf("unable to close destination file: %q", err)
+	}
+	return nil
+}

cluster/images/etcd/migrate/integration_test.go

@@ -42,7 +42,7 @@ import (
 )
 
 var (
-	testSupportedVersions = MustParseSupportedVersions("3.0.17, 3.1.12")
+	testSupportedVersions = MustParseSupportedVersions([]string{"3.0.17", "3.1.12"})
 	testVersionPrevious   = &EtcdVersion{semver.MustParse("3.0.17")}
 	testVersionLatest     = &EtcdVersion{semver.MustParse("3.1.12")}
 )

cluster/images/etcd/migrate/migrate.go

@@ -18,16 +18,14 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 )
 
 const (
-	versionFilename        = "version.txt"
-	defaultPort     uint64 = 18629
+	versionFilename = "version.txt"
 )
 
 var (
@@ -46,94 +44,43 @@ a target etcd version, this tool will upgrade or downgrade the etcd data from th
 	opts = migrateOpts{}
 )
 
-type migrateOpts struct {
-	name                 string
-	port                 uint64
-	peerListenUrls       string
-	peerAdvertiseUrls    string
-	binDir               string
-	dataDir              string
-	bundledVersionString string
-	etcdDataPrefix       string
-	ttlKeysDirectory     string
-	initialCluster       string
-	targetVersion        string
-	targetStorage        string
-	etcdServerArgs       string
-}
-
 func main() {
-	flags := migrateCmd.Flags()
-	flags.StringVar(&opts.name, "name", "", "etcd cluster member name. Defaults to etcd-{hostname}")
-	flags.Uint64Var(&opts.port, "port", defaultPort, "etcd client port to use during migration operations. This should be a different port than typically used by etcd to avoid clients accidentally connecting during upgrade/downgrade operations.")
-	flags.StringVar(&opts.peerListenUrls, "listen-peer-urls", "", "etcd --listen-peer-urls flag, required for HA clusters")
-	flags.StringVar(&opts.peerAdvertiseUrls, "initial-advertise-peer-urls", "", "etcd --initial-advertise-peer-urls flag, required for HA clusters")
-	flags.StringVar(&opts.binDir, "bin-dir", "/usr/local/bin", "directory of etcd and etcdctl binaries, must contain etcd-<version> and etcdctl-<version> for each version listed in bindled-versions")
-	flags.StringVar(&opts.dataDir, "data-dir", "", "etcd data directory of etcd server to migrate")
-	flags.StringVar(&opts.bundledVersionString, "bundled-versions", "", "comma separated list of etcd binary versions present under the bin-dir")
-	flags.StringVar(&opts.etcdDataPrefix, "etcd-data-prefix", "/registry", "etcd key prefix under which all objects are kept")
-	flags.StringVar(&opts.ttlKeysDirectory, "ttl-keys-directory", "", "etcd key prefix under which all keys with TTLs are kept. Defaults to {etcd-data-prefix}/events")
-	flags.StringVar(&opts.initialCluster, "initial-cluster", "", "comma separated list of name=endpoint pairs. Defaults to etcd-{hostname}=http://localhost:2380")
-	flags.StringVar(&opts.targetVersion, "target-version", "", "version of etcd to migrate to. Format must be '<major>.<minor>.<patch>'")
-	flags.StringVar(&opts.targetStorage, "target-storage", "", "storage version of etcd to migrate to, one of: etcd2, etcd3")
-	flags.StringVar(&opts.etcdServerArgs, "etcd-server-extra-args", "", "additional etcd server args for starting etcd servers during migration steps, --peer-* TLS cert flags should be added for etcd clusters with more than 1 member that use mutual TLS for peer communication.")
+	registerFlags(migrateCmd.Flags(), &opts)
 	err := migrateCmd.Execute()
 	if err != nil {
-		fmt.Printf("Failed to execute migratecmd: %s", err)
+		klog.Errorf("Failed to execute migratecmd: %s", err)
 	}
 }
 
-// runMigrate validates the command line flags and starts the migration.
+// runMigrate starts the migration.
 func runMigrate() {
-	if opts.name == "" {
-		hostname, err := os.Hostname()
-		if err != nil {
-			klog.Errorf("Error while getting hostname to supply default --name: %v", err)
-			os.Exit(1)
-		}
-		opts.name = fmt.Sprintf("etcd-%s", hostname)
-	}
-
-	if opts.ttlKeysDirectory == "" {
-		opts.ttlKeysDirectory = fmt.Sprintf("%s/events", opts.etcdDataPrefix)
-	}
-	if opts.initialCluster == "" {
-		opts.initialCluster = fmt.Sprintf("%s=http://localhost:2380", opts.name)
-	}
-	if opts.targetStorage == "" {
-		klog.Errorf("--target-storage is required")
-		os.Exit(1)
-	}
-	if opts.targetVersion == "" {
-		klog.Errorf("--target-version is required")
-		os.Exit(1)
-	}
-	if opts.dataDir == "" {
-		klog.Errorf("--data-dir is required")
-		os.Exit(1)
-	}
-	if opts.bundledVersionString == "" {
-		klog.Errorf("--bundled-versions is required")
-		os.Exit(1)
-	}
-
-	bundledVersions, err := ParseSupportedVersions(opts.bundledVersionString)
-	if err != nil {
-		klog.Errorf("Failed to parse --supported-versions: %v", err)
-	}
-	err = validateBundledVersions(bundledVersions, opts.binDir)
-	if err != nil {
-		klog.Errorf("Failed to validate that 'etcd-<version>' and 'etcdctl-<version>' binaries exist in --bin-dir '%s' for all --bundled-versions '%s': %v",
-			opts.binDir, opts.bundledVersionString, err)
-		os.Exit(1)
+	if err := opts.validateAndDefault(); err != nil {
+		klog.Fatalf("%v", err)
 	}
+	copyBinaries()
 
 	target := &EtcdVersionPair{
 		version:        MustParseEtcdVersion(opts.targetVersion),
 		storageVersion: MustParseEtcdStorageVersion(opts.targetStorage),
 	}
 
-	migrate(opts.name, opts.port, opts.peerListenUrls, opts.peerAdvertiseUrls, opts.binDir, opts.dataDir, opts.etcdDataPrefix, opts.ttlKeysDirectory, opts.initialCluster, target, bundledVersions, opts.etcdServerArgs)
+	migrate(
+		opts.name, opts.port, opts.peerListenUrls, opts.peerAdvertiseUrls, opts.binDir,
+		opts.dataDir, opts.etcdDataPrefix, opts.ttlKeysDirectory, opts.initialCluster,
+		target, opts.supportedVersions, opts.etcdServerArgs)
+}
+
+func copyBinaries() {
+	if val, err := lookupEnv("DO_NOT_MOVE_BINARIES"); err != nil || val != "true" {
+		etcdVersioned := fmt.Sprintf("etcd-%s", opts.targetVersion)
+		etcdctlVersioned := fmt.Sprintf("etcdctl-%s", opts.targetVersion)
+		if err := copyFile(filepath.Join(opts.binDir, etcdVersioned), filepath.Join(opts.binDir, "etcd")); err != nil {
+			klog.Fatalf("Failed to copy %s: %v", etcdVersioned, err)
+		}
+		if err := copyFile(filepath.Join(opts.binDir, etcdctlVersioned), filepath.Join(opts.binDir, "etcdctl")); err != nil {
+			klog.Fatalf("Failed to copy %s: %v", etcdctlVersioned, err)
+		}
+	}
 }
 
 // migrate opens or initializes the etcd data directory, configures the migrator, and starts the migration.
@@ -142,8 +89,7 @@ func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls
 
 	dataDir, err := OpenOrCreateDataDirectory(dataDirPath)
 	if err != nil {
-		klog.Errorf("Error opening or creating data directory %s: %v", dataDirPath, err)
-		os.Exit(1)
+		klog.Fatalf("Error opening or creating data directory %s: %v", dataDirPath, err)
 	}
 
 	cfg := &EtcdMigrateCfg{
@@ -161,31 +107,14 @@ func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls
 	}
 	client, err := NewEtcdMigrateClient(cfg)
 	if err != nil {
-		klog.Errorf("Migration failed: %v", err)
-		os.Exit(1)
+		klog.Fatalf("Migration failed: %v", err)
 	}
 	defer client.Close()
 
 	migrator := &Migrator{cfg, dataDir, client}
 
 	err = migrator.MigrateIfNeeded(target)
 	if err != nil {
-		klog.Errorf("Migration failed: %v", err)
-		os.Exit(1)
-	}
-}
-
-// validateBundledVersions checks that 'etcd-<version>' and 'etcdctl-<version>' binaries exist in the binDir
-// for each version in the bundledVersions list.
-func validateBundledVersions(bundledVersions SupportedVersions, binDir string) error {
-	for _, v := range bundledVersions {
-		for _, binaryName := range []string{"etcd", "etcdctl"} {
-			fn := filepath.Join(binDir, fmt.Sprintf("%s-%s", binaryName, v))
-			if _, err := os.Stat(fn); err != nil {
-				return fmt.Errorf("failed to validate '%s' binary exists for bundled-version '%s': %v", fn, v, err)
-			}
-
-		}
+		klog.Fatalf("Migration failed: %v", err)
 	}
-	return nil
 }