Merge pull request #367 from github/lildude/filter-cluster-names

Filter cluster nodes by role during backup and restore

Author: lildude

script/cibuild

@@ -56,7 +56,7 @@ fi
 
 # Build the deb related packages
 echo "Building deb package ..."
-if script/package-deb 1>package-deb-out.txt 2>package-deb-err.txt; then
+if DEB_BUILD_OPTIONS=nocheck script/package-deb 1>package-deb-out.txt 2>package-deb-err.txt; then
     pkg_files="$pkg_files $(cat package-deb-out.txt)"
 else
     echo "Package build failed:"

share/github-backup-utils/ghe-backup-alambic-cluster

@@ -33,7 +33,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # Alambic server hostnames
-hostnames=$(ghe_cluster_online_nodes "storage-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "storage-server")
 
 ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
 ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"

share/github-backup-utils/ghe-backup-alambic-cluster-ng

@@ -34,7 +34,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # storage server hostnames
-hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "storage-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "storage-server")
 
 tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX)
 ghe-ssh "$GHE_HOSTNAME" -- mkdir -p $tempdir

share/github-backup-utils/ghe-backup-config

@@ -286,13 +286,6 @@ ghe_remote_logger () {
     ghe-ssh "$GHE_HOSTNAME" -- logger -t backup-utils || true
 }
 
-# Usage: ghe_cluster_online_nodes role
-# Returns the online nodes with a certain role in cluster
-ghe_cluster_online_nodes () {
-    role=$1
-    echo "ghe-config --get-regexp cluster.*.$role | egrep 'true$' | awk '{ print \$1; }' | awk 'BEGIN { FS=\".\" }; { print \$2 };' | xargs -I{} -n1 bash -c 'if [ \"\$(ghe-config cluster.{}.offline)\" != true ]; then ghe-config cluster.{}.hostname; fi'" | ghe-ssh "$GHE_HOSTNAME" /bin/bash
-}
-
 # Usage: ghe_verbose <message>
 # Log if verbose mode is enabled (GHE_VERBOSE or `-v`).
 ghe_verbose() {

share/github-backup-utils/ghe-backup-git-hooks-cluster

@@ -35,7 +35,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # git server hostnames
-hostnames=$(ghe_cluster_online_nodes "git-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server")
 
 ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
 ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"

share/github-backup-utils/ghe-backup-pages-cluster

@@ -33,7 +33,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # Pages server hostnames
-hostnames=$(ghe_cluster_online_nodes "pages-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "pages-server")
 
 ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
 ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"

share/github-backup-utils/ghe-backup-repositories-cluster

@@ -67,7 +67,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # git server hostnames
-hostnames=$(ghe_cluster_online_nodes "git-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server")
 
 ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
 ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"

share/github-backup-utils/ghe-backup-repositories-cluster-ng

@@ -70,7 +70,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # git server hostnames
-hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server")
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server")
 
 ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
 ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"

share/github-backup-utils/ghe-cluster-hostnames

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-cluster-nodes <host> <prefix>
+#/
+#/ Finds all nodes of the cluster using the config on <host>.
+#/ If it is a 2.8 and later cluster version the results are returned as
+#/ prefix-uuid, otherwise the configured hostnames are returned.
+#/ Also filters nodes based on the prefix role.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-backup-* and ghe-restore-* commands in cluster environments.
+set -e
+
+# Bring in the backup configuration
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+# Check if the REMOTE DATA USER directory is set
+if [ -z $GHE_REMOTE_DATA_USER_DIR ]; then
+  echo "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting"
+  exit 1
+fi
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+GHE_HOSTNAME="$1"
+prefix="$2"
+role=$(echo "$prefix" | cut -d '-' -f1)
+
+if ghe-ssh "$GHE_HOSTNAME" test -f "$GHE_REMOTE_DATA_USER_DIR/common/uuid"; then
+  node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r "$role" -u)
+  hostnames=''
+  for uuid in $node_uuids; do
+    hostnames+="$prefix-$uuid "
+  done
+else
+  hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r "$role" -p)
+fi
+
+echo "$hostnames"