|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +set -x |
| 4 | + |
| 5 | +# save old -e status |
| 6 | +if [[ $- = *e* ]]; then |
| 7 | + olde=-e |
| 8 | +else |
| 9 | + olde=+e |
| 10 | +fi |
| 11 | + |
| 12 | +set -e |
| 13 | + |
| 14 | +BOSCO_KEY=/etc/osg/bosco.key |
| 15 | +ENDPOINT_CONFIG=/etc/endpoints.ini |
| 16 | +SKIP_WN_INSTALL=no |
| 17 | + |
| 18 | +function errexit { |
| 19 | + echo "$1" >&2 |
| 20 | + exit 1 |
| 21 | +} |
| 22 | + |
| 23 | + |
| 24 | +function debug_file_contents { |
| 25 | + filename=$1 |
| 26 | + echo "Contents of $filename" |
| 27 | + echo "====================" |
| 28 | + cat "$filename" |
| 29 | + echo "====================" |
| 30 | +} |
| 31 | + |
| 32 | +function fetch_remote_os_info { |
| 33 | + ruser=$1 |
| 34 | + rhost=$2 |
| 35 | + ssh -q -i $BOSCO_KEY "$ruser@$rhost" "cat /etc/os-release" |
| 36 | +} |
| 37 | + |
| 38 | +setup_ssh_config () { |
| 39 | + echo "Adding user ${ruser}" |
| 40 | + ssh_dir="/home/${ruser}/.ssh" |
| 41 | + # setup user and SSH dir |
| 42 | + adduser --base-dir /home/ "${ruser}" |
| 43 | + mkdir -p $ssh_dir |
| 44 | + chown "${ruser}": $ssh_dir |
| 45 | + chmod 700 $ssh_dir |
| 46 | + |
| 47 | + # copy Bosco key |
| 48 | + ssh_key=$ssh_dir/bosco_key.rsa |
| 49 | + cp $BOSCO_KEY $ssh_key |
| 50 | + chmod 600 $ssh_key |
| 51 | + chown "${ruser}": $ssh_key |
| 52 | + |
| 53 | + ssh_config=$ssh_dir/config |
| 54 | + cat <<EOF > "$ssh_config" |
| 55 | +Host $remote_fqdn |
| 56 | + Port $remote_port |
| 57 | + IdentityFile ${ssh_key} |
| 58 | + IdentitiesOnly yes |
| 59 | +EOF |
| 60 | + debug_file_contents "$ssh_config" |
| 61 | + |
| 62 | + # setup known hosts |
| 63 | + known_hosts=$ssh_dir/known_hosts |
| 64 | + echo "$REMOTE_HOST_KEY" >> "$known_hosts" |
| 65 | + debug_file_contents $known_hosts |
| 66 | + |
| 67 | + for ssh_file in $ssh_dir/config $ssh_dir/known_hosts; do |
| 68 | + chown "${ruser}": "$ssh_file" |
| 69 | + done |
| 70 | + |
| 71 | + # debugging |
| 72 | + ls -l "$ssh_dir" |
| 73 | +} |
| 74 | + |
| 75 | + |
| 76 | +# Install the WN client, CAs, and CRLs on the remote host |
| 77 | +# Store logs in /var/log/condor-ce/ to simplify serving logs via Kubernetes |
| 78 | +setup_endpoints_ini () { |
| 79 | + echo "Setting up endpoint.ini entry for ${ruser}@$remote_fqdn..." |
| 80 | + remote_os_major_ver=$1 |
| 81 | + # The WN client updater uses "remote_dir" for WN client |
| 82 | + # configuration and remote copy. We need the absolute path |
| 83 | + # specifically for fetch-crl |
| 84 | + remote_home_dir=$(ssh -q -i $BOSCO_KEY "${ruser}@$remote_fqdn" pwd) |
| 85 | + osg_ver=3.4 |
| 86 | + if [[ $remote_os_major_ver -gt 6 ]]; then |
| 87 | + osg_ver=3.5 |
| 88 | + fi |
| 89 | + cat <<EOF >> $ENDPOINT_CONFIG |
| 90 | +[Endpoint ${RESOURCE_NAME}-${ruser}] |
| 91 | +local_user = ${ruser} |
| 92 | +remote_host = $remote_fqdn |
| 93 | +remote_user = ${ruser} |
| 94 | +remote_dir = $remote_home_dir/bosco-osg-wn-client |
| 95 | +upstream_url = https://repo.opensciencegrid.org/tarball-install/${osg_ver}/osg-wn-client-latest.el${remote_os_major_ver}.x86_64.tar.gz |
| 96 | +EOF |
| 97 | +} |
| 98 | + |
| 99 | +# $REMOTE_HOST needs to be specified in the environment |
| 100 | +remote_fqdn=${REMOTE_HOST%:*} |
| 101 | +if [[ $REMOTE_HOST =~ :[0-9]+$ ]]; then |
| 102 | + remote_port=${REMOTE_HOST#*:} |
| 103 | +else |
| 104 | + remote_port=22 |
| 105 | +fi |
| 106 | + |
| 107 | +REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` |
| 108 | +[[ -n $REMOTE_HOST_KEY ]] || errexit "Failed to determine host key for $remote_fqdn:$remote_port" |
| 109 | + |
| 110 | +# HACK: Symlink the Bosco key to the location expected by |
| 111 | +# bosco_cluster so it doesn't go and try to generate a new one |
| 112 | +root_ssh_dir=/root/.ssh/ |
| 113 | +mkdir -p $root_ssh_dir |
| 114 | +chmod 700 $root_ssh_dir |
| 115 | +ln -s $BOSCO_KEY $root_ssh_dir/bosco_key.rsa |
| 116 | + |
| 117 | +cat <<EOF > /etc/ssh/ssh_config |
| 118 | +Host $remote_fqdn |
| 119 | + Port $remote_port |
| 120 | + IdentityFile ${BOSCO_KEY} |
| 121 | + ControlMaster auto |
| 122 | + ControlPath /tmp/cm-%i-%r@%h:%p |
| 123 | + ControlPersist 15m |
| 124 | +EOF |
| 125 | +debug_file_contents /etc/ssh/ssh_config |
| 126 | + |
| 127 | +echo "$REMOTE_HOST_KEY" >> /etc/ssh/ssh_known_hosts |
| 128 | +debug_file_contents /etc/ssh/ssh_known_hosts |
| 129 | + |
| 130 | +# Populate the bosco override dir from a Git repo |
| 131 | +if [[ -n $BOSCO_GIT_ENDPOINT && -n $BOSCO_DIRECTORY ]]; then |
| 132 | + OVERRIDE_DIR=/etc/condor-ce/bosco_override |
| 133 | + /usr/local/bin/bosco-override-setup.sh "$BOSCO_GIT_ENDPOINT" "$BOSCO_DIRECTORY" /etc/osg/git.key |
| 134 | +fi |
| 135 | +unset GIT_SSH_COMMAND |
| 136 | + |
| 137 | +users=$(cat /etc/grid-security/grid-mapfile /etc/grid-security/voms-mapfile | \ |
| 138 | + awk '/^"[^"]+" +[a-zA-Z0-9\-\._]+$/ {print $NF}' | \ |
| 139 | + sort -u) |
| 140 | +[[ -n $users ]] || errexit "Did not find any user mappings in the VOMS or Grid mapfiles" |
| 141 | + |
| 142 | +# Allow the condor user to run the WN client updater as the local users |
| 143 | +CONDOR_SUDO_FILE=/etc/sudoers.d/10-condor-ssh |
| 144 | +condor_sudo_users=`tr ' ' ',' <<< $users` |
| 145 | +echo "condor ALL = ($condor_sudo_users) NOPASSWD: /usr/bin/update-remote-wn-client" \ |
| 146 | + > $CONDOR_SUDO_FILE |
| 147 | +chmod 644 $CONDOR_SUDO_FILE |
| 148 | +
|
| 149 | +grep -qs '^OSG_GRID="/cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client' \ |
| 150 | + /var/lib/osg/osg-job-environment*.conf && SKIP_WN_INSTALL=yes |
| 151 | +
|
| 152 | +# Enable bosco_cluster debug output |
| 153 | +bosco_cluster_opts=(-d ) |
| 154 | +if [[ -n $OVERRIDE_DIR ]]; then |
| 155 | + if [[ -d $OVERRIDE_DIR ]]; then |
| 156 | + bosco_cluster_opts+=(-o "$OVERRIDE_DIR") |
| 157 | + else |
| 158 | + echo "WARNING: $OVERRIDE_DIR is not a directory. Skipping Bosco override." |
| 159 | + fi |
| 160 | +fi |
| 161 | +
|
| 162 | +[[ $REMOTE_BOSCO_DIR ]] && bosco_cluster_opts+=(-b "$REMOTE_BOSCO_DIR") \ |
| 163 | + || REMOTE_BOSCO_DIR=bosco |
| 164 | +
|
| 165 | +echo "Using Bosco tarball: $(bosco_findplatform --url)" |
| 166 | +for ruser in $users; do |
| 167 | + setup_ssh_config |
| 168 | +done |
| 169 | +
|
| 170 | +################### |
| 171 | +# REMOTE COMMANDS # |
| 172 | +################### |
| 173 | +
|
| 174 | +# We have to pick a user for SSH, may as well be the first one |
| 175 | +remote_os_info=$(fetch_remote_os_info "$(printf "%s\n" $users | head -n1)" "$remote_fqdn") |
| 176 | +remote_os_ver=$(echo "$remote_os_info" | awk -F '=' '/^VERSION_ID/ {print $2}' | tr -d '"') |
| 177 | +
|
| 178 | +# Skip WN client installation for non-RHEL-based remote clusters |
| 179 | +[[ $remote_os_info =~ (^|$'\n')ID_LIKE=.*(rhel|centos|fedora) ]] || SKIP_WN_INSTALL=yes |
| 180 | +
|
| 181 | +# HACK: By default, Singularity containers don't specify $HOME and |
| 182 | +# bosco_cluster needs it |
| 183 | +[[ -n $HOME ]] || HOME=/root |
| 184 | +
|
| 185 | +for ruser in $users; do |
| 186 | + echo "Installing remote Bosco installation for ${ruser}@$remote_fqdn" |
| 187 | + [[ $SKIP_WN_INSTALL == 'no' ]] && setup_endpoints_ini "${remote_os_ver%%.*}" |
| 188 | + # $REMOTE_BATCH needs to be specified in the environment |
| 189 | + bosco_cluster "${bosco_cluster_opts[@]}" -a "${ruser}@$remote_fqdn" "$REMOTE_BATCH" |
| 190 | +
|
| 191 | + echo "Installing environment files for $ruser@$remote_fqdn..." |
| 192 | + # Copy over environment files to allow for dynamic WN variables (SOFTWARE-4117) |
| 193 | + rsync -av /var/lib/osg/osg-*job-environment.conf \ |
| 194 | + "${ruser}@$remote_fqdn:$REMOTE_BOSCO_DIR/glite/etc" |
| 195 | +done |
| 196 | +
|
| 197 | +if [[ $SKIP_WN_INSTALL == 'no' ]]; then |
| 198 | + echo "Installing remote WN client tarballs..." |
| 199 | + sudo -u condor update-all-remote-wn-clients --log-dir /var/log/condor-ce/ |
| 200 | +else |
| 201 | + echo "SKIP_WNCLIENT = True" > /etc/condor-ce/config.d/50-skip-wnclient-cron.conf |
| 202 | + echo "Skipping remote WN client tarball installation, using CVMFS..." |
| 203 | +fi |
| 204 | +
|
| 205 | +set $olde |
0 commit comments