basic-storage-scale-multi-node.sh

#!/bin/sh
#
# Setup a simple gluster environment and export a volume through NFS-Ganesha.
#
# This script uses the following environment variables:/
# - GLUSTER_VOLUME: name of the gluster volume to create
#                   this name will also be used as name for the export
#
# The YUM_REPO and GERRIT_* variables are mutually exclusive.
#
# - YUM_REPO: URL to the yum repository (.repo file) for the NFS-Ganesha
#             packages. When this option is used, libntirpc-latest is enabled
#             as well. Leave empty in case patches from Gerrit need testing.
#
# - GERRIT_HOST: when triggered from a new patch submission, this is set to the
#                git server that contains the repository to use.
#
# - GERRIT_PROJECT: project that triggered the build (like ffilz/nfs-ganesha).
#
# - GERRIT_REFSPEC: git tree-ish that can be fetched and checked-out for testing.

set -x

#THE FOLLOWING LINES OF CODE DOWNLOADS STORAGE SCALE, INSTALLS IT AND CREATES A CLUSTER
#----------------------------------------------------------------------------------------------
WORKSPACE_PATH=$(pwd)
WORKING_DIR="$WORKSPACE_PATH/DOWNLOAD_STORAGE_SCALE"
mkdir -p $WORKING_DIR
cd $WORKING_DIR
echo $PWD
dnf install -y unzip
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
ls -ltr
unzip -qq awscliv2.zip
chmod +x ./aws/*
./aws/install
aws --version
aws configure set aws_access_key_id ${AWS_ACCESS_KEY}
aws configure set aws_secret_access_key ${AWS_SECRET_KEY}
aws s3api get-object --bucket centos-ci --key "version_to_use.txt" "version_to_use.txt"
VERSION_TO_USE=$(cat version_to_use.txt)
echo ${VERSION_TO_USE}
aws s3api get-object --bucket centos-ci --key "${VERSION_TO_USE}" "${VERSION_TO_USE}"
mkdir "$WORKING_DIR/INSTALL_PATH"
unzip ${VERSION_TO_USE} -d INSTALLER_PATH/

ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod og-wx ~/.ssh/authorized_keys

yum -y install kernel-devel-$(uname -r) kernel-headers-$(uname -r) cpp gcc gcc-c++ binutils numactl jre make elfutils elfutils-devel rpcbind sssd-tools openldap-clients bind-utils net-tools krb5-workstation python3
python3 -m pip install --user ansible

#Add CES IP to /etc/hosts
ip_address=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)

for new_ip in $(echo $ip_address | awk -F '.' '{for(i=$4+1;i<=255;i++){print $1"."$2"."$3"."i}}'); do ping -c 2 $new_ip; if [ "$?" == "1" ]; then USABLE_IP=$new_ip; break; fi; done

echo "$USABLE_IP    cesip1" >> /etc/hosts

INSTALLER_VERSION=$(ls INSTALLER_PATH/  --ignore="*.md5" --ignore="*.README" --ignore="*.pgp")
INSTALLER=$(readlink -f INSTALLER_PATH/${INSTALLER_VERSION})
chmod +x $INSTALLER
$INSTALLER --silent

export PATH="$PATH:$(readlink -f /usr/lpp/mmfs/*/ansible-toolkit/)"

spectrumscale setup -s 127.0.0.1 --storesecret;
spectrumscale node add $(hostname) -a -n -p;
spectrumscale node add $(node2)  -n -p;
spectrumscale node add $(node3)  -n -p;
spectrumscale config protocols -e $USABLE_IP;
spectrumscale node add -a $(hostname);
spectrumscale config gpfs -c $(hostname)_cluster;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a1 bs=1M count=8192;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a2 bs=1M count=8192;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a3 bs=1M count=8192;
spectrumscale nsd add -p $(hostname) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a1;
spectrumscale nsd add -p $(node2) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a2;
spectrumscale nsd add -p $(node3) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a3;
spectrumscale config protocols -f ${STORAGE_SCALE_VOLUME} -m /ibm/${STORAGE_SCALE_VOLUME};
spectrumscale enable nfs;
spectrumscale enable smb;
spectrumscale callhome disable;
spectrumscale config perfmon -r off;
spectrumscale node list;
spectrumscale install --precheck;
spectrumscale install;
spectrumscale deploy --precheck;
spectrumscale deploy;

spectrumscale nsd list
spectrumscale filesystem list

#----------------------------------------------------------------------------------------------

#THE FOLLOWING LINES OF CODE CLONES THE SOURCE CODE, RPMBUILD AND INSTALLS THE RPMS
#----------------------------------------------------------------------------------------------
# make sure rpcbind is running
dnf -y install rpcbind
systemctl start rpcbind

echo 'TODO: this is BAD, needs a fix in the selinux-policy'
sudo setenforce 0

systemctl stop firewalld || true

# enable repositories
dnf -y install centos-release-gluster yum-utils centos-release-ceph epel-release unzip

if [ -n "${YUM_REPO}" ]
then
	yum-config-manager --add-repo=http://artifacts.ci.centos.org/nfs-ganesha/nightly/libntirpc/libntirpc-latest.repo
	yum-config-manager --add-repo=${YUM_REPO}

	# install the latest version of gluster
	dnf -y install nfs-ganesha nfs-ganesha-gluster glusterfs-ganesha

	# start nfs-ganesha service
	if ! systemctl start nfs-ganesha
	then
		echo "+++ systemctl status nfs-ganesha.service +++"
		systemctl status nfs-ganesha.service
		echo "+++ journalctl -xe +++"
		journalctl -xe
		exit 1
	fi
else
	[ -n "${GERRIT_HOST}" ]
	[ -n "${GERRIT_PROJECT}" ]
	[ -n "${GERRIT_REFSPEC}" ]

	GIT_REPO=$(basename "${GERRIT_PROJECT}")
	GIT_URL="https://${GERRIT_HOST}/${GERRIT_PROJECT}"

        BASE_PACKAGES="git bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel rpm-build redhat-rpm-config gdb"
        BUILDREQUIRES_EXTRA="libnsl2-devel libnfsidmap-devel libwbclient-devel userspace-rcu-devel libcephfs-devel"
        if [ "${CENTOS_VERSION}" = "7" ]; then
            yum -y install libgfapi-devel
            yum -y install ${BASE_PACKAGES} libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel userspace-rcu-devel userspace-rcu python2-devel
        elif [ "${CENTOS_VERSION}" = "8s" ]; then
            yum install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel
            yum install --enablerepo=powertools -y ${BUILDREQUIRES_EXTRA}
            yum -y install selinux-policy-devel sqlite 
        elif [ "${CENTOS_VERSION}" = "9s" ]; then
            dnf install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel
            dnf install --enablerepo=crb -y ${BUILDREQUIRES_EXTRA}
            dnf -y install selinux-policy-devel sqlite
        fi

	git init "${GIT_REPO}"
	pushd "${GIT_REPO}"

        #Its observed that fetch is failing so this little hack is added! Will delete in future if it turns out useless!
	git fetch --depth=1 "${GIT_URL}" "${GERRIT_REFSPEC}" > /dev/null
        if [ $? = 0 ]; then
            echo "Fetch succeeded"
        else
            sleep 2
            git fetch "${GIT_URL}" "${GERRIT_REFSPEC}"
        fi       

	git checkout -b "${GERRIT_REFSPEC}" FETCH_HEAD

	# update libntirpc
	git submodule update --recursive --init || git submodule sync

	mkdir build
	pushd build

	cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_GPFS=ON -DUSE_DBUS=ON -D_MSPAC_SUPPORT=OFF ../src
	make dist
	rpmbuild -ta --define "_srcrpmdir $PWD" --define "_rpmdir $PWD" *.tar.gz
	rpm_arch=$(rpm -E '%{_arch}')
	ganesha_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p *.src.rpm)

	if [ -e ${rpm_arch}/libntirpc-devel*.rpm ]; then
		ntirpc_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p ${rpm_arch}/libntirpc-devel*.rpm)
		ntirpc_rpm=${rpm_arch}/libntirpc-${ntirpc_version}.${rpm_arch}.rpm
	fi

    rpm -e gpfs.nfs-ganesha gpfs.nfs-ganesha-gpfs --nodeps
	dnf -y install {x86_64,noarch}/*.rpm

	# Test block
	ulimit -a
	ulimit -c unlimited
	ulimit -a

	# start nfs-ganesha service with an empty configuration
	echo "NFSv4 { Graceless = true; }" > /etc/ganesha/ganesha.conf
     
	# This block is introduced as the line creates a ambiguity as the same is used in scale implementation
	systemctl stop nfs-ganesha
	sed -i.bak -e 's/^StateDirectory/#&/' /usr/lib/systemd/system/nfs-ganesha.service
	systemctl daemon-reload

	if ! systemctl start nfs-ganesha
	then
		echo "+++ systemctl status nfs-ganesha.service +++"
		systemctl status nfs-ganesha.service
		echo "+++ journalctl -xe +++"
		journalctl -xe
		exit 1
	fi
fi
#----------------------------------------------------------------------------------------------


#EXPORT THE NFS VOLUME
#----------------------------------------------------------------------------------------------
/usr/lpp/mmfs/bin/mmuserauth service create --data-access-method file --type userdefined
/usr/lpp/mmfs/bin/mmnfs export add /ibm/${STORAGE_SCALE_VOLUME} -c "*(Access_Type=RW,Squash=none)"

#CHECKS TO SEE IF THE VOLUME IS WORKING
#----------------------------------------------------------------------------------------------

#There's a duplicate line in the file - /var/mmfs/ces/nfs-config/gpfs.ganesha.main.conf which fails to restart
systemctl stop nfs-ganesha
/usr/lpp/mmfs/bin/mmnfs config change MINOR_VERSIONS=0,1
sleep 20
sed -i.bak -e '41d' /var/mmfs/ces/nfs-config/gpfs.ganesha.main.conf
sleep 5
systemctl daemon-reload
if ! systemctl start nfs-ganesha
then
    echo "+++ systemctl status nfs-ganesha.service +++"
    systemctl status nfs-ganesha.service
    echo "+++ journalctl -xe +++"
    journalctl -xe
    exit 1
fi

systemctl status nfs-ganesha