Skip to content

[GPFS] Add mutli node support #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Manimaran-MM merged 1 commit into from
Feb 5, 2026
+234 −0
Merged

[GPFS] Add mutli node support #133

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 234 additions & 0 deletions build_scripts/common/basic-storage-scale-multi-node.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
#!/bin/sh
#
# Setup a simple gluster environment and export a volume through NFS-Ganesha.
#
# This script uses the following environment variables:/
# - GLUSTER_VOLUME: name of the gluster volume to create
# this name will also be used as name for the export
#
# The YUM_REPO and GERRIT_* variables are mutually exclusive.
#
# - YUM_REPO: URL to the yum repository (.repo file) for the NFS-Ganesha
# packages. When this option is used, libntirpc-latest is enabled
# as well. Leave empty in case patches from Gerrit need testing.
#
# - GERRIT_HOST: when triggered from a new patch submission, this is set to the
# git server that contains the repository to use.
#
# - GERRIT_PROJECT: project that triggered the build (like ffilz/nfs-ganesha).
#
# - GERRIT_REFSPEC: git tree-ish that can be fetched and checked-out for testing.

set -x

#THE FOLLOWING LINES OF CODE DOWNLOADS STORAGE SCALE, INSTALLS IT AND CREATES A CLUSTER
#----------------------------------------------------------------------------------------------
WORKSPACE_PATH=$(pwd)
WORKING_DIR="$WORKSPACE_PATH/DOWNLOAD_STORAGE_SCALE"
mkdir -p $WORKING_DIR
cd $WORKING_DIR
echo $PWD
dnf install -y unzip
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
ls -ltr
unzip -qq awscliv2.zip
chmod +x ./aws/*
./aws/install
aws --version
aws configure set aws_access_key_id ${AWS_ACCESS_KEY}
aws configure set aws_secret_access_key ${AWS_SECRET_KEY}
aws s3api get-object --bucket centos-ci --key "version_to_use.txt" "version_to_use.txt"
VERSION_TO_USE=$(cat version_to_use.txt)
echo ${VERSION_TO_USE}
aws s3api get-object --bucket centos-ci --key "${VERSION_TO_USE}" "${VERSION_TO_USE}"
mkdir "$WORKING_DIR/INSTALL_PATH"
unzip ${VERSION_TO_USE} -d INSTALLER_PATH/

ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod og-wx ~/.ssh/authorized_keys

yum -y install kernel-devel-$(uname -r) kernel-headers-$(uname -r) cpp gcc gcc-c++ binutils numactl jre make elfutils elfutils-devel rpcbind sssd-tools openldap-clients bind-utils net-tools krb5-workstation python3
python3 -m pip install --user ansible

#Add CES IP to /etc/hosts
ip_address=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)

for new_ip in $(echo $ip_address | awk -F '.' '{for(i=$4+1;i<=255;i++){print $1"."$2"."$3"."i}}'); do ping -c 2 $new_ip; if [ "$?" == "1" ]; then USABLE_IP=$new_ip; break; fi; done

echo "$USABLE_IP cesip1" >> /etc/hosts

INSTALLER_VERSION=$(ls INSTALLER_PATH/ --ignore="*.md5" --ignore="*.README" --ignore="*.pgp")
INSTALLER=$(readlink -f INSTALLER_PATH/${INSTALLER_VERSION})
chmod +x $INSTALLER
$INSTALLER --silent

export PATH="$PATH:$(readlink -f /usr/lpp/mmfs/*/ansible-toolkit/)"

spectrumscale setup -s 127.0.0.1 --storesecret;
spectrumscale node add $(hostname) -a -n -p;
spectrumscale node add $(node2) -n -p;
spectrumscale node add $(node3) -n -p;
spectrumscale config protocols -e $USABLE_IP;
spectrumscale node add -a $(hostname);
spectrumscale config gpfs -c $(hostname)_cluster;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a1 bs=1M count=8192;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a2 bs=1M count=8192;
dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a3 bs=1M count=8192;
spectrumscale nsd add -p $(hostname) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a1;
spectrumscale nsd add -p $(node2) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a2;
spectrumscale nsd add -p $(node3) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a3;
spectrumscale config protocols -f ${STORAGE_SCALE_VOLUME} -m /ibm/${STORAGE_SCALE_VOLUME};
spectrumscale enable nfs;
spectrumscale enable smb;
spectrumscale callhome disable;
spectrumscale config perfmon -r off;
spectrumscale node list;
spectrumscale install --precheck;
spectrumscale install;
spectrumscale deploy --precheck;
spectrumscale deploy;

spectrumscale nsd list
spectrumscale filesystem list

#----------------------------------------------------------------------------------------------

#THE FOLLOWING LINES OF CODE CLONES THE SOURCE CODE, RPMBUILD AND INSTALLS THE RPMS
#----------------------------------------------------------------------------------------------
# make sure rpcbind is running
dnf -y install rpcbind
systemctl start rpcbind

echo 'TODO: this is BAD, needs a fix in the selinux-policy'
sudo setenforce 0

systemctl stop firewalld || true

# enable repositories
dnf -y install centos-release-gluster yum-utils centos-release-ceph epel-release unzip

if [ -n "${YUM_REPO}" ]
then
yum-config-manager --add-repo=http://artifacts.ci.centos.org/nfs-ganesha/nightly/libntirpc/libntirpc-latest.repo
yum-config-manager --add-repo=${YUM_REPO}

# install the latest version of gluster
dnf -y install nfs-ganesha nfs-ganesha-gluster glusterfs-ganesha

# start nfs-ganesha service
if ! systemctl start nfs-ganesha
then
echo "+++ systemctl status nfs-ganesha.service +++"
systemctl status nfs-ganesha.service
echo "+++ journalctl -xe +++"
journalctl -xe
exit 1
fi
else
[ -n "${GERRIT_HOST}" ]
[ -n "${GERRIT_PROJECT}" ]
[ -n "${GERRIT_REFSPEC}" ]

GIT_REPO=$(basename "${GERRIT_PROJECT}")
GIT_URL="https://${GERRIT_HOST}/${GERRIT_PROJECT}"

BASE_PACKAGES="git bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel rpm-build redhat-rpm-config gdb"
BUILDREQUIRES_EXTRA="libnsl2-devel libnfsidmap-devel libwbclient-devel userspace-rcu-devel libcephfs-devel"
if [ "${CENTOS_VERSION}" = "7" ]; then
yum -y install libgfapi-devel
yum -y install ${BASE_PACKAGES} libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel userspace-rcu-devel userspace-rcu python2-devel
elif [ "${CENTOS_VERSION}" = "8s" ]; then
yum install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel
yum install --enablerepo=powertools -y ${BUILDREQUIRES_EXTRA}
yum -y install selinux-policy-devel sqlite
elif [ "${CENTOS_VERSION}" = "9s" ]; then
dnf install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel
dnf install --enablerepo=crb -y ${BUILDREQUIRES_EXTRA}
dnf -y install selinux-policy-devel sqlite
fi

git init "${GIT_REPO}"
pushd "${GIT_REPO}"

#Its observed that fetch is failing so this little hack is added! Will delete in future if it turns out useless!
git fetch --depth=1 "${GIT_URL}" "${GERRIT_REFSPEC}" > /dev/null
if [ $? = 0 ]; then
echo "Fetch succeeded"
else
sleep 2
git fetch "${GIT_URL}" "${GERRIT_REFSPEC}"
fi

git checkout -b "${GERRIT_REFSPEC}" FETCH_HEAD

# update libntirpc
git submodule update --recursive --init || git submodule sync

mkdir build
pushd build

cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_GPFS=ON -DUSE_DBUS=ON -D_MSPAC_SUPPORT=OFF ../src
make dist
rpmbuild -ta --define "_srcrpmdir $PWD" --define "_rpmdir $PWD" *.tar.gz
rpm_arch=$(rpm -E '%{_arch}')
ganesha_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p *.src.rpm)

if [ -e ${rpm_arch}/libntirpc-devel*.rpm ]; then
ntirpc_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p ${rpm_arch}/libntirpc-devel*.rpm)
ntirpc_rpm=${rpm_arch}/libntirpc-${ntirpc_version}.${rpm_arch}.rpm
fi

rpm -e gpfs.nfs-ganesha gpfs.nfs-ganesha-gpfs --nodeps
dnf -y install {x86_64,noarch}/*.rpm

# Test block
ulimit -a
ulimit -c unlimited
ulimit -a

# start nfs-ganesha service with an empty configuration
echo "NFSv4 { Graceless = true; }" > /etc/ganesha/ganesha.conf

# This block is introduced as the line creates a ambiguity as the same is used in scale implementation
systemctl stop nfs-ganesha
sed -i.bak -e 's/^StateDirectory/#&/' /usr/lib/systemd/system/nfs-ganesha.service
systemctl daemon-reload

if ! systemctl start nfs-ganesha
then
echo "+++ systemctl status nfs-ganesha.service +++"
systemctl status nfs-ganesha.service
echo "+++ journalctl -xe +++"
journalctl -xe
exit 1
fi
fi
#----------------------------------------------------------------------------------------------


#EXPORT THE NFS VOLUME
#----------------------------------------------------------------------------------------------
/usr/lpp/mmfs/bin/mmuserauth service create --data-access-method file --type userdefined
/usr/lpp/mmfs/bin/mmnfs export add /ibm/${STORAGE_SCALE_VOLUME} -c "*(Access_Type=RW,Squash=none)"

#CHECKS TO SEE IF THE VOLUME IS WORKING
#----------------------------------------------------------------------------------------------

#There's a duplicate line in the file - /var/mmfs/ces/nfs-config/gpfs.ganesha.main.conf which fails to restart
systemctl stop nfs-ganesha
/usr/lpp/mmfs/bin/mmnfs config change MINOR_VERSIONS=0,1
sleep 20
sed -i.bak -e '41d' /var/mmfs/ces/nfs-config/gpfs.ganesha.main.conf
sleep 5
systemctl daemon-reload
if ! systemctl start nfs-ganesha
then
echo "+++ systemctl status nfs-ganesha.service +++"
systemctl status nfs-ganesha.service
echo "+++ journalctl -xe +++"
journalctl -xe
exit 1
fi

systemctl status nfs-ganesha