Added the git workload

Author: rohankadekodi

.gitignore

@@ -16,6 +16,8 @@ sqlite3-trace/.libs
 rsync/workload/
 rsync/rsync
 tar/workload/
+git/workload/
+linux-4.18.10.tar.gz
 
 !ycsb/core
 !ycsb/tracerecorder

README.md

@@ -81,14 +81,14 @@ $ rm -rf /mnt/pmem_emul/*
 2. `dependencies/` contains packages and scripts to resolve dependencies
 3. `kernel/` contains the Linux 4.13.0 kernel
 4. `micro/` contains the microbenchmark
-4. `leveldb/` contains LevelDB source code
-5. `rsync/` contains the rsync source code
-6. `scripts/` contains scripts to compile and run workloads and kernel
-7. `splitfs-so/` contains the SplitFS-strict shared libraries for running different workloads
-8. `sqlite3-trace/` contains SQLite3 source code
-9. `tpcc-sqlite/` contains TPCC source code
-10. `ycsb/` contains YCSB source code
-11. `tar/` contains tar source code
+5. `leveldb/` contains LevelDB source code
+6. `rsync/` contains the rsync source code
+7. `scripts/` contains scripts to compile and run workloads and kernel
+8. `splitfs-so/` contains the SplitFS-strict shared libraries for running different workloads
+9. `sqlite3-trace/` contains SQLite3 source code
+10. `tpcc-sqlite/` contains TPCC source code
+11. `ycsb/` contains YCSB source code
+12. `tar/` contains tar source code
 
 The [Experiments
 page](https://github.com/utsaslab/SplitFS/blob/master/experiments.md)

experiments.md

@@ -45,6 +45,7 @@ We evaluate and benchmark on SplitFS using different application benchmarks like
 5. TPCC: `cd scripts/tpcc; ./compile_tpcc.sh; cd ../..` -- This will compile TPCC workload
 6. rsync: `cd scripts/rsync; ./compile_rsync.sh; cd ../..` -- This will compile rsync
 7. tar: `cd scripts/tar; ./compile_tar.sh; cd ../..` -- This will compile tar
+8. git: Does not require any compilation
 
 Note: The <num_threads> argument in the compilation scripts performs the compilation with the number of threads given as input to the script, to improve the speed of compilation. 
 
@@ -54,23 +55,25 @@ Note: The <num_threads> argument in the compilation scripts performs the compila
 
 1. YCSB: `cd scripts/ycsb; ./gen_workloads.sh; cd ../..` -- This will generate the YCSB workload files to be run with LevelDB, because YCSB does not natively support LevelDB, and has been added to the benchmarks of LevelDB
 2. TPCC: `cd scripts/tpcc; ./gen_workload.sh; cd ../..` -- This will create an initial database on SQLite on which to run the TPCC workload
-3. rsync: `cd scripts/rsync/; sudo ./rsync_gen_workload.sh; cd ../..` -- This will create the rsync workload according to the backup data distribution as mentioned in the Paper
+3. rsync: `cd scripts/rsync/; sudo ./rsync_gen_workload.sh; cd ../..` -- This will create the rsync workload according to the backup data distribution as mentioned in the paper
 4. tar: `cd scripts/tar/; sudo ./gen_workload.sh; cd ../..` -- This will create the tar workload as mentioned in the paper
+5. git: `cd scripts/git/; sudo ./gen_workload.sh; cd ../..` -- This will create the git workload as mentioned in the paper
 
 ---
 
 ### Run Application Workloads
 
-1. YCSB: `cd scripts/ycsb; ./run_ycsb.sh; cd ../..` -- This will run all the YCSB workloads on LevelDB (Load A, Run A-F, Load E, Run E) with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-strict` 
+1. YCSB: `cd scripts/ycsb; ./run_ycsb.sh; cd ../..` -- This will run all the YCSB workloads on LevelDB (Load A, Run A-F, Load E, Run E) with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-strict`
 2. TPCC: `cd scripts/tpcc; ./run_tpcc.sh; cd ../..` -- This will run the TPCC workload on SQLite3 with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-POSIX`
 3. rsync: `cd scripts/rsync; ./run_rsync.sh; cd ../..` -- This will run the rsync workload with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-sync`
 4. tar: `cd scripts/tar; ./run_tar.sh; cd ../..` -- This will run the tar workload with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-POSIX, SplitFS-sync, SplitFS-strict`
+5. git: `cd scripts/git; ./run_git.sh; cd ../..` -- This will run the git workload with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-POSIX, SplitFS-sync, SplitFS-strict`
 
 ---
 
 ### Run Software Overhead Workloads
 
-1. YCSB: `cd scripts/ycsb; ./run_ycsb_soft.sh; cd ../..` -- This will run all the YCSB workloads on LevelDB (Load A, Run A) with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-strict, SplitFS-sync, SplitFS-POSIX` 
+1. YCSB: `cd scripts/ycsb; ./run_ycsb_soft.sh; cd ../..` -- This will run all the YCSB workloads on LevelDB (Load A, Run A) with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-strict, SplitFS-sync, SplitFS-POSIX`
 2. TPCC: `cd scripts/tpcc; ./run_tpcc_soft.sh; cd ../..` -- This will run the TPCC workload on SQLite3 with `ext4-DAX, NOVA strict, NOVA Relaxed, PMFS, SplitFS-strict, SplitFS-sync, SplitFS-POSIX`
 
 ---

scripts/git/Makefile

@@ -0,0 +1,40 @@
+# Run tests for splitfs
+
+# The current working directory
+CWD=$(shell pwd)
+# Root of the repo
+ROOT=$(CWD)/../..
+# SplitFS path
+SFS_PATH=/mnt/pmem_emul/
+
+all: splitfs.posix splitfs.sync splitfs.strict
+
+# Compile with posix specific flags
+splitfs.posix_compile:
+	export LEDGER_DATAJ=0 && \
+    export LEDGER_POSIX=1 && \
+	export LEDGER_GIT=1 && \
+    $(MAKE) -C ${ROOT}/splitfs clean && \
+    $(MAKE) -e -C ${ROOT}/splitfs
+
+# Compile with sync specific flags
+splitfs.sync_compile:
+	export LEDGER_DATAJ=0 && \
+    export LEDGER_POSIX=0 && \
+	export LEDGER_GIT=1 && \
+    $(MAKE) -C ${ROOT}/splitfs clean && \
+    $(MAKE) -e -C ${ROOT}/splitfs
+
+# Compile with strict specific flags
+splitfs.strict_compile:
+	export LEDGER_DATAJ=1 && \
+    export LEDGER_POSIX=0 && \
+	export LEDGER_GIT=1 && \
+    $(MAKE) -C ${ROOT}/splitfs clean && \
+    $(MAKE) -e -C ${ROOT}/splitfs
+
+splitfs.posix: splitfs.posix_compile
+
+splitfs.sync: splitfs.sync_compile
+
+splitfs.strict: splitfs.strict_compile

scripts/git/gen_workload.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -x
+
+cur_dir=$(pwd)
+root_dir=`readlink -f ../..`
+pmem_dir=/mnt/pmem_emul
+linux_tar=linux-4.18.10.tar.gz
+linux_dir=linux-4.18.10
+
+cd $root_dir
+rm $linux_tar
+wget https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.18.10.tar.gz
+
+rm -rf $pmem_dir/repo
+mkdir -p $pmem_dir/repo
+cp $root_dir/$linux_tar $pmem_dir/repo
+cd $pmem_dir/repo
+
+for i in {1..9}
+do
+    mkdir folder$i
+done
+
+tar -xf $linux_tar
+for i in {1..9}
+do
+    cp -r $linux_dir folder$i/
+done
+
+rm -rf $linux_dir
+rm $linux_tar
+
+cd $cur_dir
+rm -rf $root_dir/git/workload/*
+mkdir -p $root_dir/git/workload
+cp -r $pmem_dir/repo $root_dir/git/workload/
+

scripts/git/run_fs.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: sudo ./run_fs.sh <fs> <run_id>"
+    exit 1
+fi
+
+set -x
+
+workload=git
+fs=$1
+run_id=$2
+current_dir=$(pwd)
+git_dir=`readlink -f ../../git`
+workload_dir=$git_dir/workload
+pmem_dir=/mnt/pmem_emul
+boost_dir=`readlink -f ../../splitfs`
+result_dir=`readlink -f ../../results`
+fs_results=$result_dir/$fs/$workload
+
+if [ "$fs" == "splitfs-posix" ]; then
+    run_boost=1
+elif [ "$fs" == "splitfs-sync" ]; then
+    run_boost=1
+elif [ "$fs" == "splitfs-strict" ]; then
+    run_boost=1
+else
+    run_boost=0
+fi
+
+ulimit -c unlimited
+
+echo Sleeping for 5 seconds . . 
+sleep 5
+
+run_workload()
+{
+
+    echo ----------------------- GIT WORKLOAD ---------------------------
+
+    mkdir -p $fs_results
+    rm $fs_results/run$run_id
+
+    rm -rf $pmem_dir/*
+    cp -r $workload_dir/repo $pmem_dir && sync
+    sync && echo 3 > /proc/sys/vm/drop_caches && sync
+
+    cd $pmem_dir
+    cd repo
+    git init
+
+    sleep 5
+
+    date
+
+    if [ $run_boost -eq 1 ]; then
+        time $boost_dir/run_boost.sh -p $boost_dir -t nvp_nvp.tree git add . 2>&1 | tee $fs_results/run$run_id
+    else
+        time git add . 2>&1 | tee $fs_results/run$run_id
+    fi
+
+    date
+
+    cd $current_dir
+
+    echo Sleeping for 5 seconds . .
+    sleep 5
+
+}
+
+
+run_workload
+
+cd $current_dir

scripts/git/run_git.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+current_dir=$(pwd)
+setup_dir=`readlink -f ../configs`
+pmem_dir=/mnt/pmem_emul
+
+run_git()
+{
+    fs=$1
+    for run in 1 2 3
+    do
+        sudo rm -rf $pmem_dir/*
+        sudo taskset -c 0-7 ./run_fs.sh $fs $run
+        sleep 5
+    done
+}
+
+echo "-- ext4-DAX --"
+sudo $setup_dir/dax_config.sh
+run_git dax
+
+echo "-- NOVA Relaxed --"
+cd $setup_dir
+sudo ./nova_relaxed_config.sh
+cd $current_dir
+run_git relaxed_nova
+
+echo "-- PMFS --"
+cd $setup_dir
+sudo $setup_dir/pmfs_config.sh
+cd $current_dir
+run_git pmfs
+
+echo "-- NOVA --"
+cd $setup_dir
+sudo ./nova_config.sh
+cd $current_dir
+run_git nova
+
+echo "-- SplitFS Posix --"
+make splitfs.posix
+sudo $setup_dir/dax_config.sh
+run_git splitfs-posix
+
+echo "-- SplitFS Sync --"
+make splitfs.sync
+sudo $setup_dir/dax_config.sh
+run_git splitfs-sync
+
+echo "-- SplitFS Strict --"
+make splitfs.strict
+sudo $setup_dir/dax_config.sh
+run_git splitfs-strict

scripts/tar/gen_workload.sh

@@ -12,6 +12,7 @@ cd $root_dir
 rm $linux_tar
 wget https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.18.10.tar.gz
 
+rm -rf $pmem_dir/repo
 mkdir -p $pmem_dir/repo
 cp $root_dir/$linux_tar $pmem_dir/repo
 cd $pmem_dir/repo