Allow to configure --batch-size

Author: hoijui

run/fill-db

@@ -19,6 +19,7 @@ script_name="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
 
 # initial default values
 APP_NAME="RDF DB Filler"
+BATCH_SIZE_DEFAULT=5000
 CLONE_URL_OKH_ONTOLOGY="https://github.com/iop-alliance/OpenKnowHow.git"
 CLONE_URL_OLD_DATA="https://gitlab.opensourceecology.de/verein/projekte/okh/data.git"
 CLONE_URL_EXP_DATA="[email protected]:OSEGermany/OKH-data-experimental.git"
@@ -27,6 +28,7 @@ data_url="$CLONE_URL_OLD_DATA"
 data_branch="main"
 cleanup=false
 online=true
+batch_size="$BATCH_SIZE_DEFAULT"
 
 function print_help() {
 
@@ -37,6 +39,8 @@ function print_help() {
 	echo "Options:"
 	echo "  -h, --help"
 	echo "    Print this usage help and exit"
+	echo "  --batch-size <NUMBER>"
+	echo "    The number of Turtle fiels to load into the DB at once [default: $BATCH_SIZE_DEFAULT]"
 	echo "  --offline"
 	echo "    Do not try to fetch git repos"
 	echo "  --experimental"
@@ -54,7 +58,7 @@ function print_help() {
 	echo "  $script_name --help"
 	echo "  $script_name --experimental"
 	echo "  $script_name --custom-data \"$CLONE_URL_OLD_DATA\" main"
-	echo "  $script_name --local-data /data"
+	echo "  $script_name --local-data /data --batch-size 1000"
 	echo "  $script_name --samples"
 }
 
@@ -70,6 +74,10 @@ do
 			print_help
 			exit 0
 			;;
+		--batch-size)
+			batch_size="$1"
+			shift
+			;;
 		-o|--offline)
 			online=false
 			;;
@@ -187,7 +195,7 @@ find \
 	"$data_ttls_root" \
 	-name "*.ttl" \
 	> "$ttl_list_file"
-time while mapfile -t -n 10000 batch && ((${#batch[@]}))
+time while mapfile -t -n "$batch_size" batch && ((${#batch[@]}))
 do
 	"$jena_db_data_injector" \
 		--loc "$db_tmp_dir" \