update script to loop over file

Author: ntoulemonde

scripts/2_repo_fetch.json

@@ -1,7 +1,7 @@
 [
   {
     "path_to_folder_to_synchronize_from": "https://github.com/InseeFrLab/satellite-images-docs/tree/main/cards",
-    "last_commit_sha": "",
+    "last_commit_sha": "862f2dc2f0c12d62a38b62de48d096452a1685ca",
     "path_to_folder_to_synchronize_to": "project/2022_satellites/cards",
     "replacements": [
       {
@@ -28,7 +28,7 @@
   },
   {
     "path_to_folder_to_synchronize_from": "https://github.com/InseeFrLab/codif-ape-prez/tree/main/cards",
-    "last_commit_sha": "",
+    "last_commit_sha": "0d474addc0735a610ecb7cc54bd5362114446925",
     "path_to_folder_to_synchronize_to": "project/2022_codif_ape/cards",
     "replacements": [
       {

scripts/2_synchronize.sh

@@ -1,14 +1,14 @@
 # Install jq if not present
-sudo apt-get install jq
+# sudo apt-get install jq
 
 # Function to extract owner, repository name, and path after main/ from a GitHub URL
 extract_github_info() {
     local url="$1"
 
     # Extract owner, repository name, and path after main/
-    local owner=$(echo "$url" | awk -F'/' '{print $4}')
-    local repo=$(echo "$url" | awk -F'/' '{print $5}')
-    local path=$(echo "$url" | awk -F'main/' '{print $2}')
+    local owner=$(echo "$url" | awk -F'/' '{print $4}' | tr -d '"')
+    local repo=$(echo "$url" | awk -F'/' '{print $5}' | tr -d '"')
+    local path=$(echo "$url" | awk -F'main/' '{print $2}' | tr -d '"')  # tr -d to remove trailing "
 
     # Print the extracted information
     echo "$owner $repo $path"
@@ -23,7 +23,7 @@ get_last_commit() {
     local output=$(curl -L \
     -H "Accept: application/vnd.github+json" \
     -H "X-GitHub-Api-Version: 2022-11-28" \
-    "https://api.github.com/repos/$owner/$repo/commits?path=$path")
+    "https://api.github.com/repos/$owner/$repo/commits?path=$path" --silent)
 
     echo "$output" | jq --raw-output '.[0].sha'
 }
@@ -37,7 +37,8 @@ clone_repo() {
 
     mkdir temp
     cd temp
-    git clone "https://github.com/$owner_git/$repo_git"
+    echo "cloning https://github.com/$owner_git/$repo_git"
+    git clone "https://github.com/$owner_git/$repo_git" --quiet --depth 1 --single-branch  # To clone only last commit for efficiency and main branch
     rm -rf "../$path_ssphub"
     cp "$repo_git/$subfolder_git/" "../$path_ssphub" -rf
     cd ../
@@ -71,7 +72,7 @@ replace_in_qmd_files() {
 update_commit_sha() {
     local old_commit_sha="$1"
     local new_commit_sha="$2"
-    local json_file="scripts/2_repo_fetch.json"
+    local json_file="$3"
 
     # Check if the JSON file exists
     if [ ! -f "$json_file" ]; then
@@ -84,7 +85,7 @@ update_commit_sha() {
     '.[] |= if .last_commit_sha == $old_sha then .last_commit_sha = $new_sha else . end' \
     "$json_file" > tmp.json && mv tmp.json "$json_file"
 
-    echo "Updated last_commit_sha from $old_commit_sha to $new_commit_sha in $json_file"
+    echo "Updated last_commit_sha from ${old_commit_sha:0:6}... to ${new_commit_sha:0:6}... in $json_file"
 }
 
 
@@ -100,11 +101,16 @@ main() {
     fi
 
     # Read the JSON file and process each entry
-    jq -c '.[]' "$json_file" | while read -r entry; do
-        local path_to_folder_to_synchronize_from=$(echo "$entry" | jq -r '.path_to_folder_to_synchronize_from')
-        local last_commit_sha=$(echo "$entry" | jq -r '.last_commit_sha')
-        local path_to_folder_to_synchronize_to=$(echo "$entry" | jq -r '.path_to_folder_to_synchronize_to')
-        local replacements=$(echo "$entry" | jq -c '.replacements')
+    # Initialization for number of elements to loop over the file
+    declare -i nb_elem
+    nb_elem=$(jq 'length' "$json_file")
+
+    for ((i=1; i <=nb_elem; i++)); do
+        # Extract values of ith element of the file
+        local path_to_folder_to_synchronize_from=$(jq --argjson index "$((i-1))" '.[$index].path_to_folder_to_synchronize_from' "$json_file")
+        local last_commit_sha=$(jq --argjson index "$((i-1))" '.[$index].last_commit_sha' "$json_file" | tr -d '"')
+        local path_to_folder_to_synchronize_to=$(jq --argjson index "$((i-1))" '.[$index].path_to_folder_to_synchronize_to' "$json_file" | tr -d '"')
+        local replacements=$(jq --argjson index "$((i-1))" '.[$index].replacements' "$json_file")
 
         # Extract owner, repo, and path after main/
         local info=$(extract_github_info "$path_to_folder_to_synchronize_from")
@@ -117,32 +123,47 @@ main() {
 
         # Check if the last commit SHA is different
         if [ "$last_commit_sha" != "$new_commit_sha" ]; then
-            echo "New commit found for $path_to_folder_to_synchronize_from"
+            echo "Commit "${new_commit_sha:0:6}" found for $path_to_folder_to_synchronize_from"
+
+            # Debug
+            echo "will perform the cloning with params
+            last commit : "$last_commit_sha"
+            new_commit_sha : "$new_commit_sha"
+            owner: "$owner"
+            repo: "$repo"
+            path: "$path"
+            path_to_folder_to_synchronize_to: "$path_to_folder_to_synchronize_to"
+            replacement:  "${replacements:0:20}""
+
+            # # Delete branch
+            git branch -D origin/auto_fetch
+            git branch -d auto_fetch
 
             # Create a branch only if it hasn't been created yet
             if [ "$branch_created" = false ]; then
                 git pull origin fusion_site_ssplab
                 git checkout -b auto_fetch
-                git push -u origin auto_fetch
+                # git push -u origin auto_fetch
                 branch_created=true
-                git checkout auto_fetch
             fi
 
+            git checkout auto_fetch
+
             # Clone the repository and move the subfolder
             clone_repo "$owner" "$repo" "$path" "$path_to_folder_to_synchronize_to"
 
             # Replace patterns in .qmd files
             replace_in_qmd_files "$path_to_folder_to_synchronize_to" "$replacements"
 
             # Update the last commit SHA in the JSON file
-            update_commit_sha "$last_commit_sha" "$new_commit_sha"
+            update_commit_sha "$last_commit_sha" "$new_commit_sha" "$json_file"
 
             # Commit the changes
             git add "$path_to_folder_to_synchronize_to" "$json_file"
             git commit -m "Update $path_to_folder_to_synchronize_to based on commit $new_commit_sha made to $path_to_folder_to_synchronize_from"
-            git push
+            # # git push
         else
-            echo "No new commit found for $path_to_folder_to_synchronize_from"
+            echo "No new commit since ${last_commit_sha:0:6} found for $path_to_folder_to_synchronize_from"
         fi
     done
 }