resyncing specs test?

Author: Cloud User

.evergreen/config.yml

@@ -42,3 +42,22 @@ post:
   - func: "upload mo artifacts"
   - func: "upload test results"
   - func: "cleanup"
+
+tasks:
+  - name: resync_specs
+    commands:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
+          args:
+            - .evergreen/scripts/resync-all-specs.sh
+          working_dir: src
+
+buildvariants:
+  - name: resync_specs
+    display_name: "Resync Specs"
+    run_on: rhel80-small
+    patchable: true
+    tasks:
+      - name: resync_specs

.evergreen/resync-specs.sh

@@ -45,9 +45,12 @@ then
 fi
 
 # Ensure the JSON files are up to date.
-cd $SPECS/source
-make
-cd -
+if ! [ -n "${CI:-}" ]
+then
+  cd $SPECS/source
+  make
+  cd -
+fi
 # cpjson unified-test-format/tests/invalid unified-test-format/invalid
 # * param1: Path to spec tests dir in specifications repo
 # * param2: Path to where the corresponding tests live in Python.

.evergreen/scripts/create-pr.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+tools="../drivers-evergreen-tools"
+git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $tools
+body="$(cat "$1")"
+
+pushd $tools/.evergreen/github_app
+
+owner="mongodb"
+repo="mongo-python-driver"
+
+# Bootstrap the app.
+echo "bootstrapping"
+source utils.sh
+bootstrap drivers/comment-bot
+
+# Run the app.
+source ./secrets-export.sh
+
+# Get a github access token for the git checkout.
+echo "Getting github token..."
+
+token=$(bash ./get-access-token.sh $repo $owner)
+if [ -z "${token}" ]; then
+    echo "Failed to get github access token!"
+    popd
+    exit 1
+fi
+echo "Getting github token... done."
+popd
+
+# Make the git checkout and create a new branch.
+echo "Creating the git checkout..."
+branch="spec-resync-"$(date '+%m-%d-%Y')
+
+#git config user.email "167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com"
+#git config user.name "mongodb-dbx-release-bot[bot]"
+git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git
+git checkout -b $branch "origin/master"
+git add ./test
+git commit -am "resyncing specs test?"
+echo "Creating the git checkout... done."
+
+git push origin $branch
+resp=$(curl -L \
+    -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $token" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"${body}\",\"head\":\"${branch}\",\"base\":\"master\"}" \
+    --url https://api.github.com/repos/$owner/$repo/pulls)
+echo "this is the curl request"
+echo "$resp"
+echo $resp | jq '.html_url'
+echo "Creating the PR... done."
+
+rm -rf $dirname

.evergreen/scripts/resync-all-specs.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Run spec syncing script and create PR
+
+SPEC_DEST="$(realpath -s "./test")"
+SRC_URL="https://github.com/mongodb/specifications.git"
+# needs to be set for resunc-specs.sh
+SPEC_SRC="$(realpath -s "../specifications")"
+SCRIPT="$(realpath -s "./.evergreen/resync-specs.sh")"
+BRANCH_NAME="spec-resync-"$(date '+%m-%d-%Y')
+
+# List of directories to skip
+SKIP_DIRECTORIES=("asynchronous" "__pycache__")
+# we have a list of specs that we manually override *if the git diff is that specific line, then don't change it
+# *ask in python channel
+SKIP_FILES=()
+# ask steve for how to create PR from evergreen account(?)
+# for now, treat it like a command line thing and git add *, git commit, git push
+
+# Clone the repo if the directory does not exist
+if [[ ! -d $SPEC_SRC ]]; then
+  git clone $SRC_URL $SPEC_SRC
+  if [[ $? -ne 0 ]]; then
+    echo "Error: Failed to clone repository."
+    exit 1
+  fi
+fi
+
+# Set environment variable to the cloned repo for resync-specs.sh
+export MDB_SPECS="$SPEC_SRC"
+
+
+# Check that resync script exists and is executable
+if [[ ! -x $SCRIPT ]]; then
+  echo "Error: $SCRIPT not found or is not executable."
+  exit 1
+fi
+
+# List to store names of specs that were changed or errored during change
+changed_specs=()
+errored_specs=()
+
+# Create branch and switch to it
+#git checkout -b $BRANCH_NAME 2>/dev/null || git checkout $BRANCH_NAME
+
+for item in "$SPEC_DEST"/*; do
+  item_name=$(basename "$item")
+  if [[ " ${SKIP_DIRECTORIES[*]} " =~ ${item_name} ]]; then
+    continue
+  fi
+
+  # Check that item is not a python file
+  if [[ $item != *.py ]]; then
+    echo " doing $item_name"
+#    output=$(./$SCRIPT "$item_name" 2>&1)
+    $SCRIPT "$item_name"
+    # Check if the script ran successfully
+    if [[ $? -ne 0 ]]; then
+      echo "an error occurred"
+      errored_specs+=("$item_name")
+    else
+      # if script had output, then changes were made
+      if [[ -n "$output" ]]; then
+        echo "success"
+        changed_specs+=("$item_name")
+      fi
+    fi
+  fi
+done
+
+pr_body="Spec sync results:\n\n"
+# Output the list of changed specs
+if [[ ${#changed_specs[@]} -gt 0 ]]; then
+  pr_body+="The following specs were changed:\n"
+  for spec in "${changed_specs[@]}"; do
+    pr_body+=" - $spec\n"
+  done
+else
+  pr_body+="No changes detected in any specs.\n"
+fi
+
+# Output the list of errored specs
+if [[ ${#errored_specs[@]} -gt 0 ]]; then
+  pr_body+="\nThe following spec syncs encountered errors:\n"
+  for spec in "${errored_specs[@]}"; do
+    pr_body+=" - $spec\n"
+  done
+else
+  pr_body+="\nNo errors were encountered in any specs syncs.\n"
+fi
+
+# Output the PR body (optional step for verification)
+echo -e "$pr_body"
+echo "$pr_body" >> spec_sync.txt
+
+git diff
+
+# call scrypt to create PR for us
+.evergreen/scripts/create-pr.sh spec_sync.txt
+
+rm spec_sync.txt
+#git add $SPEC_DEST
+#git commit -m $BRANCH_NAME
+#git push -u origin $BRANCH_NAME
+#gh pr create --title "[Spec Resync] $(date '+%m-%d-%Y')" --body "Resyncing specs for review" --base main --head $BRANCH_NAME --draft

test/bson_corpus/datetime.json

@@ -24,6 +24,7 @@
         {
             "description" : "Y10K",
             "canonical_bson" : "1000000009610000DC1FD277E6000000",
+            "relaxed_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}",
             "canonical_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}"
         },
         {

test/bson_corpus/decimal128-1.json

@@ -312,6 +312,30 @@
            "canonical_bson": "18000000136400000000000a5bc138938d44c64d31cc3700",
            "degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\"}}",
            "canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"1.000000000000000000000000000000000E+999\"}}"
+        },
+        {
+            "description": "Clamped zeros with a large positive exponent",
+            "canonical_bson": "180000001364000000000000000000000000000000FE5F00",
+            "degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E+2147483647\"}}",
+            "canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E+6111\"}}"
+        },
+        {
+            "description": "Clamped zeros with a large negative exponent",
+            "canonical_bson": "180000001364000000000000000000000000000000000000",
+            "degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E-2147483647\"}}",
+            "canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E-6176\"}}"
+        },
+        {
+            "description": "Clamped negative zeros with a large positive exponent",
+            "canonical_bson": "180000001364000000000000000000000000000000FEDF00",
+            "degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E+2147483647\"}}",
+            "canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E+6111\"}}"
+        },
+        {
+            "description": "Clamped negative zeros with a large negative exponent",
+            "canonical_bson": "180000001364000000000000000000000000000000008000",
+            "degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E-2147483647\"}}",
+            "canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E-6176\"}}"
         }
     ]
 }