Skip to content

[Spec Resync] 06-16-2025 #2392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,22 @@ post:
- func: "upload mo artifacts"
- func: "upload test results"
- func: "cleanup"

tasks:
- name: resync_specs
commands:
- command: subprocess.exec
params:
binary: bash
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
args:
- .evergreen/scripts/resync-all-specs.sh
working_dir: src

buildvariants:
- name: resync_specs
display_name: "Resync Specs"
run_on: rhel80-small
patchable: true
tasks:
- name: resync_specs
9 changes: 6 additions & 3 deletions .evergreen/resync-specs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,12 @@ then
fi

# Ensure the JSON files are up to date.
cd $SPECS/source
make
cd -
if ! [ -n "${CI:-}" ]
then
cd $SPECS/source
make
cd -
fi
# cpjson unified-test-format/tests/invalid unified-test-format/invalid
# * param1: Path to spec tests dir in specifications repo
# * param2: Path to where the corresponding tests live in Python.
Expand Down
57 changes: 57 additions & 0 deletions .evergreen/scripts/create-pr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/usr/bin/env bash

tools="../drivers-evergreen-tools"
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $tools
body="$(cat "$1")"

pushd $tools/.evergreen/github_app

owner="mongodb"
repo="mongo-python-driver"

# Bootstrap the app.
echo "bootstrapping"
source utils.sh
bootstrap drivers/comment-bot

# Run the app.
source ./secrets-export.sh

# Get a github access token for the git checkout.
echo "Getting github token..."

token=$(bash ./get-access-token.sh $repo $owner)
if [ -z "${token}" ]; then
echo "Failed to get github access token!"
popd
exit 1
fi
echo "Getting github token... done."
popd

# Make the git checkout and create a new branch.
echo "Creating the git checkout..."
branch="spec-resync-"$(date '+%m-%d-%Y')

#git config user.email "167856002+mongodb-dbx-release-bot[bot]@users.noreply.github.com"
#git config user.name "mongodb-dbx-release-bot[bot]"
git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git
git checkout -b $branch "origin/master"
git add ./test
git commit -am "resyncing specs test?"
echo "Creating the git checkout... done."

git push origin $branch
resp=$(curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $token" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"${body}\",\"head\":\"${branch}\",\"base\":\"master\"}" \
--url https://api.github.com/repos/$owner/$repo/pulls)
echo "this is the curl request"
echo "$resp"
echo $resp | jq '.html_url'
echo "Creating the PR... done."

rm -rf $dirname
104 changes: 104 additions & 0 deletions .evergreen/scripts/resync-all-specs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#!/usr/bin/env bash
# Run spec syncing script and create PR

SPEC_DEST="$(realpath -s "./test")"
SRC_URL="https://github.com/mongodb/specifications.git"
# needs to be set for resunc-specs.sh
SPEC_SRC="$(realpath -s "../specifications")"
SCRIPT="$(realpath -s "./.evergreen/resync-specs.sh")"
BRANCH_NAME="spec-resync-"$(date '+%m-%d-%Y')

# List of directories to skip
SKIP_DIRECTORIES=("asynchronous" "__pycache__")
# we have a list of specs that we manually override *if the git diff is that specific line, then don't change it
# *ask in python channel
SKIP_FILES=()
# ask steve for how to create PR from evergreen account(?)
# for now, treat it like a command line thing and git add *, git commit, git push

# Clone the repo if the directory does not exist
if [[ ! -d $SPEC_SRC ]]; then
git clone $SRC_URL $SPEC_SRC
if [[ $? -ne 0 ]]; then
echo "Error: Failed to clone repository."
exit 1
fi
fi

# Set environment variable to the cloned repo for resync-specs.sh
export MDB_SPECS="$SPEC_SRC"


# Check that resync script exists and is executable
if [[ ! -x $SCRIPT ]]; then
echo "Error: $SCRIPT not found or is not executable."
exit 1
fi

# List to store names of specs that were changed or errored during change
changed_specs=()
errored_specs=()

# Create branch and switch to it
#git checkout -b $BRANCH_NAME 2>/dev/null || git checkout $BRANCH_NAME

for item in "$SPEC_DEST"/*; do
item_name=$(basename "$item")
if [[ " ${SKIP_DIRECTORIES[*]} " =~ ${item_name} ]]; then
continue
fi

# Check that item is not a python file
if [[ $item != *.py ]]; then
echo " doing $item_name"
# output=$(./$SCRIPT "$item_name" 2>&1)
$SCRIPT "$item_name"
# Check if the script ran successfully
if [[ $? -ne 0 ]]; then
echo "an error occurred"
errored_specs+=("$item_name")
else
# if script had output, then changes were made
if [[ -n "$output" ]]; then
echo "success"
changed_specs+=("$item_name")
fi
fi
fi
done

pr_body="Spec sync results:\n\n"
# Output the list of changed specs
if [[ ${#changed_specs[@]} -gt 0 ]]; then
pr_body+="The following specs were changed:\n"
for spec in "${changed_specs[@]}"; do
pr_body+=" - $spec\n"
done
else
pr_body+="No changes detected in any specs.\n"
fi

# Output the list of errored specs
if [[ ${#errored_specs[@]} -gt 0 ]]; then
pr_body+="\nThe following spec syncs encountered errors:\n"
for spec in "${errored_specs[@]}"; do
pr_body+=" - $spec\n"
done
else
pr_body+="\nNo errors were encountered in any specs syncs.\n"
fi

# Output the PR body (optional step for verification)
echo -e "$pr_body"
echo "$pr_body" >> spec_sync.txt

git diff

# call scrypt to create PR for us
.evergreen/scripts/create-pr.sh spec_sync.txt

rm spec_sync.txt
#git add $SPEC_DEST
#git commit -m $BRANCH_NAME
#git push -u origin $BRANCH_NAME
#gh pr create --title "[Spec Resync] $(date '+%m-%d-%Y')" --body "Resyncing specs for review" --base main --head $BRANCH_NAME --draft
1 change: 1 addition & 0 deletions test/bson_corpus/datetime.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
{
"description" : "Y10K",
"canonical_bson" : "1000000009610000DC1FD277E6000000",
"relaxed_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}",
"canonical_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}"
},
{
Expand Down
24 changes: 24 additions & 0 deletions test/bson_corpus/decimal128-1.json
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,30 @@
"canonical_bson": "18000000136400000000000a5bc138938d44c64d31cc3700",
"degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\"}}",
"canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"1.000000000000000000000000000000000E+999\"}}"
},
{
"description": "Clamped zeros with a large positive exponent",
"canonical_bson": "180000001364000000000000000000000000000000FE5F00",
"degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E+2147483647\"}}",
"canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E+6111\"}}"
},
{
"description": "Clamped zeros with a large negative exponent",
"canonical_bson": "180000001364000000000000000000000000000000000000",
"degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E-2147483647\"}}",
"canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"0E-6176\"}}"
},
{
"description": "Clamped negative zeros with a large positive exponent",
"canonical_bson": "180000001364000000000000000000000000000000FEDF00",
"degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E+2147483647\"}}",
"canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E+6111\"}}"
},
{
"description": "Clamped negative zeros with a large negative exponent",
"canonical_bson": "180000001364000000000000000000000000000000008000",
"degenerate_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E-2147483647\"}}",
"canonical_extjson": "{\"d\" : {\"$numberDecimal\" : \"-0E-6176\"}}"
}
]
}
Loading
Loading