Skip to content

Commit 925aeb5

Browse files
sleepySticksleepyStick
committed
fun fact, using a python script is actually *way* easier
1 parent a97d5f4 commit 925aeb5

File tree

5 files changed

+1367
-83
lines changed

5 files changed

+1367
-83
lines changed

‎.evergreen/config.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ buildvariants:
5858
- name: resync_specs
5959
display_name: "Resync Specs"
6060
run_on: rhel80-small
61-
patchable: true
61+
cron: '0 9 * * MON'
62+
patchable: false
6263
tasks:
6364
- name: resync_specs

‎.evergreen/scripts/create-pr.sh

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,17 +44,18 @@ echo "Creating the git checkout... done."
4444

4545
echo "THIS IS THE BODY"
4646
echo "$body"
47-
#git push origin $branch
48-
#resp=$(curl -L \
49-
# -X POST \
50-
# -H "Accept: application/vnd.github+json" \
51-
# -H "Authorization: Bearer $token" \
52-
# -H "X-GitHub-Api-Version: 2022-11-28" \
53-
# -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"${body}\",\"head\":\"${branch}\",\"base\":\"master\"}" \
54-
# --url https://api.github.com/repos/$owner/$repo/pulls)
55-
#echo $resp
56-
#echo $resp | jq '.html_url'
57-
#echo "Creating the PR... done."
47+
git push origin $branch
48+
echo "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}"
49+
resp=$(curl -L \
50+
-X POST \
51+
-H "Accept: application/vnd.github+json" \
52+
-H "Authorization: Bearer $token" \
53+
-H "X-GitHub-Api-Version: 2022-11-28" \
54+
-d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}" \
55+
--url https://api.github.com/repos/$owner/$repo/pulls)
56+
echo $resp
57+
echo $resp | jq '.html_url'
58+
echo "Creating the PR... done."
5859

5960
rm -rf $tools
6061

‎.evergreen/scripts/resync-all-specs.py

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
from __future__ import annotations
2+
3+
import argparse
4+
import os
5+
import pathlib
6+
import subprocess
7+
8+
9+
def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> list[str]:
10+
for spec in os.scandir(directory):
11+
if not spec.is_dir():
12+
continue
13+
14+
if spec.name in ["asynchronous"]:
15+
continue
16+
process = subprocess.run(
17+
["bash", "./.evergreen/resync-specs.sh", spec.name], # noqa: S603, S607
18+
capture_output=True,
19+
text=True,
20+
check=False,
21+
)
22+
23+
if process.returncode != 0:
24+
errored[spec.name] = process.stderr
25+
26+
process = subprocess.run(
27+
["git diff --name-only | awk -F'/' '{print $2}' | sort | uniq"], # noqa: S607
28+
shell=True, # noqa: S602
29+
capture_output=True,
30+
text=True,
31+
check=True,
32+
)
33+
# return successfully synced specs
34+
return process.stdout.strip().split()
35+
36+
37+
def check_new_spec_directories(directory: pathlib.Path) -> list[str]:
38+
spec_dir = pathlib.Path(os.environ["MDB_SPECS"]) / "source"
39+
spec_set = {
40+
entry.name.replace("-", "_")
41+
for entry in os.scandir(spec_dir)
42+
if entry.is_dir()
43+
and (pathlib.Path(entry.path) / "tests").is_dir()
44+
and len(list(os.scandir(pathlib.Path(entry.path) / "tests"))) > 1
45+
}
46+
test_set = {entry.name.replace("-", "_") for entry in os.scandir(directory) if entry.is_dir()}
47+
known_mappings = {
48+
"ocsp_support": "ocsp",
49+
"client_side_operations_timeout": "csot",
50+
"mongodb_handshake": "handshake",
51+
"load_balancers": "load_balancer",
52+
"atlas_data_lake_testing": "atlas",
53+
"connection_monitoring_and_pooling": "connection_monitoring",
54+
"command_logging_and_monitoring": "command_logging",
55+
"initial_dns_seedlist_discovery": "srv_seedlist",
56+
"server_discovery_and_monitoring": "sdam_monitoring",
57+
}
58+
59+
for k, v in known_mappings.items():
60+
if k in spec_set:
61+
spec_set.remove(k)
62+
spec_set.add(v)
63+
return list(spec_set - test_set)
64+
65+
66+
def write_summary(succeeded: list[str], errored: dict[str, str], new: list[str]) -> None:
67+
pr_body = ""
68+
if len(succeeded) > 0:
69+
pr_body += "The following specs were changed:\n- "
70+
pr_body += "\n -".join(new)
71+
pr_body += "\n"
72+
if len(errored) > 0:
73+
pr_body += "\n\nThe following spec syncs encountered errors:"
74+
for k, v in errored.items():
75+
pr_body += f"\n- {k}\n```{v}\n```"
76+
pr_body += "\n"
77+
if len(new) > 0:
78+
pr_body += "\n\nThe following directories are in the specification repository and not in our test directory:"
79+
pr_body += "\n -".join(new)
80+
pr_body += "\n"
81+
82+
if pr_body != "":
83+
with open("spec_sync.txt", "w") as f:
84+
# replacements made for to be json
85+
f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t"))
86+
87+
88+
def main():
89+
directory = pathlib.Path("./test")
90+
errored: dict[str, str] = {}
91+
succeeded = resync_specs(directory, errored)
92+
new = check_new_spec_directories(directory)
93+
write_summary(succeeded, errored, new)
94+
95+
96+
if __name__ == "__main__":
97+
parser = argparse.ArgumentParser(
98+
description="Python Script to resync all specs and generate summary for PR."
99+
)
100+
parser.add_argument("filename", help="Name of file for the summary to be written into.")
101+
args = parser.parse_args()
102+
main()

‎.evergreen/scripts/resync-all-specs.sh

Lines changed: 12 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,15 @@
11
#!/usr/bin/env bash
22
# Run spec syncing script and create PR
3+
4+
# SETUP
35
SPEC_DEST="$(realpath -s "./test")"
46
SRC_URL="https://github.com/mongodb/specifications.git"
57
# needs to be set for resunc-specs.sh
68
SPEC_SRC="$(realpath -s "../specifications")"
79
SCRIPT="$(realpath -s "./.evergreen/resync-specs.sh")"
810
BRANCH_NAME="spec-resync-"$(date '+%m-%d-%Y')
911

10-
# List of directories to skip
11-
SKIP_DIRECTORIES=("asynchronous" "__pycache__")
12-
# we have a list of specs that we manually override *if the git diff is that specific line, then don't change it
13-
# *ask in python channel
14-
SKIP_FILES=()
15-
# ask steve for how to create PR from evergreen account(?)
16-
# for now, treat it like a command line thing and git add *, git commit, git push
17-
18-
# Clone the repo if the directory does not exist
12+
# Clone the spec repo if the directory does not exist
1913
if [[ ! -d $SPEC_SRC ]]; then
2014
git clone $SRC_URL $SPEC_SRC
2115
if [[ $? -ne 0 ]]; then
@@ -24,76 +18,23 @@ if [[ ! -d $SPEC_SRC ]]; then
2418
fi
2519
fi
2620

27-
# Set environment variable to the cloned repo for resync-specs.sh
21+
# Set environment variable to the cloned spec repo for resync-specs.sh
2822
export MDB_SPECS="$SPEC_SRC"
2923

30-
# Check that resync script exists and is executable
24+
# Check that resync-specs.sh exists and is executable
3125
if [[ ! -x $SCRIPT ]]; then
3226
echo "Error: $SCRIPT not found or is not executable."
3327
exit 1
3428
fi
3529

36-
# List to store names of specs that were changed or errored during change
37-
changed_specs=()
38-
errored_specs=()
30+
PR_DESC="spec_sync.txt"
3931

40-
# Create branch and switch to it
41-
#git checkout -b $BRANCH_NAME 2>/dev/null || git checkout $BRANCH_NAME
42-
43-
for item in "$SPEC_DEST"/*; do
44-
item_name=$(basename "$item")
45-
if [[ " ${SKIP_DIRECTORIES[*]} " =~ ${item_name} ]]; then
46-
continue
47-
fi
32+
# run python script that actually does all the resyncing
33+
/opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py "$PR_DESC"
4834

49-
# Check that item is not a python file
50-
if [[ $item != *.py ]]; then
51-
echo " doing $item_name"
52-
output=$($SCRIPT "$item_name" 2>&1)
53-
# Check if the script ran successfully
54-
if [[ $? -ne 0 ]]; then
55-
echo "an error occurred"
56-
errored_specs+=($"$item_name\n\`\`\`$output\`\`\`\n\n")
57-
fi
58-
fi
59-
done
60-
61-
# Output the list of changed specs
62-
if git diff --quiet && [[ ${#errored_specs[@]} -eq 0 ]]; then
63-
# no changes made and no errors
64-
exit 0
65-
fi
6635

67-
pr_body=$'Spec sync results:\n\n'
68-
if ! git diff --quiet; then
69-
pr_body+=$'The following specs were changed:\n'
70-
pr_body+="$(git diff --name-only | awk -F'/' '{print $2}' | sort | uniq)"
36+
if [[ -f $PR_DESC ]]; then
37+
# changes were made -> call scrypt to create PR for us
38+
.evergreen/scripts/create-pr.sh "$PR_DESC"
39+
rm "$PR_DESC"
7140
fi
72-
73-
# Output the list of errored specs
74-
if [[ ${#errored_specs[@]} -gt 0 ]]; then
75-
pr_body+=$"\n\nThe following spec syncs encountered errors:\n"
76-
for spec in "${errored_specs[@]}"; do
77-
pr_body+=" - $spec\n"
78-
done
79-
else
80-
pr_body+=$"\nNo errors were encountered in any specs syncs.\n"
81-
fi
82-
83-
# Output the PR body (optional step for verification)
84-
echo "PR body"
85-
echo "$pr_body"
86-
echo "$pr_body" >> spec_sync.txt
87-
88-
echo "BEGINNING OF DIFF"
89-
git diff
90-
echo "END OF DIFF"
91-
92-
# call scrypt to create PR for us
93-
.evergreen/scripts/create-pr.sh spec_sync.txt
94-
95-
rm spec_sync.txt
96-
#git add $SPEC_DEST
97-
#git commit -m $BRANCH_NAME
98-
#git push -u origin $BRANCH_NAME
99-
#gh pr create --title "[Spec Resync] $(date '+%m-%d-%Y')" --body "Resyncing specs for review" --base main --head $BRANCH_NAME --draft

0 commit comments

Comments
 (0)