Skip to content

Commit 288ac2d

Browse files
BryanFaubleBryanFauble
authored
Release v4.9.0: Merge pull request #1214 from Sage-Bionetworks/v4.9.0-rc-dev
Release v4.9.0
2 parents 2d50b5c + 7cdada6 commit 288ac2d

File tree

123 files changed

+23140
-11611
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

123 files changed

+23140
-11611
lines changed

‎.env.example

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
## Here are the environment variables you can set for the Synapse Python client to configure OpenTelemetry tracing and metrics:
2+
# You can copy this file to `.env` and fill in the values as needed.
3+
# OTEL_SERVICE_NAME=my-service-using-synapse-python-client
4+
# OTEL_EXPORTER_OTLP_ENDPOINT=http://fill-me-in
5+
# OTEL_SERVICE_INSTANCE_ID=local_development_testing
6+
# OTEL_EXPORTER_OTLP_HEADERS=# Authorization

‎.github/scripts/delete_evaluations.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Set
3+
4+
from synapseclient import Evaluation, Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_evaluation(eval_obj: Evaluation) -> str:
14+
"""Delete an evaluation asynchronously and return the status"""
15+
try:
16+
# Need to use run_in_executor since the delete function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete(eval_obj))
19+
return f"Deleted evaluation {eval_obj.id}"
20+
except Exception as e:
21+
return f"Failed to delete evaluation {eval_obj.id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Create a semaphore to limit concurrent operations
26+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
27+
28+
# Set to track active tasks
29+
pending_tasks: Set[asyncio.Task] = set()
30+
31+
# Track if we've processed any evaluations
32+
processed_any = False
33+
34+
async def delete_with_semaphore(eval_obj: Evaluation):
35+
"""Helper function that uses the semaphore to limit concurrency"""
36+
async with semaphore:
37+
result = await delete_evaluation(eval_obj)
38+
print(result)
39+
return result
40+
41+
# Process evaluations as they come in from the paginated iterator
42+
for result in syn._GET_paginated(
43+
"/evaluation?accessType=DELETE", limit=200, offset=0
44+
):
45+
processed_any = True
46+
eval_obj = Evaluation(**result)
47+
48+
# Create a new task for this evaluation
49+
task = asyncio.create_task(delete_with_semaphore(eval_obj))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No evaluations found to delete")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/scripts/delete_projects.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import asyncio
2+
from typing import Dict, Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_project(project_id: str) -> str:
14+
"""Delete a project asynchronously and return the result status"""
15+
try:
16+
# Need to use run_in_executor since the delete function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete(project_id))
19+
return f"Deleted {project_id}"
20+
except Exception as e:
21+
return f"Failed to delete {project_id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Create a semaphore to limit concurrent operations
26+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
27+
28+
# Set to track active tasks
29+
pending_tasks: Set[asyncio.Task] = set()
30+
31+
# Track if we've processed any projects
32+
processed_any = False
33+
34+
async def delete_with_semaphore(project: Dict):
35+
"""Helper function that uses the semaphore to limit concurrency"""
36+
async with semaphore:
37+
result = await delete_project(project["id"])
38+
print(result)
39+
return result
40+
41+
# Process projects as they come in from the iterator
42+
for project in syn.getChildren(parent=None, includeTypes=["project"]):
43+
processed_any = True
44+
45+
# Create a new task for this project
46+
task = asyncio.create_task(delete_with_semaphore(project))
47+
pending_tasks.add(task)
48+
task.add_done_callback(pending_tasks.discard)
49+
50+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
51+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
52+
# Wait for at least one task to complete before continuing
53+
done, _ = await asyncio.wait(
54+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
55+
)
56+
57+
# Wait for all remaining tasks to complete
58+
if pending_tasks:
59+
await asyncio.gather(*pending_tasks)
60+
61+
if not processed_any:
62+
print("No projects found to delete")
63+
64+
65+
# Run the async main function
66+
if __name__ == "__main__":
67+
asyncio.run(main())

‎.github/scripts/delete_teams.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Dict, Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent team deletions
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_team(team_id: str) -> str:
14+
"""Delete a team asynchronously and return the team_id"""
15+
try:
16+
# Need to use run_in_executor since the delete_team function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete_team(team_id))
19+
return f"Deleted team {team_id}"
20+
except Exception as e:
21+
return f"Failed to delete team {team_id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Get all teams for the current user
26+
teams = syn._find_teams_for_principal(principal_id=syn.credentials.owner_id)
27+
28+
# Create a semaphore to limit concurrent operations
29+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
30+
31+
# Set to track active tasks
32+
pending_tasks: Set[asyncio.Task] = set()
33+
34+
# Track if we've processed any teams
35+
processed_any = False
36+
37+
async def delete_with_semaphore(team: Dict):
38+
"""Helper function that uses the semaphore to limit concurrency"""
39+
async with semaphore:
40+
result = await delete_team(team["id"])
41+
print(result)
42+
return result
43+
44+
# Process teams as they come in from the iterator
45+
for team in teams:
46+
processed_any = True
47+
48+
# Create a new task for this team
49+
task = asyncio.create_task(delete_with_semaphore(team))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No teams found to delete")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/scripts/empty_trash.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def purge_entity(entity_id: str) -> str:
14+
"""Purge an entity from trash asynchronously and return the status"""
15+
try:
16+
# Need to use run_in_executor since the restPUT function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(
19+
None, lambda: syn.restPUT(uri=f"/trashcan/purge/{entity_id}")
20+
)
21+
return f"Purged entity {entity_id} from trash"
22+
except Exception as e:
23+
return f"Failed to purge entity {entity_id}: {str(e)}"
24+
25+
26+
async def main():
27+
# Create a semaphore to limit concurrent operations
28+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
29+
30+
# Set to track active tasks
31+
pending_tasks: Set[asyncio.Task] = set()
32+
33+
# Track if we've processed any entities
34+
processed_any = False
35+
36+
async def purge_with_semaphore(entity_id: str):
37+
"""Helper function that uses the semaphore to limit concurrency"""
38+
async with semaphore:
39+
result = await purge_entity(entity_id)
40+
print(result)
41+
return result
42+
43+
# Process entities as they come in from the paginated iterator
44+
for result in syn._GET_paginated("/trashcan/view", limit=200, offset=0):
45+
processed_any = True
46+
entity_id = result["entityId"]
47+
48+
# Create a new task for this entity
49+
task = asyncio.create_task(purge_with_semaphore(entity_id))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No entities found in trash to purge")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/workflows/build.yml

Lines changed: 22 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -83,12 +83,14 @@ jobs:
8383
path: |
8484
${{ steps.get-dependencies.outputs.site_packages_loc }}
8585
${{ steps.get-dependencies.outputs.site_bin_dir }}
86-
key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v22
86+
key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v24
8787

8888
- name: Install py-dependencies
8989
if: steps.cache-dependencies.outputs.cache-hit != 'true'
9090
shell: bash
9191
run: |
92+
python -m pip install --upgrade pip
93+
9294
pip install -e ".[boto3,pandas,pysftp,tests]"
9395
9496
# ensure that numpy c extensions are installed on windows
@@ -121,40 +123,6 @@ jobs:
121123
else
122124
echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
123125
fi
124-
- name: OpenTelemtry pre-check
125-
id: otel-check
126-
if: ${{ steps.secret-check.outputs.secrets_available == 'true' && steps.secret-check.outputs.synapse_pat_available == 'true' }}
127-
shell: bash
128-
run: |
129-
# Leave disabled during normal integration test runs - Enable when we want to
130-
# collect the data.
131-
# echo "run_opentelemetry=true" >> $GITHUB_OUTPUT;
132-
echo "run_opentelemetry=false" >> $GITHUB_OUTPUT;
133-
134-
# AWS CLI is pre-installed on github hosted runners - Commented out for GH runs
135-
# curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
136-
# unzip awscliv2.zip
137-
# sudo ./aws/install
138-
# curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
139-
# sudo dpkg -i session-manager-plugin.deb
140-
# - name: Create AWS Config
141-
# if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
142-
# shell: bash
143-
# run: |
144-
# touch test.awsConfig
145-
# printf "[default]\nregion = us-east-1\ncredential_process = \"tests/integration/synapse_creds.sh\" \"https://sc.sageit.org\" \"${{ secrets.synapse_personal_access_token }}\"\n" >> test.awsConfig
146-
# chmod +x tests/integration/synapse_creds.sh
147-
# If you are exporting data using `otlp` you can start a port forwading session
148-
# - name: SSM Port Forward Start
149-
# if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
150-
# shell: bash
151-
# env:
152-
# AWS_CONFIG_FILE: "test.awsConfig"
153-
# run: |
154-
# # Start a port-forwarding session in a non-interactive way. AWS will clean-up
155-
# # stale sessions after 20 minutes of inactivity
156-
# aws ssm start-session --target i-0ffcdecd1edf375ee --document-name AWS-StartPortForwardingSession --parameters "portNumber"=["4318"],"localPortNumber"=["4318"] & disown
157-
# sleep 15
158126
159127
# run integration tests iff the decryption keys for the test configuration are available.
160128
# they will not be available in pull requests from forks.
@@ -192,23 +160,31 @@ jobs:
192160
export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
193161
export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
194162
export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
195-
if [ ${{ steps.otel-check.outputs.run_opentelemetry }} == "true" ]; then
196-
# Set to 'file' to enable OpenTelemetry export to file
197-
export SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER="file"
163+
164+
# Set env vars for OTEL
165+
export OTEL_EXPORTER_OTLP_ENDPOINT="${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}"
166+
export OTEL_SERVICE_INSTANCE_ID="${{ vars.OTEL_SERVICE_INSTANCE_ID }}"
167+
export SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED="${{ vars.SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED }}"
168+
export OTEL_EXPORTER_OTLP_HEADERS="${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}"
169+
170+
# Setup ignore patterns based on Python version
171+
IGNORE_FLAGS="--ignore=tests/integration/synapseclient/test_command_line_client.py"
172+
173+
if [ "${{ matrix.python }}" == "3.9" ]; then
174+
# For min Python version, ignore async tests
175+
IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/async/"
176+
echo "Running integration tests for Min Python version (3.9) - ignoring async tests"
177+
elif [ "${{ matrix.python }}" == "3.13" ]; then
178+
# For max Python version, ignore synchronous tests
179+
IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/synchronous/"
180+
echo "Running integration tests for Max Python version (3.13) - ignoring synchronous tests"
198181
fi
199182
200183
# use loadscope to avoid issues running tests concurrently that share scoped fixtures
201-
pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
184+
pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
202185
203186
# Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
204187
pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration/synapseclient/test_command_line_client.py
205-
- name: Upload otel spans
206-
uses: actions/upload-artifact@v4
207-
if: always()
208-
with:
209-
name: otel_spans_integration_testing_${{ matrix.os }}
210-
path: tests/integration/otel_spans_integration_testing_*.ndjson
211-
if-no-files-found: ignore
212188
- name: Upload coverage report
213189
id: upload_coverage_report
214190
uses: actions/upload-artifact@v4

0 commit comments

Comments
 (0)