|
| 1 | +# ------------------------------------------------------------------------- |
| 2 | +# Copyright (c) Microsoft Corporation. All rights reserved. |
| 3 | +# Licensed under the MIT License. See LICENSE.txt in the project root for |
| 4 | +# license information. |
| 5 | +# ------------------------------------------------------------------------- |
| 6 | +# These examples are ingested by the documentation system, and are |
| 7 | +# displayed in the SDK reference documentation. When editing these |
| 8 | +# example snippets, take into consideration how this might affect |
| 9 | +# the readability and usability of the reference documentation. |
| 10 | + |
| 11 | +import os |
| 12 | +from azure.cosmos import PartitionKey, ThroughputProperties |
| 13 | +from azure.cosmos.aio import CosmosClient |
| 14 | +import asyncio |
| 15 | +import time |
| 16 | + |
| 17 | +# Specify information to connect to the client. |
| 18 | +CLEAR_DATABASE = True |
| 19 | +CONN_STR = os.environ['CONN_STR'] |
| 20 | +# Specify information for Database and container. |
| 21 | +DB_ID = "Cosmos_Concurrency_DB" |
| 22 | +CONT_ID = "Cosmos_Concurrency_Cont" |
| 23 | +# specify partition key for the container |
| 24 | +pk = PartitionKey(path="/id") |
| 25 | + |
| 26 | +# Batch the creation of items for better optimization on performance. |
| 27 | +# Note: Error handling should be in the method being batched. As you will get |
| 28 | +# an error for each failed Cosmos DB Operation. |
| 29 | +# Note: While the Word `Batch` here is used to describe the subsets of data being created, it is not referring |
| 30 | +# to batch operations such as `Transactional Batching` which is a feature of Cosmos DB. |
| 31 | +async def create_all_the_items(prefix, c, i): |
| 32 | + await asyncio.wait( |
| 33 | + [asyncio.create_task(c.create_item({"id": prefix + str(j)})) for j in range(100)] |
| 34 | + ) |
| 35 | + print(f"Batch {i} done!") |
| 36 | + |
| 37 | +# The following demonstrates the performance difference between using sequential item creation, |
| 38 | +# sequential item creation in batches, and concurrent item creation in batches. This is to show best practice |
| 39 | +# in using Cosmos DB for performance. |
| 40 | +# It’s important to note that batching a bunch of operations can affect throughput/RUs. |
| 41 | +# To avoid using resources, it’s recommended to test things on the emulator of Cosmos DB first. |
| 42 | +# The performance improvement shown on the emulator is relative to what you will see on a live account |
| 43 | +async def main(): |
| 44 | + try: |
| 45 | + async with CosmosClient.from_connection_string(CONN_STR) as client: |
| 46 | + # For emulator: default Throughput needs to be increased |
| 47 | + # throughput_properties = ThroughputProperties(auto_scale_max_throughput=5000) |
| 48 | + # db = await client.create_database_if_not_exists(id=DB_ID, offer_throughput=throughput_properties) |
| 49 | + db = await client.create_database_if_not_exists(id=DB_ID) |
| 50 | + container = await db.create_container_if_not_exists(CONT_ID, partition_key=pk) |
| 51 | + |
| 52 | + # A: Sequential without batching |
| 53 | + timer = time.time() |
| 54 | + print("Starting Sequential Item Creation.") |
| 55 | + for i in range(20): |
| 56 | + for j in range(100): |
| 57 | + await container.create_item({"id": f"{i}-sequential-{j}"}) |
| 58 | + print(f"{(i + 1) * 100} items created!") |
| 59 | + sequential_item_time = time.time() - timer |
| 60 | + print("Time taken: " + str(sequential_item_time)) |
| 61 | + |
| 62 | + |
| 63 | + # B: Sequential batches |
| 64 | + # Batching operations can improve performance by dealing with multiple operations at a time. |
| 65 | + timer = time.time() |
| 66 | + print("Starting Sequential Batched Item Creation.") |
| 67 | + for i in range(20): |
| 68 | + await create_all_the_items(f"{i}-sequential-Batch-", container, i) |
| 69 | + sequential_batch_time = time.time() - timer |
| 70 | + print("Time taken: " + str(sequential_batch_time)) |
| 71 | + |
| 72 | + # C: Concurrent batches |
| 73 | + # By using asyncio with batching, we can create multiple batches of items concurrently, which means that |
| 74 | + # while one connection is waiting for IO (like waiting for data to arrive), |
| 75 | + # Python can switch context to another connection and make progress there. |
| 76 | + # This can lead to better utilization of system resources and can give the appearance of parallelism, |
| 77 | + # as multiple connections are making progress seemingly at the same time |
| 78 | + timer = time.time() |
| 79 | + print("Starting Concurrent Batched Item Creation.") |
| 80 | + await asyncio.wait( |
| 81 | + [asyncio.create_task(create_all_the_items(f"{i}-concurrent-Batch", container, i)) for i in range(20)] |
| 82 | + ) |
| 83 | + concurrent_batch_time = time.time() - timer |
| 84 | + print("Time taken: " + str(concurrent_batch_time)) |
| 85 | + |
| 86 | + # Calculate performance improvement on time metrics. |
| 87 | + sequential_per = round((sequential_item_time - sequential_batch_time / sequential_item_time) * 100, 2) |
| 88 | + print(f"Sequential Batching is {sequential_per}% faster than Sequential Item Creation") |
| 89 | + concurrent_per = round((sequential_item_time - concurrent_batch_time / sequential_item_time) * 100, 2) |
| 90 | + print(f"Concurrent Batching is {concurrent_per}% faster than Sequential Item Creation") |
| 91 | + |
| 92 | + item_list = [i async for i in container.read_all_items()] |
| 93 | + print(f"End of the test. Read {len(item_list)} items.") |
| 94 | + |
| 95 | + finally: |
| 96 | + if CLEAR_DATABASE: |
| 97 | + await clear_database() |
| 98 | + |
| 99 | + |
| 100 | +async def clear_database(): |
| 101 | + async with CosmosClient.from_connection_string(CONN_STR) as client: |
| 102 | + await asyncio.create_task(client.delete_database(DB_ID)) |
| 103 | + print(f"Deleted {DB_ID} database.") |
| 104 | + |
| 105 | + |
| 106 | +if __name__ == "__main__": |
| 107 | + asyncio.run(main()) |
| 108 | + |
0 commit comments