Merge pull request #18 from realpython/concurrency-overview

Concurrency overview

Author: dbader

concurrency-overview/README.md

@@ -0,0 +1,9 @@
+# Speed Up Your Python Program With Concurrency: Code Examples
+
+Corresponding code to the Real Python tutorial, "[Speed up your Python Program with Concurrency](https://realpython.com/python-concurrency-overview/)."
+
+To run the code here, use:
+
+    pip install -r requirements.txt
+
+This will ensure you have the required packages.

concurrency-overview/cpu_mp.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import multiprocessing
+import time
+
+
+def cpu_bound(number):
+    return sum(i * i for i in range(number))
+
+
+def find_sums(numbers):
+    with multiprocessing.Pool() as pool:
+        pool.map(cpu_bound, numbers)
+
+
+if __name__ == "__main__":
+    numbers = [5_000_000 + x for x in range(20)]
+
+    start_time = time.time()
+    find_sums(numbers)
+    duration = time.time() - start_time
+    print(f"Duration {duration} seconds")

concurrency-overview/cpu_non_concurrent.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import time
+
+
+def cpu_bound(number):
+    return sum(i * i for i in range(number))
+
+
+def find_sums(numbers):
+    for number in numbers:
+        cpu_bound(number)
+
+
+if __name__ == "__main__":
+    numbers = [5_000_000 + x for x in range(20)]
+
+    start_time = time.time()
+    find_sums(numbers)
+    duration = time.time() - start_time
+    print(f"Duration {duration} seconds")

concurrency-overview/cpu_threading.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import concurrent.futures
+import time
+
+
+def cpu_bound(number):
+    return sum(i * i for i in range(number))
+
+
+def find_sums(numbers):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        executor.map(cpu_bound, numbers)
+
+
+if __name__ == "__main__":
+    numbers = [5_000_000 + x for x in range(20)]
+
+    start_time = time.time()
+    find_sums(numbers)
+    duration = time.time() - start_time
+    print(f"Duration {duration} seconds")

concurrency-overview/io_asyncio.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+import asyncio
+import time
+import aiohttp
+
+
+async def download_site(session, url):
+    async with session.get(url) as response:
+        print("Read {0} from {1}".format(response.content_length, url))
+
+
+async def download_all_sites(sites):
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for url in sites:
+            task = asyncio.ensure_future(download_site(session, url))
+            tasks.append(task)
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+
+if __name__ == "__main__":
+    sites = [
+        "http://www.jython.org",
+        "http://olympus.realpython.org/dice",
+    ] * 80
+    start_time = time.time()
+    asyncio.get_event_loop().run_until_complete(download_all_sites(sites))
+    duration = time.time() - start_time
+    print(f"Downloaded {len(sites)} sites in {duration} seconds")

concurrency-overview/io_mp.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import requests
+import multiprocessing
+import time
+
+session = None
+
+
+def set_global_session():
+    global session
+    if not session:
+        session = requests.Session()
+
+
+def download_site(url):
+    with session.get(url) as response:
+        name = multiprocessing.current_process().name
+        print(f"{name}:Read {len(response.content)} from {url}")
+
+
+def download_all_sites(sites):
+    with multiprocessing.Pool(initializer=set_global_session) as pool:
+        pool.map(download_site, sites)
+
+
+if __name__ == "__main__":
+    sites = [
+        "http://www.jython.org",
+        "http://olympus.realpython.org/dice",
+    ] * 80
+    start_time = time.time()
+    download_all_sites(sites)
+    duration = time.time() - start_time
+    print(f"Downloaded {len(sites)} in {duration} seconds")

concurrency-overview/io_non_concurrent.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import requests
+import time
+
+
+def download_site(url, session):
+    with session.get(url) as response:
+        print(f"Read {len(response.content)} from {url}")
+
+
+def download_all_sites(sites):
+    with requests.Session() as session:
+        for url in sites:
+            download_site(url, session)
+
+
+if __name__ == "__main__":
+    sites = [
+        "http://www.jython.org",
+        "http://olympus.realpython.org/dice",
+    ] * 80
+    start_time = time.time()
+    download_all_sites(sites)
+    duration = time.time() - start_time
+    print(f"Downloaded {len(sites)} in {duration} seconds")

concurrency-overview/io_threading.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import concurrent.futures
+import requests
+import threading
+import time
+
+
+thread_local = threading.local()
+
+
+def get_session():
+    if not getattr(thread_local, "session", None):
+        thread_local.session = requests.Session()
+    return thread_local.session
+
+
+def download_site(url):
+    session = get_session()
+    with session.get(url) as response:
+        print(f"Read {len(response.content)} from {url}")
+
+
+def download_all_sites(sites):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        executor.map(download_site, sites)
+
+
+if __name__ == "__main__":
+    sites = [
+        "http://www.jython.org",
+        "http://olympus.realpython.org/dice",
+    ] * 80
+    start_time = time.time()
+    download_all_sites(sites)
+    duration = time.time() - start_time
+    print(f"Downloaded {len(sites)} in {duration} seconds")

concurrency-overview/race_condition.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+import concurrent.futures
+
+
+counter = 0
+
+
+def increment_counter(fake_value):
+    global counter
+    for _ in range(100):
+        counter += 1
+
+
+if __name__ == "__main__":
+    fake_data = [x for x in range(5000)]
+    counter = 0
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5000) as executor:
+        executor.map(increment_counter, fake_data)

concurrency-overview/requirements.txt

@@ -0,0 +1,39 @@
+aiohttp==3.4.4
+asks==2.0.0
+astroid==2.0.4
+async-generator==1.10
+async-timeout==3.0.1
+atomicwrites==1.2.0
+attrs==18.1.0
+certifi==2018.8.13
+chardet==3.0.4
+contextvars==2.3
+flake8==3.5.0
+h11==0.8.1
+idna==2.7
+immutables==0.6
+isort==4.3.4
+lazy-object-proxy==1.3.1
+mccabe==0.6.1
+more-itertools==4.3.0
+multidict==4.4.2
+multio==0.2.3
+outcome==0.1.0
+pathlib2==2.3.2
+pkg-resources==0.0.0
+pluggy==0.7.1
+py==1.6.0
+pycodestyle==2.3.1
+pyflakes==1.6.0
+pylint==2.1.1
+pytest==3.7.3
+requests==2.19.1
+six==1.11.0
+sniffio==1.0.0
+sortedcontainers==2.0.4
+tqdm==4.25.0
+trio==0.6.0
+typed-ast==1.1.0
+urllib3==1.23
+wrapt==1.10.11
+yarl==1.2.6