Jost migenda/optimisation intro

.gitignore

@@ -49,3 +49,5 @@ docs/
 # translation temp files
 po/*~
 
+# Python cache directory
+__pycache__/

README.md

@@ -34,8 +34,6 @@ options(repos = c(
 # Setup install from github
 install.packages("devtools")
 library(devtools)
-# Install Uni of Shef Varnish theme
-install_github("RSE-Sheffield/uos-varnish")
 # Install remaining official carpentries packages
 install.packages(c("sandpaper", "tinkr", "pegboard"))
 ```

config.yaml

@@ -65,6 +65,7 @@ episodes:
 - profiling-lines.md
 - profiling-conclusion.md
 - optimisation-introduction.md
+- optimisation-using-python.md
 - optimisation-data-structures-algorithms.md
 - long-break1.md
 - optimisation-minimise-python.md
@@ -75,7 +76,7 @@ episodes:
 # Information for Learners
 learners: 
 - setup.md
-- registration.md
+# - registration.md
 - acknowledgements.md
 - ppp.md
 - reference.md
@@ -91,8 +92,5 @@ profiles:
 # This space below is where custom yaml items (e.g. pinning
 # sandpaper and varnish versions) should live
 
-varnish: RSE-Sheffield/uos-varnish@main
-url: 'https://rse.shef.ac.uk/pando-python'
-
-analytics: |
-  <script defer src="https://cloud.umami.is/script.js" data-website-id="95b3ca57-3bd9-47b0-b1ca-fe819ef65c80"></script>
+# varnish: RSE-Sheffield/uos-varnish@main
+# url: 'https://rse.shef.ac.uk/pando-python'

episodes/files/snippets/builtin_str_split.py

@@ -0,0 +1,42 @@
+import random
+from timeit import timeit
+
+N = 10_000  # Number of elements in the list
+
+# Ensure every list is the same
+random.seed(12)
+f = [f" {i:0>6d} {random.random():8.4f} " for i in range(N)]
+
+def manualSplit():  # bad habits
+    data = {}
+    for line in f:
+        first_char = line.find("0")
+        end_time = line.find(" ", first_char, -1)
+
+        energy_found = line.find(".", end_time, -1)
+        begin_energy = line.rfind(" ", end_time, energy_found)
+        end_energy = line.find(" ", energy_found, -1)
+        if end_energy == -1:
+            end_energy = len(line)
+
+        time = line[first_char:end_time]
+        energy = line[begin_energy:end_energy]
+
+        data[time] = energy
+    return data
+
+def builtinSplit():
+    data = {}
+    for line in f:
+        time, energy = line.split()
+        data[time] = energy
+    return data
+
+def dictComprehension():
+    return {time: energy for time, energy in (line.split() for line in f)}
+
+
+repeats = 1000
+print(f"manualSplit: {timeit(manualSplit, globals=globals(), number=repeats):.3f}ms")
+print(f"builtinSplit: {timeit(builtinSplit, globals=globals(), number=repeats):.3f}ms")
+print(f"dictComprehension: {timeit(dictComprehension, globals=globals(), number=repeats):.3f}ms")

episodes/files/snippets/builtin_sum.py

@@ -0,0 +1,30 @@
+import random
+from timeit import timeit
+
+N = 100_000  # Number of elements in the list
+
+# Ensure every list is the same
+random.seed(12)
+my_data = [random.random() for i in range(N)]
+
+
+def manualSumC():  # bad habits
+    n = 0
+    for i in range(len(my_data)):
+        n += my_data[i]
+    return n
+
+def manualSumPy():  # slightly improved
+    n = 0
+    for evt_count in my_data:
+        n += evt_count
+    return n
+
+def builtinSum():  # fastest and most readable
+    return sum(my_data)
+
+
+repeats = 1000
+print(f"manualSumC: {timeit(manualSumC, globals=globals(), number=repeats):.3f}ms")
+print(f"manualSumPy: {timeit(manualSumPy, globals=globals(), number=repeats):.3f}ms")
+print(f"builtinSum: {timeit(builtinSum, globals=globals(), number=repeats):.3f}ms")

episodes/files/snippets/numpy_array_resize.py

@@ -0,0 +1,25 @@
+from timeit import timeit
+import numpy
+
+N = 100_000  # Number of elements in list/array
+
+def list_append():
+    ls = []
+    for i in range(N):
+        ls.append(i)
+
+def array_resize():
+    ar = numpy.zeros(1)
+    for i in range(1, N):
+        ar.resize(i+1)
+        ar[i] = i
+
+def array_preallocate():
+    ar = numpy.zeros(N)
+    for i in range(1, N):
+        ar[i] = i
+
+repeats = 1000
+print(f"list_append: {timeit(list_append, number=repeats):.2f}ms")
+print(f"array_resize: {timeit(array_resize, number=repeats):.2f}ms")
+print(f"array_preallocate: {timeit(array_preallocate, number=repeats):.2f}ms")

episodes/files/snippets/parallel-download.py

@@ -0,0 +1,44 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from timeit import timeit
+import requests  # install with `pip install requests`
+
+
+def download_file(url, filename):
+    response = requests.get(url)
+    with open(filename, 'wb') as f:
+        f.write(response.content)
+    return filename
+
+downloaded_files = []
+
+def sequentialDownload():
+    for mass in range(10, 20):
+        url = f"https://github.com/SNEWS2/snewpy-models-ccsn/raw/refs/heads/main/models/Warren_2020/stir_a1.23/stir_multimessenger_a1.23_m{mass}.0.h5"
+        f = download_file(url, f"seq_{mass}.h5")
+        downloaded_files.append(f)
+
+def parallelDownload():
+    pool = ThreadPoolExecutor(max_workers=6)
+    jobs = []
+    for mass in range(10, 20):
+        url = f"https://github.com/SNEWS2/snewpy-models-ccsn/raw/refs/heads/main/models/Warren_2020/stir_a1.23/stir_multimessenger_a1.23_m{mass}.0.h5"
+        local_filename = f"par_{mass}.h5"
+        jobs.append(pool.submit(download_file, url, local_filename))
+
+    for result in as_completed(jobs):        
+        if result.exception() is None:
+            # handle return values of the parallelised function
+            f = result.result()
+            downloaded_files.append(f)
+        else:
+            # handle errors
+            print(result.exception())
+
+    pool.shutdown(wait=False)
+
+
+print(f"sequentialDownload: {timeit(sequentialDownload, globals=globals(), number=1):.3f} s")
+print(downloaded_files)
+downloaded_files = []
+print(f"parallelDownload: {timeit(parallelDownload, globals=globals(), number=1):.3f} s")
+print(downloaded_files)

episodes/files/snippets/read-write.py

@@ -0,0 +1,50 @@
+import os, time
+
+# Generate 10MB
+data_len = 10000000
+data = os.urandom(data_len)
+file_ct = 1000
+file_len = int(data_len/file_ct)
+
+# Write one large file
+start = time.perf_counter()
+large_file = open("large.bin", "wb")
+large_file.write(data)
+large_file.close ()
+large_write_s = time.perf_counter() - start
+
+# Write multiple small files
+start = time.perf_counter()
+for i in range(file_ct):
+    small_file = open(f"small_{i}.bin", "wb")
+    small_file.write(data[file_len*i:file_len*(i+1)])
+    small_file.close()
+small_write_s = time.perf_counter() - start
+
+# Read back the large file
+start = time.perf_counter()
+large_file = open("large.bin", "rb")
+t = large_file.read(data_len)
+large_file.close ()
+large_read_s = time.perf_counter() - start
+
+# Read back the small files
+start = time.perf_counter()
+for i in range(file_ct):
+    small_file = open(f"small_{i}.bin", "rb")
+    t = small_file.read(file_len)
+    small_file.close()
+small_read_s = time.perf_counter() - start
+
+# Print Summary
+print(f"{1:5d}x{data_len/1000000}MB Write: {large_write_s:.5f} seconds")
+print(f"{file_ct:5d}x{file_len/1000}KB Write: {small_write_s:.5f} seconds")
+print(f"{1:5d}x{data_len/1000000}MB Read: {large_read_s:.5f} seconds")
+print(f"{file_ct:5d}x{file_len/1000}KB Read: {small_read_s:.5f} seconds")
+print(f"{file_ct:5d}x{file_len/1000}KB Write was {small_write_s/large_write_s:.1f} slower than 1x{data_len/1000000}MB Write")
+print(f"{file_ct:5d}x{file_len/1000}KB Read was {small_read_s/large_read_s:.1f} slower than 1x{data_len/1000000}MB Read")
+
+# Cleanup
+os.remove("large.bin")
+for i in range(file_ct):
+    os.remove(f"small_{i}.bin")

episodes/files/snippets/test_demonstration.py

@@ -0,0 +1,8 @@
+# A simple function to be tested, this could instead be an imported package
+def squared(x):
+    return x**2
+
+# A simple test case
+def test_example():
+    assert squared(5) == 24
+

episodes/optimisation-conclusion.md

@@ -25,6 +25,7 @@ Hopefully with the information from this course you will be in a better position
 
 This course's website can be used as a reference manual when profiling your own code.
 
+<!--
 ::::::::::::::::::::::::::::::::::::: callout
 
 ## Your Feedback is Required!
@@ -34,6 +35,7 @@ Please complete [this Google form](https://forms.gle/C82uWBEou3FMrQs99) to let u
 Your feedback enables us to improve the course for future attendees!
 
 :::::::::::::::::::::::::::::::::::::::::::::
+-->
 
 ::::::::::::::::::::::::::::::::::::: keypoints
 

episodes/optimisation-data-structures-algorithms.md

@@ -65,7 +65,8 @@ CPython for example uses [`newsize + (newsize >> 3) + 6`](https://github.com/pyt
 
 This has two implications:
 
-* If you are creating large static lists, they will use upto 12.5% excess memory.
+* If you are creating large static lists, they may use up to 12.5% excess memory.
+<!-- I think that only applies when resizing a list? IIUC, when creating a list of a particular size from scratch, CPython will not overallocate as much memory. See the `list_preallocate_exact` function in `listobject.c`. -->
 * If you are growing a list with `append()`, there will be large amounts of redundant allocations and copies as the list grows.
 
 ### List Comprehension
@@ -74,7 +75,7 @@ If creating a list via `append()` is undesirable, the natural alternative is to
 
 List comprehension can be twice as fast at building lists than using `append()`.
 This is primarily because list-comprehension allows Python to offload much of the computation into faster C code.
-General python loops in contrast can be used for much more, so they remain in Python bytecode during computation which has additional overheads.
+General Python loops in contrast can be used for much more, so they remain in Python bytecode during computation which has additional overheads.
 
 This can be demonstrated with the below benchmark:
 
@@ -112,7 +113,7 @@ Results will vary between Python versions, hardware and list lengths. But in thi
 
 ## Tuples
 
-In contrast, Python's tuples are immutable static arrays (similar to strings), their elements cannot be modified and they cannot be resized.
+In contrast to lists, Python's tuples are immutable static arrays (similar to strings): Their elements cannot be modified and they cannot be resized.
 
 Their potential use-cases are greatly reduced due to these two limitations, they are only suitable for groups of immutable properties.
 
@@ -152,6 +153,22 @@ Since Python 3.6, the items within a dictionary will iterate in the order that t
 
 <!-- simple explanation of how a hash-based data structure works -->
 Python's dictionaries are implemented as hashing data structures.
+Explaining how these work will get a bit technical, so let’s start with an analogy:
+
+A Python list is like having a single long bookshelf. When you buy a new book (append a new element to the list), you place it at the far end of the shelf, right after all the previous books.
+
+A hashing data structure is more like a bookcase, with several shelves, one for each genre: There’s a shelf for detective fiction, a shelf for romance novels, a shelf for sci-fi stories, and so on. When you buy a new romance novel, you place it on the appropriate shelf, next to the previous books in that genre.
+And as you get more books, at some point you’ll move to a larger bookcase with more shelves (and thus more fine-grained genre categories), to make sure you don’t have too many books on a single shelf.
+
+Now, let’s say a friend asks me whether I have the book “Dune”.
+If I had my books arranged on a single bookshelf (in a list), I would have to look through every book I own in order to find “Dune”.
+However, if I had a bookcase with several shelves (a hashing data structure), I know immediately that I need to check the sci-fi shelf, so I’d be able to find it much more quickly!
+
+
+::::::::::::::::::::::::::::::::::::: callout
+
+### Technical explanation
+
 Within a hashing data structure each inserted key is hashed to produce a (hopefully unique) integer key.
 The dictionary is pre-allocated to a default size, and the key is assigned the index within the dictionary equivalent to the hash modulo the length of the dictionary.
 If that index doesn't already contain another key, the key (and any associated values) can be inserted.
@@ -160,7 +177,7 @@ When the hashing data structure exceeds a given load factor (e.g. 2/3 of indices
 
 ![An visual explanation of linear probing, CPython uses an advanced form of this.](episodes/fig/hash_linear_probing.png){alt="A diagram demonstrating how the keys (hashes) 37, 64, 14, 94, 67 are inserted into a hash table with 11 indices. This is followed by the insertion of 59, 80 and 39 which require linear probing to be inserted due to collisions."}
 
-To retrieve or check for the existence of a key within a hashing data structure, the key is hashed again and a process equivalent to insertion is repeated. However, now the key at each index is checked for equality with the one provided. If any empty index is found before an equivalent key, then the key must not be present in the ata structure.
+To retrieve or check for the existence of a key within a hashing data structure, the key is hashed again and a process equivalent to insertion is repeated. However, now the key at each index is checked for equality with the one provided. If any empty index is found before an equivalent key, then the key must not be present in the data structure.
 
 
 ### Keys
@@ -189,6 +206,8 @@ dict[MyKey("one", 2, 3.0)] = 12
 ```
 The only limitation is that where two objects are equal they must have the same hash, hence all member variables which contribute to `__eq__()` should also contribute to `__hash__()` and vice versa (it's fine to have irrelevant or redundant internal members contribute to neither).
 
+:::::::::::::::::::::::::::::::::::::
+
 ## Sets
 
 Sets are dictionaries without the values (both are declared using `{}`), a collection of unique keys equivalent to the mathematical set. *Modern CPython now uses a set implementation distinct from that of it's dictionary, however they still behave much the same in terms of performance characteristics.*
@@ -325,7 +344,7 @@ def binary_search_list():
         if k != len(ls) and ls[k] == i:
             j += 1
 
-            
+
 repeats = 1000
 gen_time = timeit(generateInputs, number=repeats)
 print(f"search_set: {timeit(search_set, number=repeats)-gen_time:.2f}ms")
@@ -334,7 +353,7 @@ print(f"binary_search_list: {timeit(binary_search_list, number=repeats)-gen_time
 ```
 
 Searching the set is fastest performing 25,000 searches in 0.04ms.
-This is  followed by the binary search of the (sorted) list which is 145x slower, although the list has been filtered for duplicates. A list still containing duplicates would be longer, leading to a more expensive search.
+This is followed by the binary search of the (sorted) list which is 145x slower, although the list has been filtered for duplicates. A list still containing duplicates would be longer, leading to a more expensive search.
 The linear search of the list is more than 56,600x slower than the fastest, it really shouldn't be used!
 
 ```output