From edd1fbdb76a97cc18893d0279ca9c7dde48a34cc Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 21:35:22 -0400
Subject: [PATCH 1/6] add start of Gradescope autograder

---
 .gitignore                                    | 37 +------------------
 extras/autograder/source/requirements.txt     |  1 +
 extras/autograder/source/run_autograder       | 10 +++++
 extras/autograder/source/run_tests.py         |  9 +++++
 extras/autograder/source/setup.sh             |  7 ++++
 extras/autograder/source/tests/test_simple.py | 10 +++++
 extras/autograder/submission/example.ipynb    | 16 ++++++++
 extras/autograder/submission/example.py       |  0
 extras/lib/school.py                          |  2 +
 extras/terraform/.gitignore                   | 34 +++++++++++++++++
 meta/instructor_guide.md                      | 16 ++++++++
 pyproject.toml                                |  5 +++
 12 files changed, 111 insertions(+), 36 deletions(-)
 create mode 100644 extras/autograder/source/requirements.txt
 create mode 100755 extras/autograder/source/run_autograder
 create mode 100644 extras/autograder/source/run_tests.py
 create mode 100755 extras/autograder/source/setup.sh
 create mode 100644 extras/autograder/source/tests/test_simple.py
 create mode 100644 extras/autograder/submission/example.ipynb
 create mode 100644 extras/autograder/submission/example.py
 create mode 100644 extras/terraform/.gitignore

diff --git a/.gitignore b/.gitignore
index ec5a4d07..fc096c10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,39 +15,4 @@ Untitled.ipynb
 # override the global
 !.vscode/settings.json
 
-## Terraform ##
-
-# Local .terraform directories
-**/.terraform/*
-
-# .tfstate files
-*.tfstate
-*.tfstate.*
-
-# Crash log files
-crash.log
-
-# Exclude all .tfvars files, which are likely to contain sentitive data, such as
-# password, private keys, and other secrets. These should not be part of version
-# control as they are data points which are potentially sensitive and subject
-# to change depending on the environment.
-#
-*.tfvars
-
-# Ignore override files as they are usually used to override resources locally and so
-# are not checked in
-override.tf
-override.tf.json
-*_override.tf
-*_override.tf.json
-
-# Include override files you do wish to add to version control using negated pattern
-#
-# !example_override.tf
-
-# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
-# example: *tfplan*
-
-# Ignore CLI configuration files
-.terraformrc
-terraform.rc
+extras/autograder/results/
diff --git a/extras/autograder/source/requirements.txt b/extras/autograder/source/requirements.txt
new file mode 100644
index 00000000..2198626a
--- /dev/null
+++ b/extras/autograder/source/requirements.txt
@@ -0,0 +1 @@
+gradescope_utils
diff --git a/extras/autograder/source/run_autograder b/extras/autograder/source/run_autograder
new file mode 100755
index 00000000..0cb759c1
--- /dev/null
+++ b/extras/autograder/source/run_autograder
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# based on
+# https://github.com/gradescope/autograder_samples/blob/master/python/src/run_autograder
+
+set -ex
+
+cd /autograder/source
+
+python3 run_tests.py
diff --git a/extras/autograder/source/run_tests.py b/extras/autograder/source/run_tests.py
new file mode 100644
index 00000000..f0825eab
--- /dev/null
+++ b/extras/autograder/source/run_tests.py
@@ -0,0 +1,9 @@
+"""based on https://github.com/gradescope/autograder_samples/blob/master/python/src/run_tests.py"""
+
+import unittest
+from gradescope_utils.autograder_utils.json_test_runner import JSONTestRunner
+
+if __name__ == '__main__':
+    suite = unittest.defaultTestLoader.discover('tests')
+    with open('/autograder/results/results.json', 'w') as f:
+        JSONTestRunner(visibility='visible', stream=f).run(suite)
diff --git a/extras/autograder/source/setup.sh b/extras/autograder/source/setup.sh
new file mode 100755
index 00000000..eb16be20
--- /dev/null
+++ b/extras/autograder/source/setup.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+apt-get install -y python3 python3-pip
+
+pip3 install -r /autograder/source/requirements.txt
diff --git a/extras/autograder/source/tests/test_simple.py b/extras/autograder/source/tests/test_simple.py
new file mode 100644
index 00000000..a6b89382
--- /dev/null
+++ b/extras/autograder/source/tests/test_simple.py
@@ -0,0 +1,10 @@
+import unittest
+from gradescope_utils.autograder_utils.decorators import weight, number
+
+
+class TestSimpleArithmetic(unittest.TestCase):
+    @weight(1)
+    @number("1.1")
+    def test_eval_add(self):
+        """Evaluate 1 + 1"""
+        self.assertEqual(1+1, 2)
diff --git a/extras/autograder/submission/example.ipynb b/extras/autograder/submission/example.ipynb
new file mode 100644
index 00000000..b3310258
--- /dev/null
+++ b/extras/autograder/submission/example.ipynb
@@ -0,0 +1,16 @@
+{
+ "cells": [],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python-public-policy",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/extras/autograder/submission/example.py b/extras/autograder/submission/example.py
new file mode 100644
index 00000000..e69de29b
diff --git a/extras/lib/school.py b/extras/lib/school.py
index e87e3e08..ffe0da58 100644
--- a/extras/lib/school.py
+++ b/extras/lib/school.py
@@ -148,6 +148,7 @@ class SchoolText:
     ".zoom.us/rec",
     "- [google colab](https://colab.research.google.com/)",
     "anaconda",
+    "autograder",  # matches "grader"
     "built around it",  # referring to Colab
     "columbia's graduate school of architecture",  # bio
     "conda activate",
@@ -155,6 +156,7 @@ class SchoolText:
     "create the environment",
     "dictreader",
     "for row in reader",
+    "gradescope_utils",  # matches "gradescope"
     "hannahkates/nyu-python-public-policy",
     "https://community.canvaslms.com/t5/canvas-basics-guide/what-are-grading-schemes/ta-p/41",
     "jupyterhub_url",
diff --git a/extras/terraform/.gitignore b/extras/terraform/.gitignore
new file mode 100644
index 00000000..18221f8e
--- /dev/null
+++ b/extras/terraform/.gitignore
@@ -0,0 +1,34 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+#
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
diff --git a/meta/instructor_guide.md b/meta/instructor_guide.md
index 8001b546..8561283f 100644
--- a/meta/instructor_guide.md
+++ b/meta/instructor_guide.md
@@ -124,6 +124,22 @@ Most of the issues are around Plotly rendering. Things that have been hit repeat
 
 - Comments in [`environment.yml`](https://github.com/afeld/python-public-policy/blob/main/extras/environment.yml)
 - [Student troubleshooting guide](../assignments.md#common-issues)
+
+## Autograder
+
+Based on the [Gradescope instructions](https://gradescope-autograders.readthedocs.io/en/latest/manual_docker/):
+
+```sh
+cd extras/autograder
+mkdir -p results
+
+docker run --rm \
+  -v .:/autograder \
+  gradescope/autograder-base \
+  /bin/bash -c "/autograder/source/setup.sh && /autograder/source/run_autograder"
+
+cat results/results.json
+```
 {%- endif %}
 
 ## Contacts
diff --git a/pyproject.toml b/pyproject.toml
index aa4949aa..44804ccb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,2 +1,7 @@
 [tool.black]
 line-length = 100
+
+[tool.pytest.ini_options]
+addopts = [
+    "--ignore=extras/autograder/",
+]

From d79845c5935151ea7d9d7d983ae685501acf37d5 Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 21:47:02 -0400
Subject: [PATCH 2/6] move local autograding to Make target

---
 Makefile                 | 11 +++++++++++
 meta/instructor_guide.md | 12 ++----------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 85f0abe5..fd56e411 100644
--- a/Makefile
+++ b/Makefile
@@ -30,3 +30,14 @@ update_packages:
 	./extras/scripts/update_lectures.sh
 
 	echo "Please update homework notebooks separately, in python-public-policy-assignments"
+
+# based on https://gradescope-autograders.readthedocs.io/en/latest/manual_docker/
+autograde:
+	mkdir -p ./extras/autograder/results
+
+	docker run --rm \
+		-v ./extras/autograder:/autograder \
+		gradescope/autograder-base \
+		/bin/bash -c "/autograder/source/setup.sh && /autograder/source/run_autograder"
+
+	cat ./extras/autograder/results/results.json
diff --git a/meta/instructor_guide.md b/meta/instructor_guide.md
index 8561283f..dfbd7d75 100644
--- a/meta/instructor_guide.md
+++ b/meta/instructor_guide.md
@@ -127,18 +127,10 @@ Most of the issues are around Plotly rendering. Things that have been hit repeat
 
 ## Autograder
 
-Based on the [Gradescope instructions](https://gradescope-autograders.readthedocs.io/en/latest/manual_docker/):
+Requires [Docker](https://www.docker.com/).
 
 ```sh
-cd extras/autograder
-mkdir -p results
-
-docker run --rm \
-  -v .:/autograder \
-  gradescope/autograder-base \
-  /bin/bash -c "/autograder/source/setup.sh && /autograder/source/run_autograder"
-
-cat results/results.json
+make autograde
 ```
 {%- endif %}
 

From b6d986e62c62f281585b36adc2307599b55246c5 Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 22:08:52 -0400
Subject: [PATCH 3/6] add command to build autograder

---
 .gitignore | 1 +
 Makefile   | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/.gitignore b/.gitignore
index fc096c10..c8ebdbf3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ Untitled.ipynb
 !.vscode/settings.json
 
 extras/autograder/results/
+autograder.zip
diff --git a/Makefile b/Makefile
index fd56e411..43291afd 100644
--- a/Makefile
+++ b/Makefile
@@ -41,3 +41,9 @@ autograde:
 		/bin/bash -c "/autograder/source/setup.sh && /autograder/source/run_autograder"
 
 	cat ./extras/autograder/results/results.json
+
+build_autograder:
+	# https://stackoverflow.com/a/17351814/358804
+	git archive -o ./extras/autograder.zip HEAD:./extras/autograder/source
+
+	echo "Now upload extras/autograder.zip to Gradescope."

From 1770216e77fe5024797843e165e3acd683e647ff Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 22:32:39 -0400
Subject: [PATCH 4/6] test for one Python file and one notebook file

---
 extras/autograder/source/tests/test_files.py  | 21 +++++++++++++++++++
 extras/autograder/source/tests/test_simple.py | 10 ---------
 extras/environment.yml                        |  3 +++
 3 files changed, 24 insertions(+), 10 deletions(-)
 create mode 100644 extras/autograder/source/tests/test_files.py
 delete mode 100644 extras/autograder/source/tests/test_simple.py

diff --git a/extras/autograder/source/tests/test_files.py b/extras/autograder/source/tests/test_files.py
new file mode 100644
index 00000000..b7b022cc
--- /dev/null
+++ b/extras/autograder/source/tests/test_files.py
@@ -0,0 +1,21 @@
+"""https://github.com/gradescope/gradescope-utils/tree/master/gradescope_utils/autograder_utils#readme"""
+
+from unittest import TestCase
+import os
+from gradescope_utils.autograder_utils.decorators import number
+
+
+class TestFiles(TestCase):
+    @number("1.1")
+    def test_notebook_and_py_file(self):
+        """There should be exactly one notebook and one Python file submitted"""
+
+        files = os.listdir("/autograder/submission")
+
+        self.assertEqual(len(files), 2, "There should be exactly two files submitted.")
+
+        for ext in [".ipynb", ".py"]:
+            ext_files = [f for f in files if f.endswith(ext)]
+            self.assertEqual(
+                len(ext_files), 1, f"There should be exactly one {ext} file."
+            )
diff --git a/extras/autograder/source/tests/test_simple.py b/extras/autograder/source/tests/test_simple.py
deleted file mode 100644
index a6b89382..00000000
--- a/extras/autograder/source/tests/test_simple.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import unittest
-from gradescope_utils.autograder_utils.decorators import weight, number
-
-
-class TestSimpleArithmetic(unittest.TestCase):
-    @weight(1)
-    @number("1.1")
-    def test_eval_add(self):
-        """Evaluate 1 + 1"""
-        self.assertEqual(1+1, 2)
diff --git a/extras/environment.yml b/extras/environment.yml
index 5980f964..73db5b8a 100644
--- a/extras/environment.yml
+++ b/extras/environment.yml
@@ -7,6 +7,9 @@ dependencies:
   - black
   - nbqa
   - notebook=7.*
+  - pip
+  - pip:
+      - gradescope_utils
 
   # extensions
   - jupyter-resource-usage

From 16ea834f2ad09e23b65f61ab4eaa6fd93a3950e5 Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 22:41:32 -0400
Subject: [PATCH 5/6] simplify autograding code

---
 extras/autograder/source/tests/test_files.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/extras/autograder/source/tests/test_files.py b/extras/autograder/source/tests/test_files.py
index b7b022cc..263dbbc5 100644
--- a/extras/autograder/source/tests/test_files.py
+++ b/extras/autograder/source/tests/test_files.py
@@ -11,11 +11,8 @@ def test_notebook_and_py_file(self):
         """There should be exactly one notebook and one Python file submitted"""
 
         files = os.listdir("/autograder/submission")
-
-        self.assertEqual(len(files), 2, "There should be exactly two files submitted.")
-
-        for ext in [".ipynb", ".py"]:
-            ext_files = [f for f in files if f.endswith(ext)]
-            self.assertEqual(
-                len(ext_files), 1, f"There should be exactly one {ext} file."
-            )
+        extensions = [os.path.splitext(filename)[1] for filename in files]
+        extensions.sort()
+        self.assertListEqual(
+            extensions, [".ipynb", ".py"], f"Files submitted: {', '.join(files)}"
+        )

From ed62c3b1abeae58f9ced32a12b359af680f26f14 Mon Sep 17 00:00:00 2001
From: Aidan Feldman <aidan.feldman@gmail.com>
Date: Sat, 15 Mar 2025 22:51:28 -0400
Subject: [PATCH 6/6] ignore submission files from repo

---
 .gitignore                                 |  1 +
 Makefile                                   |  2 +-
 extras/autograder/submission/example.ipynb | 16 ----------------
 extras/autograder/submission/example.py    |  0
 meta/instructor_guide.md                   |  2 +-
 5 files changed, 3 insertions(+), 18 deletions(-)
 delete mode 100644 extras/autograder/submission/example.ipynb
 delete mode 100644 extras/autograder/submission/example.py

diff --git a/.gitignore b/.gitignore
index c8ebdbf3..2bf4b979 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,4 +16,5 @@ Untitled.ipynb
 !.vscode/settings.json
 
 extras/autograder/results/
+extras/autograder/submission/
 autograder.zip
diff --git a/Makefile b/Makefile
index 43291afd..e9c7a5a7 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ update_packages:
 
 # based on https://gradescope-autograders.readthedocs.io/en/latest/manual_docker/
 autograde:
-	mkdir -p ./extras/autograder/results
+	mkdir -p ./extras/autograder/results ./extras/autograder/results
 
 	docker run --rm \
 		-v ./extras/autograder:/autograder \
diff --git a/extras/autograder/submission/example.ipynb b/extras/autograder/submission/example.ipynb
deleted file mode 100644
index b3310258..00000000
--- a/extras/autograder/submission/example.ipynb
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "cells": [],
- "metadata": {
-  "kernelspec": {
-   "display_name": "python-public-policy",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "name": "python",
-   "version": "3.11.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/extras/autograder/submission/example.py b/extras/autograder/submission/example.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/meta/instructor_guide.md b/meta/instructor_guide.md
index dfbd7d75..19449c68 100644
--- a/meta/instructor_guide.md
+++ b/meta/instructor_guide.md
@@ -127,7 +127,7 @@ Most of the issues are around Plotly rendering. Things that have been hit repeat
 
 ## Autograder
 
-Requires [Docker](https://www.docker.com/).
+Requires [Docker](https://www.docker.com/). Put files in `extras/autograder/submission/`, then run:
 
 ```sh
 make autograde