ci: swith type checker from ignoring files to specific lines

Author: ahal

pyproject.toml

@@ -34,32 +34,4 @@ known-first-party = ["taskgraph"]
 
 [tool.pyright]
 include = ["src"]
-exclude = [  # TODO fix errors in these files
-    "src/taskgraph/decision.py",
-    "src/taskgraph/docker.py",
-    "src/taskgraph/files_changed.py",
-    "src/taskgraph/generator.py",
-    "src/taskgraph/graph.py",
-    "src/taskgraph/main.py",
-    "src/taskgraph/morph.py",
-    "src/taskgraph/parameters.py",
-    "src/taskgraph/taskgraph.py",
-    "src/taskgraph/actions/cancel.py",
-    "src/taskgraph/actions/cancel_all.py",
-    "src/taskgraph/actions/rebuild_cached_tasks.py",
-    "src/taskgraph/actions/util.py",
-    "src/taskgraph/optimize/base.py",
-    "src/taskgraph/run-task/robustcheckout.py",
-    "src/taskgraph/transforms/fetch.py",
-    "src/taskgraph/transforms/task.py",
-    "src/taskgraph/transforms/run/run_task.py",
-    "src/taskgraph/util/cached_tasks.py",
-    "src/taskgraph/util/decision.py",
-    "src/taskgraph/util/python_path.py",
-    "src/taskgraph/util/schema.py",
-    "src/taskgraph/util/taskcluster.py",
-    "src/taskgraph/util/vcs.py",
-    "src/taskgraph/util/workertypes.py",
-    "src/taskgraph/util/yaml.py",
-]
 reportIncompatibleMethodOverride = false

src/taskgraph/actions/cancel.py

@@ -29,7 +29,7 @@ def cancel_action(parameters, graph_config, input, task_group_id, task_id):
     try:
         cancel_task(task_id, use_proxy=True)
     except requests.HTTPError as e:
-        if e.response.status_code == 409:
+        if e.response.status_code == 409:  # type: ignore
             # A 409 response indicates that this task is past its deadline.  It
             # cannot be cancelled at this time, but it's also not running
             # anymore, so we can ignore this error.

src/taskgraph/actions/cancel_all.py

@@ -38,7 +38,7 @@ def do_cancel_task(task_id):
         try:
             cancel_task(task_id, use_proxy=True)
         except requests.HTTPError as e:
-            if e.response.status_code == 409:
+            if e.response.status_code == 409:  # type: ignore
                 # A 409 response indicates that this task is past its deadline.  It
                 # cannot be cancelled at this time, but it's also not running
                 # anymore, so we can ignore this error.

src/taskgraph/actions/rebuild_cached_tasks.py

@@ -22,7 +22,7 @@ def rebuild_cached_tasks_action(
     )
     cached_tasks = [
         label
-        for label, task in full_task_graph.tasks.items()
+        for label, task in full_task_graph.tasks.items()  # type: ignore
         if task.attributes.get("cached_task", False)
     ]
     if cached_tasks:

src/taskgraph/actions/util.py

@@ -60,7 +60,7 @@ def fetch_action(task_id):
                 run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
                 label_to_taskid.update(run_label_to_id)
             except HTTPError as e:
-                if e.response.status_code != 404:
+                if e.response.status_code != 404:  # type: ignore
                     raise
                 logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
 
@@ -79,7 +79,7 @@ def fetch_cron(task_id):
                 run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
                 label_to_taskid.update(run_label_to_id)
             except HTTPError as e:
-                if e.response.status_code != 404:
+                if e.response.status_code != 404:  # type: ignore
                     raise
                 logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
 
@@ -160,7 +160,7 @@ def create_tasks(
 
     target_graph = full_task_graph.graph.transitive_closure(to_run)
     target_task_graph = TaskGraph(
-        {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph
+        {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph  # type: ignore
     )
     target_task_graph.for_each_task(update_parent)
     if decision_task_id and decision_task_id != os.environ.get("TASK_ID"):

src/taskgraph/decision.py

@@ -359,7 +359,7 @@ def write_artifact(filename, data):
         import gzip
 
         with gzip.open(path, "wb") as f:
-            f.write(json.dumps(data))
+            f.write(json.dumps(data))  # type: ignore
     else:
         raise TypeError(f"Don't know how to write to {filename}")
 

src/taskgraph/docker.py

@@ -173,11 +173,11 @@ def download_and_modify_image():
         req = get_session().get(url, stream=True)
         req.raise_for_status()
 
-        with zstd.ZstdDecompressor().stream_reader(req.raw) as ifh:
+        with zstd.ZstdDecompressor().stream_reader(req.raw) as ifh:  # type: ignore
             tarin = tarfile.open(
                 mode="r|",
                 fileobj=ifh,
-                bufsize=zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
+                bufsize=zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,  # type: ignore
             )
 
             # Stream through each member of the downloaded tar file individually.
@@ -194,8 +194,8 @@ def download_and_modify_image():
                 # image tags.
                 if member.name == "repositories":
                     # Read and parse repositories
-                    repos = json.loads(reader.read())
-                    reader.close()
+                    repos = json.loads(reader.read())  # type: ignore
+                    reader.close()  # type: ignore
 
                     # If there is more than one image or tag, we can't handle it
                     # here.
@@ -217,16 +217,16 @@ def download_and_modify_image():
                 # Then emit its content.
                 remaining = member.size
                 while remaining:
-                    length = min(remaining, zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
-                    buf = reader.read(length)
+                    length = min(remaining, zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)  # type: ignore
+                    buf = reader.read(length)  # type: ignore
                     remaining -= len(buf)
                     yield buf
                 # Pad to fill a 512 bytes block, per tar format.
                 remainder = member.size % 512
                 if remainder:
                     yield ("\0" * (512 - remainder)).encode("utf-8")
 
-                reader.close()
+                reader.close()  # type: ignore
 
     subprocess.run(
         ["docker", "image", "load"], input=b"".join(download_and_modify_image())

src/taskgraph/generator.py

@@ -138,7 +138,7 @@ def __init__(
         self._write_artifacts = write_artifacts
 
         # start the generator
-        self._run = self._run()
+        self._run = self._run()  # type: ignore
         self._run_results = {}
 
     @property
@@ -295,7 +295,7 @@ def _run(self):
         for kind in kinds.values():
             for dep in kind.config.get("kind-dependencies", []):
                 edges.add((kind.name, dep, "kind-dependency"))
-        kind_graph = Graph(set(kinds), edges)
+        kind_graph = Graph(set(kinds), edges)  # type: ignore
 
         if target_kinds:
             kind_graph = kind_graph.transitive_closure(
@@ -321,7 +321,7 @@ def _run(self):
                     raise Exception("duplicate tasks with label " + task.label)
                 all_tasks[task.label] = task
             logger.info(f"Generated {len(new_tasks)} tasks for kind {kind_name}")
-        full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
+        full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))  # type: ignore
         yield self.verify("full_task_set", full_task_set, graph_config, parameters)
 
         logger.info("Generating full task graph")
@@ -334,7 +334,7 @@ def _run(self):
                     )
                 edges.add((t.label, dep, depname))
 
-        full_task_graph = TaskGraph(all_tasks, Graph(full_task_set.graph.nodes, edges))
+        full_task_graph = TaskGraph(all_tasks, Graph(full_task_set.graph.nodes, edges))  # type: ignore
         logger.info(
             "Full task graph contains %d tasks and %d dependencies"
             % (len(full_task_set.graph.nodes), len(edges))
@@ -343,13 +343,13 @@ def _run(self):
 
         logger.info("Generating target task set")
         target_task_set = TaskGraph(
-            dict(all_tasks), Graph(set(all_tasks.keys()), set())
+            dict(all_tasks), Graph(set(all_tasks.keys()), set())  # type: ignore
         )
         for fltr in filters:
             old_len = len(target_task_set.graph.nodes)
             target_tasks = set(fltr(target_task_set, parameters, graph_config))
             target_task_set = TaskGraph(
-                {l: all_tasks[l] for l in target_tasks}, Graph(target_tasks, set())
+                {l: all_tasks[l] for l in target_tasks}, Graph(target_tasks, set())  # type: ignore
             )
             logger.info(
                 "Filter %s pruned %d tasks (%d remain)"
@@ -363,7 +363,7 @@ def _run(self):
         if parameters["enable_always_target"]:
             always_target_tasks = {
                 t.label
-                for t in full_task_graph.tasks.values()
+                for t in full_task_graph.tasks.values()  # type: ignore
                 if t.attributes.get("always_target")
                 if parameters["enable_always_target"] is True
                 or t.kind in parameters["enable_always_target"]
@@ -372,12 +372,12 @@ def _run(self):
             always_target_tasks = set()
         logger.info(
             "Adding %d tasks with `always_target` attribute"
-            % (len(always_target_tasks) - len(always_target_tasks & target_tasks))
+            % (len(always_target_tasks) - len(always_target_tasks & target_tasks))  # type: ignore
         )
-        requested_tasks = target_tasks | always_target_tasks
+        requested_tasks = target_tasks | always_target_tasks  # type: ignore
         target_graph = full_task_graph.graph.transitive_closure(requested_tasks)
         target_task_graph = TaskGraph(
-            {l: all_tasks[l] for l in target_graph.nodes}, target_graph
+            {l: all_tasks[l] for l in target_graph.nodes}, target_graph  # type: ignore
         )
         yield self.verify(
             "target_task_graph", target_task_graph, graph_config, parameters
@@ -422,7 +422,7 @@ def _run(self):
     def _run_until(self, name):
         while name not in self._run_results:
             try:
-                k, v = next(self._run)
+                k, v = next(self._run)  # type: ignore
             except StopIteration:
                 raise AttributeError(f"No such run result {name}")
             self._run_results[k] = v

src/taskgraph/graph.py

@@ -68,7 +68,7 @@ def transitive_closure(self, nodes, reverse=False):
             add_nodes = {(left if reverse else right) for (left, right, _) in add_edges}
             new_nodes = nodes | add_nodes
             new_edges = edges | add_edges
-        return Graph(new_nodes, new_edges)
+        return Graph(new_nodes, new_edges)  # type: ignore
 
     def _visit(self, reverse):
         queue = collections.deque(sorted(self.nodes))

src/taskgraph/main.py

@@ -88,10 +88,10 @@ def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys):
                     if regexprogram.match(dep):
                         filterededges.add((key, dep, depname))
 
-        taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))
+        taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))  # type: ignore
 
     if exclude_keys:
-        for label, task in taskgraph.tasks.items():
+        for label, task in taskgraph.tasks.items():  # type: ignore
             task = task.to_json()
             for key in exclude_keys:
                 obj = task
@@ -490,7 +490,7 @@ def show_taskgraph(options):
             base_path = os.path.join(
                 diffdir, f"{options['graph_attr']}_{base_rev_file}"
             )
-            cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_rev_file}")
+            cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_rev_file}")  # type: ignore
 
             params_name = None
             if len(parameters) > 1:
@@ -862,7 +862,7 @@ def init_taskgraph(options):
     context = {"project_name": root.name}
 
     try:
-        repo_url = repo.get_url(remote=repo.remote_name)
+        repo_url = repo.get_url(remote=repo.remote_name)  # type: ignore
     except RuntimeError:
         repo_url = ""
 
@@ -895,7 +895,7 @@ def init_taskgraph(options):
         directory="template",
         extra_context=context,
         no_input=options["no_input"],
-        output_dir=root.parent,
+        output_dir=root.parent,  # type: ignore
         overwrite_if_exists=True,
     )