From 9f403b09dc27bd307e9a4a9d79c3e3e785e2274d Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Sat, 13 Jun 2020 16:39:54 -0400 Subject: [PATCH] use graph.nodes(data='payload') to simpify code see https://networkx.github.io/documentation/stable/reference/classes/generated/networkx.Graph.nodes.html#networkx.Graph.nodes --- conda_forge_tick/auto_tick.py | 12 +- conda_forge_tick/contexts.py | 149 +++++++++---------- conda_forge_tick/make_graph.py | 16 +- conda_forge_tick/migrators/core.py | 4 +- conda_forge_tick/migrators/migration_yaml.py | 8 +- conda_forge_tick/status_report.py | 9 +- conda_forge_tick/update_upstream_versions.py | 11 +- 7 files changed, 103 insertions(+), 106 deletions(-) diff --git a/conda_forge_tick/auto_tick.py b/conda_forge_tick/auto_tick.py index c87ab03f3..88814cfce 100644 --- a/conda_forge_tick/auto_tick.py +++ b/conda_forge_tick/auto_tick.py @@ -326,8 +326,8 @@ def add_replacement_migrator( """ total_graph = copy.deepcopy(gx) - for node, node_attrs in gx.nodes.items(): - requirements = node_attrs["payload"].get("requirements", {}) + for node, node_attrs in gx.nodes(data='payload'): + requirements = node_attrs.get("requirements", {}) rq = ( requirements.get("build", set()) | requirements.get("host", set()) @@ -476,14 +476,14 @@ def migration_factory( # TODO: use the inbuilt LUT in the graph output_to_feedstock = { output: name - for name, node in gx.nodes.items() - for output in node.get("payload", {}).get("outputs_names", []) + for name, node in gx.nodes(data='payload') + for output in node.get("outputs_names", []) } all_package_names = set(gx.nodes) | set( sum( [ - node.get("payload", {}).get("outputs_names", []) - for node in gx.nodes.values() + node.get("outputs_names", []) + for _, node in gx.nodes(data='payload') ], [], ) diff --git a/conda_forge_tick/contexts.py b/conda_forge_tick/contexts.py index ab74bd2f0..2e159c597 100644 --- a/conda_forge_tick/contexts.py +++ b/conda_forge_tick/contexts.py @@ -1,75 +1,74 @@ -import copy -from dataclasses import dataclass -from networkx import DiGraph -import typing -import threading -import github3 - -if typing.TYPE_CHECKING: - from conda_forge_tick.migrators import Migrator - from conda_forge_tick.migrators_types import AttrsTypedDict - - -@dataclass -class GithubContext: - github_username: str - github_password: str - circle_build_url: str - github_token: typing.Optional[str] = "" - dry_run: bool = True - _tl: threading.local = threading.local() - - @property - def gh(self) -> github3.GitHub: - if getattr(self._tl, "gh", None) is None: - if self.github_token: - gh = github3.login(token=self.github_token) - else: - gh = github3.login(self.github_username, self.github_password) - setattr(self._tl, "gh", gh) - return self._tl.gh - - -@dataclass -class MigratorSessionContext(GithubContext): - """Singleton session context. There should generally only be one of these""" - - graph: DiGraph = None - smithy_version: str = "" - pinning_version: str = "" - prjson_dir = "pr_json" - rever_dir: str = "./feedstocks/" - quiet = True - - -@dataclass -class MigratorContext: - """The context for a given migrator. This houses the runtime information that a migrator needs""" - - session: MigratorSessionContext - migrator: "Migrator" - _effective_graph: DiGraph = None - - @property - def github_username(self) -> str: - return self.session.github_username - - @property - def effective_graph(self) -> DiGraph: - if self._effective_graph is None: - gx2 = copy.deepcopy(getattr(self.migrator, "graph", self.session.graph)) - - # Prune graph to only things that need builds right now - for node, node_attrs in self.session.graph.nodes.items(): - attrs = node_attrs.get("payload", {}) - if node in gx2 and self.migrator.filter(attrs): - gx2.remove_node(node) - self._effective_graph = gx2 - return self._effective_graph - - -@dataclass -class FeedstockContext: - package_name: str - feedstock_name: str - attrs: "AttrsTypedDict" +import copy +from dataclasses import dataclass +from networkx import DiGraph +import typing +import threading +import github3 + +if typing.TYPE_CHECKING: + from conda_forge_tick.migrators import Migrator + from conda_forge_tick.migrators_types import AttrsTypedDict + + +@dataclass +class GithubContext: + github_username: str + github_password: str + circle_build_url: str + github_token: typing.Optional[str] = "" + dry_run: bool = True + _tl: threading.local = threading.local() + + @property + def gh(self) -> github3.GitHub: + if getattr(self._tl, "gh", None) is None: + if self.github_token: + gh = github3.login(token=self.github_token) + else: + gh = github3.login(self.github_username, self.github_password) + setattr(self._tl, "gh", gh) + return self._tl.gh + + +@dataclass +class MigratorSessionContext(GithubContext): + """Singleton session context. There should generally only be one of these""" + + graph: DiGraph = None + smithy_version: str = "" + pinning_version: str = "" + prjson_dir = "pr_json" + rever_dir: str = "./feedstocks/" + quiet = True + + +@dataclass +class MigratorContext: + """The context for a given migrator. This houses the runtime information that a migrator needs""" + + session: MigratorSessionContext + migrator: "Migrator" + _effective_graph: DiGraph = None + + @property + def github_username(self) -> str: + return self.session.github_username + + @property + def effective_graph(self) -> DiGraph: + if self._effective_graph is None: + gx2 = copy.deepcopy(getattr(self.migrator, "graph", self.session.graph)) + + # Prune graph to only things that need builds right now + for node, attrs in self.session.graph.nodes(data="payload"): + if node in gx2 and self.migrator.filter(attrs): + gx2.remove_node(node) + self._effective_graph = gx2 + return self._effective_graph + + +@dataclass +class FeedstockContext: + package_name: str + feedstock_name: str + attrs: "AttrsTypedDict" diff --git a/conda_forge_tick/make_graph.py b/conda_forge_tick/make_graph.py index 61cc1078d..197ee26b1 100644 --- a/conda_forge_tick/make_graph.py +++ b/conda_forge_tick/make_graph.py @@ -291,21 +291,21 @@ def make_graph( # make the outputs look up table so we can link properly outputs_lut = { k: node_name - for node_name, node in gx.nodes.items() - for k in node.get("payload", {}).get("outputs_names", []) + for node_name, node in gx.nodes(data='payload') + for k in node.get("outputs_names", []) } # add this as an attr so we can use later gx.graph["outputs_lut"] = outputs_lut strong_exports = { node_name - for node_name, node in gx.nodes.items() - if node.get("payload").get("strong_exports", False) + for node_name, node in gx.nodes(data='payload') + if node.get("strong_exports", False) } # This drops all the edge data and only keeps the node data gx = nx.create_empty_copy(gx) # TODO: label these edges with the kind of dep they are and their platform - for node, node_attrs in gx2.nodes.items(): - with node_attrs["payload"] as attrs: + for node, attrs in gx2.nodes(data='payload'): + with attrs: # replace output package names with feedstock names via LUT deps = set( map( @@ -335,8 +335,8 @@ def make_graph( def update_nodes_with_bot_rerun(gx): """Go through all the open PRs and check if they are rerun""" - for name, node in gx.nodes.items(): - with node['payload'] as payload: + for name, payload in gx.nodes(data='payload'): + with payload: for migration in payload.get('PRed', []): pr_json = migration.get('PR', {}) # if there is a valid PR and it isn't currently listed as rerun diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index b7566957f..302659f65 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -453,8 +453,8 @@ def __init__( else: self.outputs_lut = { k: node_name - for node_name, node in self.graph.nodes.items() - for k in node.get("payload", {}).get("outputs_names", []) + for node_name, node in self.graph.nodes(data='payload') + for k in node.get("outputs_names", []) } self.name = name diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index 7366a7f5a..aa0c65a1f 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -151,8 +151,8 @@ def __init__( number_pred = len( [ k - for k, v in self.graph.nodes.items() - if self.migrator_uid(v.get("payload", {})) + for k, v in self.graph.nodes(data='payload') + if self.migrator_uid(v) in [vv.get("data", {}) for vv in v.get("payload", {}).get("PRed", [])] ], ) @@ -422,11 +422,11 @@ def create_rebuild_graph( total_graph = copy.deepcopy(gx) excluded_feedstocks = set() if excluded_feedstocks is None else excluded_feedstocks - for node, node_attrs in gx.nodes.items(): + for node, attrs in gx.nodes(data='payload'): # always keep pinning if node == 'conda-forge-pinning': continue - attrs: "AttrsTypedDict" = node_attrs["payload"] + attrs: "AttrsTypedDict" requirements = attrs.get("requirements", {}) host = requirements.get("host", set()) build = requirements.get("build", set()) diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 68910b50f..5caa40c89 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -85,8 +85,7 @@ def graph_migrator_status( if 'conda-forge-pinning' in gx2.nodes(): gx2.remove_node('conda-forge-pinning') - for node, node_attrs in gx2.nodes.items(): - attrs = node_attrs["payload"] + for node, attrs in gx2.nodes(data="payload"): # remove archived from status if attrs.get("archived", False): continue @@ -235,11 +234,11 @@ def main(args: Any = None) -> None: lst = [ k - for k, v in mctx.graph.nodes.items() + for k, v in mctx.graph.nodes(data="payload") if len( [ z - for z in v.get("payload", {}).get("PRed", []) + for z in v.get("PRed", []) if z.get("PR", {}).get("state", "closed") == "open" and z.get("data", {}).get("migrator_name", "") == "Version" ], @@ -259,7 +258,7 @@ def main(args: Any = None) -> None: lm = LicenseMigrator() lst = [ - k for k, v in mctx.graph.nodes.items() if not lm.filter(v.get("payload", {})) + k for k, v in mctx.graph.nodes(data="payload") if not lm.filter(v) ] with open("./status/unlicensed.json", "w") as f: json.dump( diff --git a/conda_forge_tick/update_upstream_versions.py b/conda_forge_tick/update_upstream_versions.py index 4308ed32e..0c670e4f4 100644 --- a/conda_forge_tick/update_upstream_versions.py +++ b/conda_forge_tick/update_upstream_versions.py @@ -498,12 +498,11 @@ def get_latest_version( def _update_upstream_versions_sequential( gx: nx.DiGraph, sources: Iterable[AbstractSource], ) -> None: - _all_nodes = [t for t in gx.nodes.items()] + _all_nodes = [t for t in gx.nodes(data='payload')] random.shuffle(_all_nodes) to_update = [] - for node, node_attrs in _all_nodes: - attrs = node_attrs["payload"] + for node, attrs in _all_nodes: if attrs.get("bad") or attrs.get("archived"): attrs["new_version"] = False continue @@ -534,10 +533,10 @@ def _update_upstream_versions_process_pool( # this has to be threads because the url hashing code uses a Pipe which # cannot be spawned from a process with executor(kind="dask", max_workers=20) as pool: - _all_nodes = [t for t in gx.nodes.items()] + _all_nodes = [t for t in gx.nodes(data='payload')] random.shuffle(_all_nodes) - for node, node_attrs in tqdm.tqdm(_all_nodes): - with node_attrs["payload"] as attrs: + for node, attrs in tqdm.tqdm(_all_nodes): + with attrs: if node == "ca-policy-lcg": attrs["new_version"] = False continue