From 101156bdc7c8df8c1d0eba15b80609158ae48f9c Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sun, 12 Sep 2021 12:09:26 -0500 Subject: [PATCH 1/2] skip version updates depending on a regex from the feedstock --- conda_forge_tick/update_sources.py | 76 ++++++++++++++++---- conda_forge_tick/update_upstream_versions.py | 2 +- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/conda_forge_tick/update_sources.py b/conda_forge_tick/update_sources.py index d173bfbfe..16869cc6c 100644 --- a/conda_forge_tick/update_sources.py +++ b/conda_forge_tick/update_sources.py @@ -113,13 +113,36 @@ class AbstractSource(abc.ABC): name: str @abc.abstractmethod - def get_version(self, url: str) -> Optional[str]: + def get_version(self, url: str, meta_yaml) -> Optional[str]: pass @abc.abstractmethod def get_url(self, meta_yaml) -> Optional[str]: pass + def get_bot_settings(self, meta_yaml): + if "conda-forge.yml" not in meta_yaml: + return {} + conda_forge_yml = meta_yaml["conda-forge.yml"] + # First check for the text for an early exit + # to avoid parsing yaml if possible + if "bot" not conda_forge_yaml: + return {} + # Parse the yaml now + parsed_yml = yaml.safe_load(conda_forge_yml) + return parsed_yml.get("bot", {}) + + def skip_version(self, version, meta_yaml) -> bool: + if not version: + return True + bot_settings = self.get_bot_settings(meta_yaml) + if not bot_settings: + return False + if "version_regex" not in bot_settings: + return False + ver_regex = bot_settings["version_regex"] + return re.match(ver_regex, version) is None + class VersionFromFeed(AbstractSource): ver_prefix_remove = ["release-", "releases%2F", "v_", "v.", "v"] @@ -137,7 +160,7 @@ class VersionFromFeed(AbstractSource): "git", ] - def get_version(self, url) -> Optional[str]: + def get_version(self, url, meta_yaml) -> Optional[str]: data = feedparser.parse(url) if data["bozo"] == 1: return None @@ -151,6 +174,8 @@ def get_version(self, url) -> Optional[str]: continue # Extract version number starting at the first digit. ver = re.search(r"(\d+[^\s]*)", ver).group(0) + if self.skip_version(ver, meta_yaml): + continue vers.append(ver) if vers: return max(vers, key=lambda x: VersionOrder(x.replace("-", "."))) @@ -169,7 +194,7 @@ def get_url(self, meta_yaml) -> Optional[str]: pkg = meta_yaml["url"].split("/")[6] return f"https://pypi.org/pypi/{pkg}/json" - def get_version(self, url) -> Optional[str]: + def get_version(self, url, meta_yaml) -> Optional[str]: r = requests.get(url) # If it is a pre-release don't give back the pre-release version most_recent_version = r.json()["info"]["version"].strip() @@ -178,6 +203,10 @@ def get_version(self, url) -> Optional[str]: if all(parse_version(v).is_prerelease for v in r.json()["releases"]): return most_recent_version return False + # FIXME: instead of skipping here, use the latest version that + # is not skipped. + if self.skip_version(most_recent_version, meta_yaml): + return False return most_recent_version @@ -191,13 +220,16 @@ def get_url(self, meta_yaml) -> Optional[str]: pkg = meta_yaml["url"].split("/")[3:-2] return f"https://registry.npmjs.org/{'/'.join(pkg)}" - def get_version(self, url: str) -> Optional[str]: + def get_version(self, url: str, meta_yaml) -> Optional[str]: r = requests.get(url) if not r.ok: return False latest = r.json()["dist-tags"].get("latest", "").strip() # If it is a pre-release don't give back the pre-release version - if not len(latest) or parse_version(latest).is_prerelease: + if not len(latest) or parse_version(latest).is_prerelease or \ + self.skip_version(latest, meta_yaml): + # FIXME: instead of skipping here, use the latest version that + # is not skipped. return False return latest @@ -263,8 +295,15 @@ def get_url(self, meta_yaml) -> Optional[str]: return None return None - def get_version(self, url) -> Optional[str]: - return str(url[1]).replace("-", "_") if url[1] else None + def get_version(self, url, meta_yaml) -> Optional[str]: + if not url[1]: + return None + ver = str(url[1]).replace("-", "_") + # FIXME: instead of skipping here, use the latest version that + # is not skipped. This requires looking at the CRAN archive index + # which we do not currently look at. + if self.skip_version(ver, meta_yaml): + return None ROS_DISTRO_INDEX: Optional[dict] = None @@ -341,8 +380,14 @@ def get_url(self, meta_yaml: "MetaYamlTypedDict") -> Optional[str]: return final_url - def get_version(self, url): - return self.version_url_cache[url] + def get_version(self, url, meta_yaml): + ver = self.version_url_cache[url] + # FIXME: instead of skipping here, use the latest version that + # is not skipped. + if self.skip_version(ver, meta_yaml): + return False + return ver + def get_sha256(url: str) -> Optional[str]: @@ -428,8 +473,9 @@ def get_url(self, meta_yaml) -> Optional[str]: orig_ver = current_ver found = True count = 0 - max_count = 10 + max_count = self.get_bot_settings(meta_yaml).get("version_update_max_check", 10) + found_ver = orig_ver while found and count < max_count: found = False for next_ver in self.next_ver_func(current_ver): @@ -500,15 +546,17 @@ def get_url(self, meta_yaml) -> Optional[str]: return None current_sha256 = new_sha256 logger.debug("version %s is ok for url %s", current_ver, url_to_use) + if not self.skip_version(current_ver, meta_yaml): + found_ver = current_ver break - if current_ver != orig_ver: - logger.debug("using version %s", current_ver) - return current_ver + if found_ver != orig_ver: + logger.debug("using version %s", found_ver) + return found_ver return None - def get_version(self, url: str) -> str: + def get_version(self, url: str, meta_yaml) -> str: return url diff --git a/conda_forge_tick/update_upstream_versions.py b/conda_forge_tick/update_upstream_versions.py index 8e7524a41..be5768f53 100644 --- a/conda_forge_tick/update_upstream_versions.py +++ b/conda_forge_tick/update_upstream_versions.py @@ -43,7 +43,7 @@ def get_latest_version( logger.debug("url: %s", url) if url is None: continue - ver = source.get_version(url) + ver = source.get_version(url, meta_yaml) logger.debug("ver: %s", ver) if ver: version_data["new_version"] = ver From 0ed56b4881aca627888fbf42b2a64d87a6211c60 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sun, 12 Sep 2021 12:10:40 -0500 Subject: [PATCH 2/2] Fix typo --- conda_forge_tick/update_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_sources.py b/conda_forge_tick/update_sources.py index 16869cc6c..400014d2d 100644 --- a/conda_forge_tick/update_sources.py +++ b/conda_forge_tick/update_sources.py @@ -126,7 +126,7 @@ def get_bot_settings(self, meta_yaml): conda_forge_yml = meta_yaml["conda-forge.yml"] # First check for the text for an early exit # to avoid parsing yaml if possible - if "bot" not conda_forge_yaml: + if "bot" not in conda_forge_yaml: return {} # Parse the yaml now parsed_yml = yaml.safe_load(conda_forge_yml)