Enhancement (#12)

* Change copyright template

* fix pdoc version

* fix look for news files

* refactored some git helpers

* changed API for creating news files

* flake8

* fix pdoc version

* News files

Author: acabarbaye

Pipfile

@@ -16,7 +16,7 @@ pytest = "*"
 pyfakefs = "*"
 pytest-cov = "*"
 wheel = "*"
-continuous-delivery-scripts = {path = "."}
+continuous-delivery-scripts = {editable = true, path = "."}
 pre-commit = "*"
 
 [pipenv]

azure-pipelines/build-release.yml

@@ -39,6 +39,11 @@ stages:
             Linux_Py_3_8:
               python.version: '3.8'
               vmImageName: ubuntu-latest
+              uploadCoverage: "false"
+
+            Linux_Py_3_9:
+              python.version: '3.9'
+              vmImageName: ubuntu-latest
               uploadCoverage: "true"
 
             Windows_Py_3_6:
@@ -112,9 +117,9 @@ stages:
 
         steps:
           - task: UsePythonVersion@0
-            displayName: 'Use Python 3.7'
+            displayName: 'Use Python 3.9'
             inputs:
-              versionSpec: '3.7'
+              versionSpec: '3.9'
 
           - template: steps/determine-current-branch.yml
 
@@ -135,9 +140,9 @@ stages:
 
         steps:
           - task: UsePythonVersion@0
-            displayName: 'Use Python 3.7'
+            displayName: 'Use Python 3.9'
             inputs:
-              versionSpec: '3.7'
+              versionSpec: '3.9'
 
           - template: steps/install-development-dependencies.yml
 
@@ -158,9 +163,9 @@ stages:
 
         steps:
           - task: UsePythonVersion@0
-            displayName: 'Use Python 3.7'
+            displayName: 'Use Python 3.9'
             inputs:
-              versionSpec: '3.7'
+              versionSpec: '3.9'
 
           - template: steps/install-development-dependencies.yml
 
@@ -202,7 +207,7 @@ stages:
               steps:
                 - task: UsePythonVersion@0
                   inputs:
-                    versionSpec: '3.7'
+                    versionSpec: '3.9'
 
                 - template: steps/determine-current-branch.yml
 
@@ -237,7 +242,7 @@ stages:
               steps:
                 - task: UsePythonVersion@0
                   inputs:
-                    versionSpec: '3.7'
+                    versionSpec: '3.9'
 
                 - template: steps/determine-current-branch.yml
 

continuous_delivery_scripts/assert_news.py

@@ -13,6 +13,7 @@
 from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable
 from continuous_delivery_scripts.utils.git_helpers import ProjectTempClone, LocalProjectRepository, GitWrapper
 from continuous_delivery_scripts.utils.logging import log_exception, set_log_level
+from continuous_delivery_scripts.utils.news_file import create_news_file
 
 logger = logging.getLogger(__name__)
 
@@ -92,6 +93,39 @@ def validate_news_files(git: GitWrapper, root_dir: str, news_dir: str) -> None:
         validate_news_file(absolute_file_path)
 
 
+def add_news_files(git: GitWrapper, news_dir: str) -> None:
+    """Adds a news file if the branch corresponds to an dependency update.
+
+    Args:
+        git: Instance of GitWrapper.
+        news_dir: Relative path to news directory.
+    """
+    current_branch = str(git.get_current_branch())
+    is_dependency_update, groups = git.is_current_branch_of_type(
+        str(configuration.get_value(ConfigurationVariable.DEPENDENCY_UPDATE_BRANCH_PATTERN))
+    )
+    if not is_dependency_update:
+        raise EnvironmentError(f"Branch {current_branch} must contain a news file.")
+    if not configuration.get_value(ConfigurationVariable.AUTOGENERATE_NEWS_FILE_ON_DEPENDENCY_UPDATE):
+        raise EnvironmentError(f"Branch {current_branch} must contain a news file.")
+
+    create_news_file(
+        news_dir,
+        str(configuration.get_value(ConfigurationVariable.DEPENDENCY_UPDATE_NEWS_MESSAGE)).format(
+            message=", ".join(groups)
+        ),
+        configuration.get_value(ConfigurationVariable.DEPENDENCY_UPDATE_NEWS_TYPE),
+    )
+
+
+def _commit_news_file(git: GitWrapper, news_dir: str) -> None:
+    logger.info("Committing news file...")
+    git.add(news_dir)
+    git.commit("Adding news file")
+    git.push()
+    git.pull()
+
+
 def main() -> None:
     """Asserts the new PR comprises at least one news file and it adheres to the required standard."""
     parser = argparse.ArgumentParser(description="Check correctly formatted news files exist on feature branch.")
@@ -114,7 +148,12 @@ def main() -> None:
                 validate_news_files(git=git, news_dir=news_dir, root_dir=root_dir)
             except Exception as e:
                 log_exception(logger, e)
-                sys.exit(1)
+                try:
+                    add_news_files(git, absolute_news_dir)
+                    _commit_news_file(git, absolute_news_dir)
+                except Exception as e2:
+                    log_exception(logger, e2)
+                    sys.exit(1)
 
 
 if __name__ == "__main__":

continuous_delivery_scripts/create_news_file.py

@@ -8,70 +8,28 @@
     create-news-file "Fixed a bug" --type bugfix
 """
 import argparse
-import enum
 import logging
-import pathlib
+
 import sys
-from datetime import datetime
 
 from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable
 from continuous_delivery_scripts.assert_news import validate_news_file
 from continuous_delivery_scripts.utils.logging import log_exception
+from continuous_delivery_scripts.utils.news_file import NewsType, create_news_file
 
 logger = logging.getLogger(__name__)
 
-
 NEWS_DIR = configuration.get_value(ConfigurationVariable.NEWS_DIR)
 
 
-class NewsType(enum.Enum):
-    """Describes the type of news we're writing."""
-
-    bugfix = 0
-    doc = 1
-    feature = 2
-    major = 3
-    misc = 4
-    removal = 5
-
-
-def create_news_file(news_text: str, news_type: NewsType) -> pathlib.Path:
-    """Facilitates creating a news file, determining it's file name based on the type."""
-    file_path = determine_news_file_path(news_type)
-    _write_file(file_path, news_text)
-    return file_path
-
-
-def determine_news_file_path(news_type: NewsType) -> pathlib.Path:
-    """Returns an available file path for given news type."""
-    news_file_name = _determine_todays_news_file_name()
-    news_file_path = pathlib.Path(NEWS_DIR, f"{news_file_name}.{news_type.name}")
-    inc = 0
-    while news_file_path.exists():
-        inc += 1
-        news_file_path = news_file_path.with_name(f"{news_file_name}{inc:0=2}.{news_type.name}")
-    return news_file_path
-
-
-def _write_file(file_path: pathlib.Path, text: str) -> None:
-    file_path.parent.mkdir(parents=True, exist_ok=True)
-    if not text.endswith("\n"):
-        text = f"{text}\n"
-    file_path.write_text(text)
-
-
-def _determine_todays_news_file_name() -> str:
-    return datetime.now().strftime("%Y%m%d%H%M")
-
-
 def main() -> int:
     """Parses cli arguments and creates a news file."""
     parser = argparse.ArgumentParser()
     parser.add_argument("news_text", help="Contents of the news file.")
     parser.add_argument("--type", help="News type to create.", choices=[t.name for t in NewsType], default="feature")
 
     args = parser.parse_args()
-    created_file = create_news_file(args.news_text, NewsType[args.type])
+    created_file = create_news_file(str(NEWS_DIR), args.news_text, NewsType[args.type])
 
     try:
         validate_news_file(created_file)

continuous_delivery_scripts/license_files.py

@@ -25,7 +25,17 @@
 ADDITIONAL_EXTENSIONS = ["python=.toml", "c=.go"]
 
 
-def add_licence_header(verbose_count: int) -> None:
+def insert_licence_header(verbose_count: int) -> None:
+    """Inserts a copyright notice at the top of every source file of the current project.
+
+    Wrapper over the [licenseheaders tool](https://github.com/johann-petrak/licenseheaders).
+    """
+    # copyright (https://github.com/knipknap/copyright) was first considered but
+    # comprises quite a few bugs and does not seem active anymore.
+    add_licence_header(verbose_count, Path(configuration.get_value(ConfigurationVariable.PROJECT_ROOT)))
+
+
+def add_licence_header(verbose_count: int, src: Path) -> None:
     """Puts a copyright notice at the top of every source file.
 
     Wrapper over the [licenseheaders tool](https://github.com/johann-petrak/licenseheaders).
@@ -40,7 +50,7 @@ def add_licence_header(verbose_count: int) -> None:
         logger.debug(f"Creates template file in {str(template_file_path)}")
         template_file.write(template_string.encode("utf8"))
         template_file.close()
-        copyright_config = get_tool_config(template_file_path)
+        copyright_config = get_tool_config(template_file_path, src)
         _call_licensehearders(copyright_config, verbose_count)
 
 
@@ -65,12 +75,12 @@ def _to_copyright_date_string(start: int, current: int) -> str:
     return f"{current}" if current == start else f"{start}-{current}"
 
 
-def get_tool_config(template_file: Path) -> dict:
+def get_tool_config(template_file: Path, src: Path) -> dict:
     """Gets the configuration for licenseheaders."""
     copyright_dates = _determines_copyright_dates()
     return {
         "owner": configuration.get_value(ConfigurationVariable.ORGANISATION),
-        "dir": configuration.get_value(ConfigurationVariable.PROJECT_ROOT),
+        "dir": src,
         "projname": configuration.get_value(ConfigurationVariable.PROJECT_NAME),
         "tmpl": str(template_file),
         "years": copyright_dates,
@@ -86,7 +96,7 @@ def main() -> int:
     args = parser.parse_args()
     set_log_level(args.verbose)
     try:
-        add_licence_header(args.verbose)
+        insert_licence_header(args.verbose)
     except Exception as e:
         log_exception(logger, e)
         return 1

continuous_delivery_scripts/tag_and_release.py

@@ -13,11 +13,8 @@
 from continuous_delivery_scripts.generate_docs import generate_documentation
 from continuous_delivery_scripts.generate_news import version_project
 from continuous_delivery_scripts.language_specifics import get_language_specifics
-from continuous_delivery_scripts.license_files import add_licence_header
-from continuous_delivery_scripts.report_third_party_ip import (
-    generate_spdx_project_reports,
-    SpdxProject,
-)
+from continuous_delivery_scripts.license_files import insert_licence_header
+from continuous_delivery_scripts.report_third_party_ip import generate_spdx_project_reports, SpdxProject
 from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable
 from continuous_delivery_scripts.utils.definitions import CommitType
 from continuous_delivery_scripts.utils.git_helpers import ProjectTempClone, GitWrapper
@@ -50,7 +47,7 @@ def tag_and_release(mode: CommitType, current_branch: Optional[str] = None) -> N
     _update_documentation()
     # Adding the licensing summaries in /docs after folder has been cleared and regenerated.
     spdx_project = _update_licensing_summary()
-    add_licence_header(0)
+    insert_licence_header(0)
     _update_repository(mode, is_new_version, version, current_branch)
     if is_new_version:
         if spdx_project:

continuous_delivery_scripts/utils/configuration.py

@@ -13,6 +13,7 @@
 import toml
 
 from .filesystem_helpers import find_file_in_tree
+from .news_file import NewsType
 
 logger = logging.getLogger(__name__)
 
@@ -49,6 +50,10 @@ class ConfigurationVariable(enum.Enum):
     ACCEPTED_THIRD_PARTY_LICENCES = 28
     PACKAGES_WITH_CHECKED_LICENCE = 29
     PROGRAMMING_LANGUAGE = 30
+    AUTOGENERATE_NEWS_FILE_ON_DEPENDENCY_UPDATE = 31
+    DEPENDENCY_UPDATE_BRANCH_PATTERN = 32
+    DEPENDENCY_UPDATE_NEWS_MESSAGE = 33
+    DEPENDENCY_UPDATE_NEWS_TYPE = 34
 
     @staticmethod
     def choices() -> List[str]:
@@ -145,11 +150,16 @@ class StaticConfig(GenericConfig):
     LOGGER_FORMAT = "%(levelname)s: %(message)s"
     BOT_USERNAME = "Monty Bot"
     BOT_EMAIL = "[email protected]"
-    ORGANISATION = "Arm"
+    ORGANISATION = "Arm Limited"
     ORGANISATION_EMAIL = "[email protected]"
     FILE_LICENCE_IDENTIFIER = "Apache-2.0"
     COPYRIGHT_START_DATE = 2020
     PROGRAMMING_LANGUAGE = "NoOp"
+    AWS_BUCKET = "Unknown"
+    AUTOGENERATE_NEWS_FILE_ON_DEPENDENCY_UPDATE = True
+    DEPENDENCY_UPDATE_NEWS_MESSAGE = "Dependency upgrade: {message}"
+    DEPENDENCY_UPDATE_NEWS_TYPE = NewsType.bugfix.name
+    DEPENDENCY_UPDATE_BRANCH_PATTERN = r"^\s*[Dd]ependabot\/.+\/(?P<DEPENDENCY>.+)"
     ACCEPTED_THIRD_PARTY_LICENCES = ["Apache-2.0", "BSD*", "JSON", "MIT", "Python-2.0", "PSF-2.0", "MPL-2.0"]
     PACKAGES_WITH_CHECKED_LICENCE: List[str] = []
 

continuous_delivery_scripts/utils/git_helpers.py

@@ -189,10 +189,8 @@ def is_release_branch(self, branch_name: Optional[str]) -> bool:
             True if the branch is used for `release` code; False otherwise
         """
         branch_pattern = configuration.get_value(ConfigurationVariable.RELEASE_BRANCH_PATTERN)
-        if not branch_pattern or not branch_name:
-            return False
-        is_release: Optional[Any] = re.search(branch_pattern, str(branch_name))
-        return True if is_release else False
+        is_release, _ = self._is_branch_of_type(branch_name, branch_pattern)
+        return is_release
 
     def fetch(self) -> None:
         """Fetches latest changes."""
@@ -529,11 +527,23 @@ def _get_remote(self) -> Optional[Any]:
 
     def list_files_added_on_current_branch(self) -> List[str]:
         """Returns a list of files changed against master branch."""
-        master_branch_commit = self.repo.commit(self.get_master_branch())
+        master_branch = self.get_master_branch()
+        beta_branch = self.get_beta_branch()
+        master_branch_commit = self.repo.commit(master_branch)
+        beta_branch_commit = self.repo.commit(beta_branch)
         current_branch_commit = self.repo.commit(self.get_current_branch())
-        changes = self.get_changes_list(
-            self.get_branch_point(master_branch_commit, current_branch_commit), current_branch_commit, change_type="a"
-        )
+        # Finding the baseline branch to consider
+        master_branch_point = self.repo.commit(self.get_branch_point(master_branch_commit, current_branch_commit))
+        beta_branch_point = self.repo.commit(self.get_branch_point(beta_branch_commit, current_branch_commit))
+        branch_point = master_branch_point
+        if not master_branch:
+            branch_point = beta_branch_point
+        elif beta_branch and master_branch:
+            if beta_branch_point.committed_datetime > master_branch_point.committed_datetime:
+                # The branch point off `beta` is more recent than off `master`.
+                # Hence, the difference between current and `beta` should be considered.
+                branch_point = beta_branch_point
+        changes = self.get_changes_list(branch_point, current_branch_commit, change_type="a")
         return changes
 
     def is_current_branch_feature(self) -> bool:
@@ -544,6 +554,18 @@ def is_current_branch_feature(self) -> bool:
         is_release = self.is_release_branch(current_branch)
         return not (is_master or is_beta or is_release)
 
+    def is_current_branch_of_type(self, pattern: str) -> (bool, Optional[List[Any]]):
+        """Returns boolean indicating whether the current branch follows the pattern and the list of groups if any."""
+        return self._is_branch_of_type(self.get_current_branch(), pattern)
+
+    def _is_branch_of_type(self, branch_name: Optional[str], pattern: Optional[str]) -> (bool, Optional[List[Any]]):
+        if not pattern:
+            return False, None
+        if not branch_name:
+            return False, None
+        match = re.search(pattern, str(branch_name))
+        return True if match else False, match.groups() if match else None
+
     @property
     def uncommitted_changes(self) -> List[Path]:
         """Gets list of uncommitted files.

continuous_delivery_scripts/utils/language_specifics_base.py

@@ -20,7 +20,7 @@ def get_language_from_file_name(filename: str) -> str:
     return Path(filename).resolve().with_suffix("").name
 
 
-GENERIC_LICENCE_HEADER_TEMPLATE = """Copyright (C) {date} {author}. All rights reserved.
+GENERIC_LICENCE_HEADER_TEMPLATE = """Copyright (C) {date} {author} or its affiliates and Contributors. All rights reserved.
 SPDX-License-Identifier: {licence_identifier}
 """