feat: show commit date into check reports and add async GitHub parser

- Add showing commit date when comparing GitHub works;
- Use sessions for interactions between GitHub;
- Some logic optimizations and add async github parser module.

Author: Artanias

Makefile

@@ -1,8 +1,8 @@
-UTIL_VERSION            := 0.3.4
+UTIL_VERSION            := 0.3.5
 UTIL_NAME               := codeplag
 PWD                     := $(shell pwd)
 
-BASE_DOCKER_VERSION     := 1.1
+BASE_DOCKER_VERSION     := 1.2
 BASE_DOCKER_TAG         := $(shell echo $(UTIL_NAME)-base-ubuntu20.04:$(BASE_DOCKER_VERSION) | tr A-Z a-z)
 TEST_DOCKER_TAG         := $(shell echo $(UTIL_NAME)-test-ubuntu20.04:$(UTIL_VERSION) | tr A-Z a-z)
 DOCKER_TAG              ?= $(shell echo $(UTIL_NAME)-ubuntu20.04:$(UTIL_VERSION) | tr A-Z a-z)

docker.mk

@@ -21,11 +21,16 @@ docker-test: docker-test-image
 		"$(TEST_DOCKER_TAG)"
 
 docker-autotest: docker-test-image
-	docker run --rm \
-		--volume $(PWD)/test:/usr/src/$(UTIL_NAME)/test \
-		--env-file .env \
-		"$(TEST_DOCKER_TAG)" bash -c \
-		"make && make autotest"
+	@if [ $(shell find . -maxdepth 1 -type f -name .env | wc --lines) != 1 ]; then \
+		echo "Requires '.env' file with provided GitHub token for running autotests."; \
+		exit 200; \
+	else \
+		docker run --rm \
+			--volume $(PWD)/test:/usr/src/$(UTIL_NAME)/test \
+			--env-file .env \
+			"$(TEST_DOCKER_TAG)" bash -c \
+			"make && make autotest"; \
+	fi
 
 docker-build-package: docker-test-image
 	docker run --rm \
@@ -50,7 +55,9 @@ docker-image: docker-base-image docker-test-image
 	)
 
 docker-run: docker-image
+	@touch .env
 	docker run --rm --tty --interactive \
+		--env-file .env \
 		"$(DOCKER_TAG)"
 
 docker-rmi:

setup.py

@@ -15,6 +15,9 @@
     'python-decouple~=3.6',
     'requests~=2.31.0',
     'typing-extensions~=4.3.0',
+    'aiohttp~=3.8.5',
+    'cachetools==5.3.1',
+    'gidgethub~=5.3.0',
 ]
 UTIL_NAME = os.getenv('UTIL_NAME')
 UTIL_VERSION = os.getenv('UTIL_VERSION')

src/codeplag/cplag/utils.py

@@ -11,6 +11,7 @@
 from codeplag.display import eprint
 from codeplag.getfeatures import AbstractGetter, get_files_path_from_directory
 from codeplag.types import ASTFeatures
+from webparsers.types import WorkInfo
 
 
 def get_cursor_from_file(filepath: Path,
@@ -83,20 +84,21 @@ def __init__(
             path_regexp=path_regexp
         )
 
-    def get_from_content(self, file_content: str, url_to_file: str) -> Optional[ASTFeatures]:
+    def get_from_content(self, work_info: WorkInfo) -> Optional[ASTFeatures]:
         with open(FILE_DOWNLOAD_PATH, 'w', encoding='utf-8') as out_file:
-            out_file.write(file_content)
+            out_file.write(work_info.code)
         cursor = get_cursor_from_file(FILE_DOWNLOAD_PATH, COMPILE_ARGS)
-        if not cursor:
+        if cursor is None:
             self.logger.error(
-                "Unsuccessfully attempt to get AST from the file %s.", url_to_file
+                "Unsuccessfully attempt to get AST from the file %s.", work_info.link
             )
             return
 
         # hook for correct filtering info while parsing source code
         features = get_features(cursor, FILE_DOWNLOAD_PATH)
         os.remove(FILE_DOWNLOAD_PATH)
-        features.filepath = url_to_file
+        features.filepath = work_info.link
+        features.modify_date = work_info.commit.date
 
         return features
 

src/codeplag/display.py

@@ -99,12 +99,21 @@ def print_compare_result(features1: ASTFeatures,
 
     print(" " * 40)
     print('+' * 40)
-    print(
-        'May be similar:',
-        features1.filepath,
-        features2.filepath,
-        end='\n\n', sep='\n'
-    )
+    if features1.modify_date is not None and features2.modify_date is not None:
+        message = (
+            '-----\n'
+            f'{features1.filepath}\n{features1.modify_date}\n'
+            '-----\n'
+            f'{features2.filepath}\n{features2.modify_date}\n'
+            '-----\n'
+        )
+    else:
+        message = (
+            f'{features1.filepath}\n'
+            f'{features2.filepath}\n'
+        )
+
+    print('May be similar:', message, end='\n\n', sep='\n')
     main_metrics_df = pd.DataFrame(
         [compare_info.fast], index=['Similarity'],
         columns=pd.Index(

src/codeplag/getfeatures.py

@@ -7,6 +7,7 @@
 from typing import List, Literal, Optional, Union, overload
 
 from decouple import Config, RepositoryEnv
+from requests import Session
 
 from codeplag.consts import (
     ALL_EXTENSIONS,
@@ -18,6 +19,7 @@
 from codeplag.logger import get_logger
 from codeplag.types import ASTFeatures, Extension, Extensions
 from webparsers.github_parser import GitHubParser
+from webparsers.types import WorkInfo
 
 
 def get_files_path_from_directory(
@@ -101,7 +103,7 @@ def _set_access_token(self, env_path: Optional[Path]) -> None:
             self._access_token: str = os.environ.get('ACCESS_TOKEN', '')
         else:
             env_config = Config(RepositoryEnv(env_path))
-            self._access_token: str = env_config.get('ACCESS_TOKEN', default='')
+            self._access_token: str = env_config.get('ACCESS_TOKEN', default='')  # type: ignore
 
         if not self._access_token:
             self.logger.warning('GitHub access token is not defined.')
@@ -111,11 +113,12 @@ def _set_github_parser(self, branch_policy: bool) -> None:
             file_extensions=SUPPORTED_EXTENSIONS[self.extension],
             check_all=branch_policy,
             access_token=self._access_token,
-            logger=get_logger('webparsers', LOG_PATH)
+            logger=get_logger('webparsers', LOG_PATH),
+            session=Session()
         )
 
     @abstractmethod
-    def get_from_content(self, file_content: str, url_to_file: str) -> Optional[ASTFeatures]:
+    def get_from_content(self, work_info: WorkInfo) -> Optional[ASTFeatures]:
         ...
 
     @abstractmethod
@@ -165,8 +168,8 @@ def get_from_github_files(self, github_files: List[str]) -> List[ASTFeatures]:
 
         self.logger.debug(f"{GET_FRAZE} GitHub urls")
         for github_file in github_files:
-            file_content = self.github_parser.get_file_from_url(github_file)[0]
-            features = self.get_from_content(file_content, github_file)
+            work_info = self.github_parser.get_file_from_url(github_file)
+            features = self.get_from_content(work_info)
             if features:
                 works.append(features)
 
@@ -200,8 +203,8 @@ def get_from_github_project_folders(
             gh_prj_files = self.github_parser.get_files_generator_from_dir_url(
                 github_project, path_regexp=self.path_regexp
             )
-            for file_content, url_file in gh_prj_files:
-                features = self.get_from_content(file_content, url_file)
+            for work_info in gh_prj_files:
+                features = self.get_from_content(work_info)
                 if features is None:
                     continue
 
@@ -251,8 +254,8 @@ def get_from_users_repos(
             files = self.github_parser.get_files_generator_from_repo_url(
                 repo.html_url, path_regexp=self.path_regexp
             )
-            for file_content, url_file in files:
-                features = self.get_from_content(file_content, url_file)
+            for work_info in files:
+                features = self.get_from_content(work_info)
                 if features is None:
                     continue
 

src/codeplag/pyplag/utils.py

@@ -9,6 +9,7 @@
 from codeplag.logger import get_logger
 from codeplag.pyplag.astwalkers import ASTWalker
 from codeplag.types import ASTFeatures
+from webparsers.types import WorkInfo
 
 # TODO: Remove from globals
 logger = get_logger(__name__, LOG_PATH)
@@ -130,15 +131,16 @@ def __init__(
             path_regexp=path_regexp
         )
 
-    def get_from_content(self, file_content: str, url_to_file: str) -> Optional[ASTFeatures]:
-        tree = get_ast_from_content(file_content, url_to_file)
+    def get_from_content(self, work_info: WorkInfo) -> Optional[ASTFeatures]:
+        tree = get_ast_from_content(work_info.code, work_info.link)
         if tree is not None:
-            return get_features_from_ast(tree, url_to_file)
+            features = get_features_from_ast(tree, work_info.link)
+            features.modify_date = work_info.commit.date
+            return features
 
         self.logger.error(
-            "Unsuccessfully attempt to get AST from the file %s.", url_to_file
+            "Unsuccessfully attempt to get AST from the file %s.", work_info.link
         )
-        return
 
     def get_from_files(self, files: List[Path]) -> List[ASTFeatures]:
         if not files:

src/codeplag/types.py

@@ -48,6 +48,7 @@ class ASTFeatures:
     """Class contains the source code metadata."""
 
     filepath: Union[Path, str]
+    modify_date: Optional[str] = None
 
     count_of_nodes: int = 0
     head_nodes: List[str] = field(default_factory=list)
@@ -86,10 +87,13 @@ class CompareInfo(NamedTuple):
     structure: Optional[StructuresInfo] = None
 
 
+# TODO: Rework it structure
 class WorksReport(TypedDict):
     date: str
     first_path: str
     second_path: str
+    first_modify_date: NotRequired[str]
+    second_modify_date: NotRequired[str]
     first_heads: List[str]
     second_heads: List[str]
     fast: Dict[str, int]  # dict from FastMetrics

src/codeplag/utils.py

@@ -207,6 +207,10 @@ def save_result(self,
             fast=fast_metrics._asdict(),
             structure=struct_info_dict
         )
+        if first_work.modify_date:
+            report["first_modify_date"] = first_work.modify_date
+        if second_work.modify_date:
+            report["second_modify_date"] = second_work.modify_date
 
         try:
             report_file = self.reports / f'{uuid.uuid4().hex}.json'