feat: add possibility to create general html report from reports files

- Logic for create general html report from reports files;
- Add language setting;
- Extends examples of using the util in the README file;
- Test coverage of html report generation logic.

Refs: #179

Author: Artanias

Makefile

@@ -1,11 +1,11 @@
-UTIL_VERSION            := 0.3.7
+UTIL_VERSION            := 0.4.0
 UTIL_NAME               := codeplag
 PWD                     := $(shell pwd)
 
 USER_UID                ?= $(shell id --user)
 USER_GID                ?= $(shell id --group)
 
-BASE_DOCKER_VERSION     := 1.2
+BASE_DOCKER_VERSION     := 1.3
 BASE_DOCKER_TAG         := $(shell echo $(UTIL_NAME)-base-ubuntu20.04:$(BASE_DOCKER_VERSION) | tr A-Z a-z)
 TEST_DOCKER_TAG         := $(shell echo $(UTIL_NAME)-test-ubuntu20.04:$(UTIL_VERSION) | tr A-Z a-z)
 DOCKER_TAG              ?= $(shell echo $(UTIL_NAME)-ubuntu20.04:$(UTIL_VERSION) | tr A-Z a-z)
@@ -16,6 +16,7 @@ PYTHONPATH              := $(PWD)/src/:$(PWD)/test/auto
 LOGS_PATH               := /var/log/$(UTIL_NAME)
 CODEPLAG_LOG_PATH       := $(LOGS_PATH)/$(UTIL_NAME).log
 CONFIG_PATH             := /etc/$(UTIL_NAME)/settings.conf
+LIB_PATH                := /var/lib/$(UTIL_NAME)
 DEBIAN_PACKAGES_PATH    := debian/deb
 
 SOURCE_SUB_FILES        := src/$(UTIL_NAME)/consts.py
@@ -44,6 +45,7 @@ substitute = @sed \
 		-e "s|@DEVEL_SUFFIX@|${DEVEL_SUFFIX}|g" \
 		-e "s|@PYTHON_REQUIRED_LIBS@|${PYTHON_REQUIRED_LIBS}|g" \
 		-e "s|@LOGS_PATH@|${LOGS_PATH}|g" \
+		-e "s|@LIB_PATH@|${LIB_PATH}|g" \
 		-e "s|@CONFIG_PATH@|${CONFIG_PATH}|g" \
 		-e "s|@BASE_DOCKER_TAG@|${BASE_DOCKER_TAG}|g" \
 		-e "s|@DEBIAN_PACKAGES_PATH@|${DEBIAN_PACKAGES_PATH}|g" \
@@ -82,6 +84,10 @@ install: substitute-sources man
 
 	install -D -d -m 0755 $(DESTDIR)/$(LOGS_PATH)
 	install -D -m 0666 /dev/null $(DESTDIR)/$(CODEPLAG_LOG_PATH)
+	install -D -d -m 0755 $(DESTDIR)/$(LIB_PATH)
+
+	install -D -m 0666 src/templates/report_ru.templ $(DESTDIR)/$(LIB_PATH)/report_ru.templ
+	install -D -m 0666 src/templates/report_en.templ $(DESTDIR)/$(LIB_PATH)/report_en.templ
 
 	if [ ! -f $(DESTDIR)/$(CONFIG_PATH) ]; then \
 		install -D -d -m 0755 $(DESTDIR)/etc/$(UTIL_NAME); \
@@ -102,11 +108,11 @@ package: substitute-debian
 	)
 
 test: substitute-sources
-	pytest test/unit -q
+	pytest test/unit -vv
 	make clean-cache
 
 autotest:
-	pytest test/auto -q
+	pytest test/auto -vv
 	make clean-cache
 
 pre-commit:

README.md

@@ -118,7 +118,19 @@
 
 ## 4. Demo examples (works in the project directory and with an installed codeplag package)
 
-- Python analyzer
+- Show help: `$ codeplag --help`
+- Show help of subcommands (and further along the chain similarly): `$ codeplag check --help`
+- Setting up the util:
+  ```
+  # Setup check threshold to 70
+  # Language to English
+  # Show check progress
+  # Extension of reports 'csv'
+  # Reports path to '/usr/src/works'
+  # Path to environment variables '/usr/src/works/.env'
+  $ codeplag settings modify --threshold 70 --language en --show_progress 1 --reports_extension csv --reports /usr/src/works --environment /usr/src/works/.env
+  ```
+- Python analyzer:
   ```
   $ codeplag check --extension py --files src/codeplag/pyplag/astwalkers.py --directories src/codeplag/pyplag
   $ codeplag check --extension py --directories src/codeplag/algorithms src
@@ -129,11 +141,11 @@
   $ codeplag check --extension py --github-project-folders https://github.com/OSLL/code-plagiarism/blob/main/src/codeplag/pyplag --github-user OSLL --repo-regexp code-
   $ codeplag check --extension py --github-project-folders https://github.com/OSLL/code-plagiarism/blob/main/src/codeplag/pyplag --directories src/codeplag/pyplag/
   ```
-
-- C++/C analyzer
+- C++/C analyzer:
   ```
   $ codeplag check --extension cpp --directories src/codeplag/cplag/tests/data src/ --files test/codeplag/cplag/data/sample1.cpp test/codeplag/cplag/data/sample2.cpp
   $ codeplag check --extension cpp --github-files https://github.com/OSLL/code-plagiarism/blob/main/test/codeplag/cplag/data/sample3.cpp https://github.com/OSLL/code-plagiarism/blob/main/test/codeplag/cplag/data/sample4.cpp
   $ codeplag check --extension cpp --github-project-folders https://github.com/OSLL/code-plagiarism/tree/main/test
   $ codeplag check --extension cpp --github-user OSLL --repo-regexp "code-plag"
   ```
+- Create html report: `codeplag report create --path /usr/src/works`

setup.py

@@ -16,6 +16,7 @@
     "requests~=2.31.0",
     "typing-extensions~=4.3.0",
     "aiohttp~=3.8.5",
+    "Jinja2~=3.1.2",
     "cachetools==5.3.1",
     "gidgethub~=5.3.0",
 ]

src/codeplag/__init__.py

@@ -20,7 +20,7 @@ def main() -> Literal[0, 1, 2]:
     logger = get_logger(__name__, LOG_PATH, verbose=parsed_args["verbose"])
     codeplag_util = CodeplagEngine(logger, parsed_args)
     try:
-        codeplag_util.run()
+        code = codeplag_util.run()
     except KeyboardInterrupt:
         logger.warning("The util stopped by keyboard interrupt.")
         return 1
@@ -33,4 +33,4 @@ def main() -> Literal[0, 1, 2]:
         logger.debug("Trace:", exc_info=True)
         return 2
 
-    return 0
+    return code

src/codeplag/codeplagcli.py

@@ -9,7 +9,9 @@
 from webparsers.types import GitHubContentUrl
 
 from codeplag.consts import (
+    DEFAULT_GENERAL_REPORT_NAME,
     EXTENSION_CHOICE,
+    LANGUAGE_CHOICE,
     MODE_CHOICE,
     REPORTS_EXTENSION_CHOICE,
     UTIL_NAME,
@@ -67,44 +69,20 @@ def __new__(cls, *args, **kwargs):
 class CodeplagCLI(argparse.ArgumentParser):
     """The argument parser of the codeplag util."""
 
-    def __init__(self):
-        super(CodeplagCLI, self).__init__(
-            prog=UTIL_NAME,
-            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-            description="Program help to find similar parts of source "
-            "codes for the different languages.",
-        )
-        self.add_argument(
-            "-v",
-            "--version",
-            help="Print current version number and exit.",
-            action="version",
-            version=f"{UTIL_NAME} {UTIL_VERSION}",
-        )
-        self.add_argument(
-            "--verbose",
-            help="Show debug messages.",
-            action="store_true",
-        )
-
-        subparsers = self.add_subparsers(
-            help="Commands help.",
-            parser_class=argparse.ArgumentParser,
-            required=True,
-            metavar="COMMAND",
-            dest="root",
-        )
-
+    def __add_settings_path(self, subparsers: argparse._SubParsersAction) -> None:
         settings = subparsers.add_parser(
             "settings",
             help=f"Modifies and shows static settings of the '{UTIL_NAME}' util.",
         )
+
         settings_commands = settings.add_subparsers(
             help=f"Settings commands of the '{UTIL_NAME}' util.",
             required=True,
             metavar="COMMAND",
             dest="settings",
         )
+
+        # settings modify
         settings_modify = settings_commands.add_parser(
             "modify",
             help=f"Manage the '{UTIL_NAME}' util settings.",
@@ -149,12 +127,21 @@ def __init__(self):
             choices=range(50, 100),
             metavar="{50, 51, ..., 99}",
         )
+        settings_modify.add_argument(
+            "-l",
+            "--language",
+            help="The language of help messages, generated reports, errors.",
+            type=str,
+            choices=LANGUAGE_CHOICE,
+        )
 
+        # settings show
         settings_commands.add_parser(
             "show",
             help=f"Show the '{UTIL_NAME}' util settings.",
         )
 
+    def __add_check_path(self, subparsers: argparse._SubParsersAction) -> None:
         check = subparsers.add_parser("check", help="Start searching similar works.")
         check.add_argument(
             "-d",
@@ -239,3 +226,64 @@ def __init__(self):
             action=CheckUniqueStore,
             default=[],
         )
+
+    def __add_report_path(self, subparsers: argparse._SubParsersAction) -> None:
+        report = subparsers.add_parser(
+            "report",
+            help=f"Handling generated by the {UTIL_NAME} reports as creating html "
+            "report file or show it on console.",
+        )
+
+        report_commands = report.add_subparsers(
+            help=f"Report commands of the '{UTIL_NAME}' util.",
+            required=True,
+            metavar="COMMAND",
+            dest="report",
+        )
+
+        # report create
+        report_create = report_commands.add_parser(
+            "create",
+            help="Generate general report from created some time ago report files.",
+        )
+        report_create.add_argument(
+            "-p",
+            "--path",
+            help="Path to save generated general report. "
+            "If it's directory, than creates file in it with "
+            f"name '{DEFAULT_GENERAL_REPORT_NAME}'.",
+            required=True,
+            type=Path,
+        )
+
+    def __init__(self):
+        super(CodeplagCLI, self).__init__(
+            prog=UTIL_NAME,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            description="Program help to find similar parts of source "
+            "codes for the different languages.",
+        )
+        self.add_argument(
+            "-v",
+            "--version",
+            help="Print current version number and exit.",
+            action="version",
+            version=f"{UTIL_NAME} {UTIL_VERSION}",
+        )
+        self.add_argument(
+            "--verbose",
+            help="Show debug messages.",
+            action="store_true",
+        )
+
+        subparsers = self.add_subparsers(
+            help="Commands help.",
+            parser_class=argparse.ArgumentParser,
+            required=True,
+            metavar="COMMAND",
+            dest="root",
+        )
+
+        self.__add_settings_path(subparsers)
+        self.__add_check_path(subparsers)
+        self.__add_report_path(subparsers)

src/codeplag/config.py

@@ -5,7 +5,12 @@
 
 from typing_extensions import NotRequired
 
-from codeplag.consts import CONFIG_PATH, DEFAULT_THRESHOLD
+from codeplag.consts import (
+    CONFIG_PATH,
+    DEFAULT_LANGUAGE,
+    DEFAULT_REPORT_EXTENSION,
+    DEFAULT_THRESHOLD,
+)
 from codeplag.types import Settings
 
 
@@ -74,5 +79,8 @@ def write_settings_conf(settings: Settings) -> None:
 
 
 DefaultSettingsConfig = Settings(
-    threshold=DEFAULT_THRESHOLD, show_progress=0, reports_extension="csv"
+    threshold=DEFAULT_THRESHOLD,
+    show_progress=0,
+    reports_extension=DEFAULT_REPORT_EXTENSION,
+    language=DEFAULT_LANGUAGE,
 )

src/codeplag/consts.tmp.py

@@ -2,7 +2,14 @@
 from pathlib import Path
 from typing import Dict, Final, List, Tuple
 
-from codeplag.types import Extension, Extensions, Mode, ReportsExtension, Threshold
+from codeplag.types import (
+    Extension,
+    Extensions,
+    Language,
+    Mode,
+    ReportsExtension,
+    Threshold,
+)
 
 UTIL_NAME: Final[str] = "@UTIL_NAME@"
 UTIL_VERSION: Final[str] = "@UTIL_VERSION@"
@@ -11,6 +18,17 @@
 CONFIG_PATH: Final[Path] = Path("@CONFIG_PATH@")
 FILE_DOWNLOAD_PATH: Final[Path] = Path(f"/tmp/{UTIL_NAME}_download.out")
 LOG_PATH: Final[Path] = Path("@CODEPLAG_LOG_PATH@")
+TEMPLATE_PATH: Final[Dict[Language, Path]] = {
+    "ru": Path("@LIB_PATH@/report_ru.templ"),
+    "en": Path("@LIB_PATH@/report_en.templ"),
+}
+
+# Default values
+DEFAULT_THRESHOLD: Final[Threshold] = 65
+DEFAULT_WEIGHTS: Final[Tuple[float, float, float, float]] = (1.0, 0.4, 0.4, 0.4)
+DEFAULT_LANGUAGE: Final[Language] = "en"
+DEFAULT_REPORT_EXTENSION: Final[ReportsExtension] = "csv"
+DEFAULT_GENERAL_REPORT_NAME: Final[str] = "report.html"
 
 GET_FRAZE: Final[str] = "Getting works features from"
 
@@ -34,10 +52,10 @@
     "compliance_matrix",
 )
 
-DEFAULT_THRESHOLD: Final[Threshold] = 65
 MODE_CHOICE: Final[List[Mode]] = ["many_to_many", "one_to_one"]
 REPORTS_EXTENSION_CHOICE: Final[List[ReportsExtension]] = ["csv", "json"]
 EXTENSION_CHOICE: Final[List[Extension]] = ["py", "cpp"]
+LANGUAGE_CHOICE: Final[List[Language]] = ["en", "ru"]
 ALL_EXTENSIONS: Final[Tuple[re.Pattern]] = (re.compile(r"\..*$"),)
 # Don't  checks changing values by key
 SUPPORTED_EXTENSIONS: Final[Dict[Extension, Extensions]] = {

src/codeplag/display.py

@@ -1,9 +1,8 @@
 import sys
 from enum import Enum
 from functools import partial
-from typing import List
+from typing import List, Optional
 
-import numpy as np
 import pandas as pd
 
 from codeplag.types import ASTFeatures, CompareInfo, NodeCodePlace
@@ -86,7 +85,7 @@ def print_compare_result(
     features1: ASTFeatures,
     features2: ASTFeatures,
     compare_info: CompareInfo,
-    threshold: int = 60,
+    compliance_matrix_df: Optional[pd.DataFrame] = None,
 ) -> None:
     """The function prints the result of comparing two files
 
@@ -131,24 +130,7 @@ def print_compare_result(
     print(additional_metrics_df)
     print()
 
-    if (compare_info.structure.similarity * 100) > threshold:
-        data = np.zeros(
-            (
-                compare_info.structure.compliance_matrix.shape[0],
-                compare_info.structure.compliance_matrix.shape[1],
-            ),
-            dtype=np.float64,
-        )
-        for row in range(compare_info.structure.compliance_matrix.shape[0]):
-            for col in range(compare_info.structure.compliance_matrix.shape[1]):
-                data[row][col] = (
-                    compare_info.structure.compliance_matrix[row][col][0]
-                    / compare_info.structure.compliance_matrix[row][col][1]
-                )
-        compliance_matrix_df = pd.DataFrame(
-            data=data, index=features1.head_nodes, columns=features2.head_nodes
-        )
-
+    if compliance_matrix_df is not None:
         print(compliance_matrix_df, "\n")
 
     print("+" * 40)