diff --git a/README.md b/README.md index d1b28d2..1ee0cbb 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,9 @@ The config file needs to specify the following sections and parameters: - `path`: `organization/repository` part of GitHub URL - `url`: the full URL of the repository to clone from (may be local path) - `local_path`: local directory to clone the repository to (without the repository name itself) -- Technology profiles: same as in [example config](config/config.ini) -- DFD: empty section + - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` - Analysis Settings (optional) - `development_mode`: boolean, turns on development mode - - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD` It is possible to provide these parameters also by command line, see `python3 code2DFD.py --help` for exact usage diff --git a/code2DFD.py b/code2DFD.py index b2085a9..23486da 100644 --- a/code2DFD.py +++ b/code2DFD.py @@ -3,24 +3,12 @@ # Author: Simon Schneider, 2023 # Contact: simon.schneider@tuhh.de import os -from configparser import ConfigParser from datetime import datetime import argparse from core.dfd_extraction import perform_analysis from output_generators.logger import logger -import tmp.tmp as tmp - -CONFIG_SECTIONS = ["Analysis Settings", "Repository", "Technology Profiles", "DFD"] -COMMUNICATIONS_TECH_LIST = '[("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"),\ - ("FeignClient", "fgn"), ("Implicit Connections", "imp"),\ - ("Database Connections", "dbc"), ("HTML", "html"),\ - ("Docker-Compose", "dcm")]' -DEFAULT_CONFIG = ConfigParser() -for section in CONFIG_SECTIONS: - DEFAULT_CONFIG.add_section(section) -DEFAULT_CONFIG.set("Analysis Settings", "development_mode", "False") -DEFAULT_CONFIG.set("Technology Profiles", "communication_techs_list", COMMUNICATIONS_TECH_LIST) +from core.config import code2dfd_config def api_invocation(url: str, commit: str) -> dict: @@ -32,18 +20,14 @@ def api_invocation(url: str, commit: str) -> dict: start_time = datetime.now() logger.info("*** New execution ***") - logger.debug("Initializing config to tmp file") - for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.tmp_config.add_section(section) - for entry in DEFAULT_CONFIG[section]: - tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry]) + logger.debug("Initializing config") # Overwrite repo_path from config file with the one from the API call - tmp.tmp_config.set("Repository", "url", url) - tmp.tmp_config.set("Repository", "local_path", - os.path.join(os.getcwd(), "analysed_repositories")) + code2dfd_config.set("Repository", "url", url) + code2dfd_config.set("Repository", "local_path", + os.path.join(os.getcwd(), "analysed_repositories")) if commit is not None: - tmp.tmp_config.set("Analysis Settings", "commit", commit) + code2dfd_config.set("Repository", "commit", commit) # Call extraction codeable_models, traceability = perform_analysis() @@ -69,8 +53,8 @@ def cli_invocation(): repository.add_argument("--repo_url", type=str, help="URL to clone the repository from (might be local path)") repository.add_argument("--repo_local_path", type=str, help="Location to clone repository to (default: 'analysed_repositories' in CWD)") repository.add_argument("--github_handle", type=str, help="Handle of a GitHub repository containing the application to be analyzed") + repository.add_argument("--commit", type=str, help="Analyze repository at this commit") settings = parser.add_argument_group("Analysis Settings", "Parameters for additional analysis settings") - settings.add_argument("--commit", type=str, help="Analyze repository at this commit") settings.add_argument("--development_mode", action='store_true', help="Switch on development mode") args = parser.parse_args() @@ -79,37 +63,31 @@ def cli_invocation(): if args.config_path: # Copy config to tmp file - logger.debug("Copying config file to tmp file") - tmp.tmp_config.read(args.config_path) - else: + logger.debug("Copying config file to internal config") + code2dfd_config.read(args.config_path) # global ini_config - logger.debug("Initializing tmp file with default config") - for section in CONFIG_SECTIONS: # Copying what is needed from default to temp - tmp.tmp_config.add_section(section) - for entry in DEFAULT_CONFIG[section]: - tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry]) if args.repo_url: - tmp.tmp_config.set("Repository", "url", args.repo_url) + code2dfd_config.set("Repository", "url", args.repo_url) elif args.github_handle: - tmp.tmp_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") - elif not tmp.tmp_config.has_option("Repository", "url"): + code2dfd_config.set("Repository", "url", f"https://github.com/{args.github_handle.strip('/')}") + elif not code2dfd_config.has_option("Repository", "url"): raise AttributeError("Parameter [Repository][url] must be provided either in config file or by --repo_url") if args.repo_local_path: - tmp.tmp_config.set("Repository", "local_path", args.local_path) - elif not tmp.tmp_config.has_option("Repository", "local_path"): - tmp.tmp_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) + code2dfd_config.set("Repository", "local_path", args.local_path) + elif not code2dfd_config.has_option("Repository", "local_path"): + code2dfd_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories")) if args.development_mode: - tmp.tmp_config.set("Analysis Settings", "development_mode", "True") + code2dfd_config.set("Analysis Settings", "development_mode", "True") if args.commit is not None: commit = args.commit[:7] - tmp.tmp_config.set("Analysis Settings", "commit", commit) - elif tmp.tmp_config.has_option("Analysis Settings", "commit"): - commit = tmp.tmp_config.get("Analysis Settings", "commit")[:7] - tmp.tmp_config.set("Analysis Settings", "commit", commit) + code2dfd_config.set("Repository", "commit", commit) + elif code2dfd_config.has_option("Repository", "commit"): + commit = code2dfd_config.get("Repository", "commit")[:7] + code2dfd_config.set("Repository", "commit", commit) perform_analysis() diff --git a/config/config.ini b/config/config.ini index 6ad38ed..c93564d 100644 --- a/config/config.ini +++ b/config/config.ini @@ -1,5 +1,6 @@ [Repository] url = https://github.com/apssouza22/java-microservice +commit = 056414c4c938e536f467a3f37532194b860d96a3 ;url = https://github.com/callistaenterprise/blog-microservices ;url = https://github.com/fernandoabcampos/spring-netflix-oss-microservices ;url = https://github.com/georgwittberger/apache-spring-boot-microservice-example @@ -17,12 +18,5 @@ url = https://github.com/apssouza22/java-microservice ;url = https://github.com/rohitghatol/spring-boot-microservices ;url = https://github.com/yidongnan/spring-cloud-netflix-example -[Technology Profiles] -communication_techs_list = [("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"), ("FeignClient", "fgn"), ("Implicit Connections", "imp"), ("Database Connections", "dbc"), ("HTML", "html"), ("Docker-Compose", "dcm")] - [Analysis Settings] development_mode = False -; commit for apssouza22/java-microservice -;commit = 056414c4c938e536f467a3f37532194b860d96a3 - -[DFD] diff --git a/core/config.py b/core/config.py new file mode 100644 index 0000000..2b709de --- /dev/null +++ b/core/config.py @@ -0,0 +1,8 @@ +from configparser import ConfigParser + +code2dfd_config = ConfigParser() + +CONFIG_SECTIONS = ["Analysis Settings", "Repository"] +for section in CONFIG_SECTIONS: + code2dfd_config.add_section(section) +code2dfd_config.set("Analysis Settings", "development_mode", "False") diff --git a/core/dfd_extraction.py b/core/dfd_extraction.py index fe0d9a9..557770d 100644 --- a/core/dfd_extraction.py +++ b/core/dfd_extraction.py @@ -1,4 +1,3 @@ -import ast from datetime import datetime from itertools import combinations import os @@ -7,7 +6,7 @@ import output_generators.codeable_model as codeable_model import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.json_architecture as json_architecture import output_generators.json_edges as json_edges import output_generators.traceability as traceability @@ -48,27 +47,25 @@ from technology_specific_extractors.zookeeper.zoo_entry import detect_zookeeper from technology_specific_extractors.zuul.zul_entry import detect_zuul -from core.DFD import CDFD - def perform_analysis(): """ Entrypoint for the DFD extraction that initializes the repository """ - local_path = tmp.tmp_config.get("Repository", "local_path") - url_path = tmp.tmp_config.get("Repository", "url") + local_path = code2dfd_config.get("Repository", "local_path") + url_path = code2dfd_config.get("Repository", "url") os.makedirs(local_path, exist_ok=True) repository = Repository(path_to_repo=url_path, clone_repo_to=local_path) with repository._prep_repo(url_path) as git_repo: - tmp.tmp_config.set("Repository", "local_path", str(git_repo.path)) + code2dfd_config.set("Repository", "local_path", str(git_repo.path)) head = git_repo.get_head().hash[:7] - if tmp.tmp_config.has_option("Analysis Settings", "commit"): - commit = tmp.tmp_config.get("Analysis Settings", "commit") + if code2dfd_config.has_option("Repository", "commit"): + commit = code2dfd_config.get("Repository", "commit") else: commit = head repo_name = git_repo.project_name - tmp.tmp_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) + code2dfd_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit)) git_repo.checkout(commit) print(f"\nStart extraction of DFD for {repo_name} on commit {commit} at {datetime.now().strftime('%H:%M:%S')}") codeable_models, traceability_content = DFD_extraction() @@ -82,57 +79,48 @@ def perform_analysis(): def DFD_extraction(): """Main function for the extraction, calling all technology-specific extractors, managing output etc. """ - dfd = CDFD("TestDFD") - - microservices, information_flows, external_components = dict(), dict(), dict() - - microservices = tech_sw.get_microservices(dfd) + dfd = { + "microservices": dict(), + "information_flows": dict(), + "external_components": dict() + } + + print("Extracting microservices") + tech_sw.set_microservices(dfd) - microservices = detect_databases(microservices) - microservices = overwrite_port(microservices) - microservices = detect_ssl_services(microservices) + detect_databases(dfd) + overwrite_port(dfd) + detect_ssl_services(dfd) print("Extracted services from build- and IaC-files") # Parse internal and external configuration files - microservices, information_flows, external_components = detect_spring_config(microservices, information_flows, external_components, dfd) - microservices = detect_eureka_server_only(microservices, dfd) - microservices = overwrite_port(microservices) + detect_spring_config(dfd) + detect_eureka_server_only(dfd) + overwrite_port(dfd) # Classify brokers (needed for information flows) - microservices = classify_brokers(microservices) + classify_brokers(dfd) # Check authentication information of services - microservices = detect_authentication_scopes(microservices, dfd) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + detect_authentication_scopes(dfd) # Get information flows - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - - new_information_flows = tech_sw.get_information_flows(dfd) - external_components = ast.literal_eval(tmp.tmp_config["DFD"]["external_components"]) - - # Merge old and new - for new_flow in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = dict() - information_flows[id] = new_information_flows[new_flow] + print("Extracting information flows") + tech_sw.set_information_flows(dfd) + print("Extracted information flows from API-calls, message brokers, and database connections") # Detect everything else / execute all technology implementations print("Classifying all services") - microservices = tech_sw.get_microservices(dfd) - microservices, information_flows, external_components = classify_microservices(microservices, information_flows, external_components, dfd) + classify_microservices(dfd) # Merging print("Merging duplicate items") - merge_duplicate_flows(information_flows) + information_flows = merge_duplicate_flows(dfd["information_flows"]) - merge_duplicate_nodes(microservices) - merge_duplicate_nodes(external_components) + microservices = merge_duplicate_nodes(dfd["microservices"]) + external_components = merge_duplicate_nodes(dfd["external_components"]) merge_duplicate_annotations(microservices) merge_duplicate_annotations(information_flows) @@ -140,80 +128,76 @@ def DFD_extraction(): print("Cleaning database connections") - clean_database_connections(microservices, information_flows) + dfd = { + "microservices": microservices, + "information_flows": information_flows, + "external_components": external_components + } + + clean_database_connections(dfd) # Printing print("\nFinished extraction") # Saving - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - - plaintext.write_plaintext(microservices, information_flows, external_components) - codeable_models, codeable_models_path = codeable_model.output_codeable_model(microservices, information_flows, external_components) + plaintext.write_plaintext(dfd) + codeable_models, codeable_models_path = codeable_model.output_codeable_model(dfd) traceability_content = traceability.output_traceability() visualizer.output_png(codeable_models_path) - json_edges.generate_json_edges(information_flows) - json_architecture.generate_json_architecture(microservices, information_flows, external_components) - - #calculate_metrics.calculate_single_system(repo_path) - - #check_traceability.check_traceability(microservices, information_flows, external_components, traceability_content) + json_edges.generate_json_edges(dfd) + json_architecture.generate_json_architecture(dfd) return codeable_models, traceability_content -def classify_brokers(microservices: dict) -> dict: +def classify_brokers(dfd: dict): """Classifies kafka and rabbitmq servers, because they are needed for the information flows. """ - microservices = detect_rabbitmq_server(microservices) - microservices = detect_kafka_server(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - return microservices + detect_rabbitmq_server(dfd) + detect_kafka_server(dfd) -def classify_microservices(microservices: dict, information_flows: dict, external_components: dict, dfd) -> tuple[dict, dict, dict]: +def classify_microservices(dfd): """Tries to determine the microservice's funcitonality. """ - microservices, information_flows = detect_eureka(microservices, information_flows, dfd) - microservices, information_flows, external_components = detect_zuul(microservices, information_flows, external_components, dfd) - microservices, information_flows, external_components = detect_spring_cloud_gateway(microservices, information_flows, external_components, dfd) - microservices, information_flows = detect_spring_oauth(microservices, information_flows, dfd) - microservices, information_flows = detect_consul(microservices, information_flows, dfd) - microservices, information_flows = detect_hystrix_dashboard(microservices, information_flows, dfd) - microservices, information_flows = detect_turbine(microservices, information_flows, dfd) - microservices, information_flows = detect_local_logging(microservices, information_flows, dfd) - microservices, information_flows = detect_zipkin_server(microservices, information_flows, dfd) - microservices, information_flows = detect_spring_admin_server(microservices, information_flows, dfd) - microservices, information_flows = detect_prometheus_server(microservices, information_flows, dfd) - microservices, information_flows = detect_circuit_breakers(microservices, information_flows, dfd) - microservices, information_flows = detect_load_balancers(microservices, information_flows, dfd) - microservices, information_flows = detect_ribbon_load_balancers(microservices, information_flows, dfd) - microservices, information_flows = detect_hystrix_circuit_breakers(microservices, information_flows, dfd) - microservices, information_flows = detect_zookeeper(microservices, information_flows, dfd) - microservices, information_flows = detect_kibana(microservices, information_flows, dfd) - microservices, information_flows = detect_elasticsearch(microservices, information_flows, dfd) - microservices, information_flows, external_components = detect_logstash(microservices, information_flows, external_components, dfd) - microservices, information_flows, external_components = detect_nginx(microservices, information_flows, external_components, dfd) - microservices, information_flows = detect_grafana(microservices, information_flows, dfd) - microservices, information_flows = detect_spring_encryption(microservices, information_flows, dfd) - microservices = detect_endpoints(microservices, dfd) - - microservices, information_flows, external_components = detect_miscellaneous(microservices, information_flows, external_components) - microservices, information_flows, external_components = detect_apachehttpd_webserver(microservices, information_flows, external_components, dfd) - microservices = classify_internal_infrastructural(microservices) - microservices = set_plaintext_credentials(microservices) - - return microservices, information_flows, external_components - - -def overwrite_port(microservices: dict) -> dict: + detect_eureka(dfd) + detect_zuul(dfd) + detect_spring_cloud_gateway(dfd) + detect_spring_oauth(dfd) + detect_consul(dfd) + detect_hystrix_dashboard(dfd) + detect_turbine(dfd) + detect_local_logging(dfd) + detect_zipkin_server(dfd) + detect_spring_admin_server(dfd) + detect_prometheus_server(dfd) + detect_circuit_breakers(dfd) + detect_load_balancers(dfd) + detect_ribbon_load_balancers(dfd) + detect_hystrix_circuit_breakers(dfd) + detect_zookeeper(dfd) + detect_kibana(dfd) + detect_elasticsearch(dfd) + detect_logstash(dfd) + detect_nginx(dfd) + detect_grafana(dfd) + detect_spring_encryption(dfd) + detect_endpoints(dfd) + + detect_miscellaneous(dfd) + detect_apachehttpd_webserver(dfd) + classify_internal_infrastructural(dfd) + set_plaintext_credentials(dfd) + + +def overwrite_port(dfd: dict): """Writes port from properties to tagged vallues. """ + microservices = dfd["microservices"] + for microservice in microservices.values(): for prop in microservice.get("properties", []): if prop[0] == "port": @@ -240,13 +224,17 @@ def overwrite_port(microservices: dict) -> dict: traceability.add_trace(trace) microservice["tagged_values"] = microservice.get("tagged_values", list()) + [("Port", port)] - return microservices + dfd["microservices"] = microservices -def detect_miscellaneous(microservices: dict, information_flows: dict, external_components: dict) -> tuple[dict, dict, dict]: +def detect_miscellaneous(dfd: dict): """Goes through properties extracted for each service to check for some things that don't fit anywhere else (mail servers, external websites, etc.). """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + for microservice in microservices.values(): for prop in microservice.get("properties", []): # external mail server @@ -393,7 +381,9 @@ def detect_miscellaneous(microservices: dict, information_flows: dict, external_ traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def merge_duplicate_flows(information_flows: dict): @@ -431,6 +421,8 @@ def merge_duplicate_flows(information_flows: dict): for k in to_delete: del information_flows[k] + return information_flows + def merge_duplicate_nodes(nodes: dict): """Merge duplicate nodes @@ -458,6 +450,8 @@ def merge_duplicate_nodes(nodes: dict): for k in to_delete: del nodes[k] + return nodes + def merge_duplicate_annotations(collection: dict): """Merge annotations of all items diff --git a/core/file_interaction.py b/core/file_interaction.py index 4535eef..527c8e8 100644 --- a/core/file_interaction.py +++ b/core/file_interaction.py @@ -4,8 +4,7 @@ from pathlib import Path, PurePosixPath from output_generators.logger import logger -import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config count = 0 @@ -61,7 +60,7 @@ def search_keywords(keywords: str): """Searches keywords locally using grep. """ - repo_folder = tmp.tmp_config["Repository"]["local_path"] + repo_folder = code2dfd_config["Repository"]["local_path"] results = dict() @@ -171,7 +170,7 @@ def file_as_lines(path): """Downloads and splits raw file into lines. """ - local_path = tmp.tmp_config.get("Repository", "local_path") + local_path = code2dfd_config.get("Repository", "local_path") local_path = os.path.join(local_path, path) with open(local_path, "r") as file: @@ -179,38 +178,6 @@ def file_as_lines(path): return file_as_lines -def detect_microservice(file_path, dfd): - """Finds microservice that a file belongs to. - """ - - microservices_set = tech_sw.get_microservices(dfd) - microservices = [microservices_set[x]["name"] for x in microservices_set.keys()] - - file_path_parts = Path(file_path).parts - count = 0 - part = 0 - found = False - - while part < len(file_path_parts) and not found: - for m in microservices: - if m == file_path_parts[part]: - microservice = m - count += 1 - if count > 0: - found = True - part += 1 - if count == 1: - return microservice - else: - print("\tFound " + str(count) + " microservices for file " + str(file_path) +". \ - \n\tPlease choose microservice that the file belongs to: ") - i = 1 - for m in microservices: - print("\t[" + str(i) + "] " + str(m)) - i += 1 - return microservices[int(input("\n\t > ")) - 1] - - def find_variable(parameter: str, file) -> str: """ Looks for passed ´parameter´ in passed ´file´ or other files. """ @@ -287,7 +254,7 @@ def resolve_url(url: str, microservice: str, dfd) -> str: """Tries to resolve a url into one of the microserices. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] target_service = False if "http" in url: @@ -329,7 +296,7 @@ def resolve_url(url: str, microservice: str, dfd) -> str: def check_dockerfile(build_path: str): """Checks if under the service's build-path there is a dockerfile. If yes, returns it. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] # find docker-compose path, since build-path is relative to that raw_files = get_file_as_lines("docker-compose.yml") @@ -368,7 +335,7 @@ def file_exists(file_name: str) -> bool: """Checks if a file exists in the repository. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(local_repo_path)) @@ -391,7 +358,7 @@ def get_repo_contents_local(path: str) -> set: repo = set() - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] to_crawl = local_repo_path if path: @@ -414,7 +381,7 @@ def get_file_as_yaml(filename: str) -> dict: files = dict() - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() @@ -437,7 +404,7 @@ def get_file_as_lines(filename: str) -> dict: files = dict() - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] out = subprocess.Popen(['find', local_path, '-name', filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() diff --git a/core/parse_files.py b/core/parse_files.py index 8c04498..7334378 100644 --- a/core/parse_files.py +++ b/core/parse_files.py @@ -4,7 +4,7 @@ import ruamel.yaml import core.file_interaction as fi -import tmp.tmp as tmp +from core.config import code2dfd_config from output_generators.logger import logger @@ -173,7 +173,7 @@ def parse_yaml_file(file_path: str) -> str: """ - local_path = tmp.tmp_config.get("Repository", "local_path") + local_path = code2dfd_config.get("Repository", "local_path") file_path = os.path.join(local_path, file_path) yaml = ruamel.yaml.YAML() yaml.Constructor = MyConstructor diff --git a/core/technology_switch.py b/core/technology_switch.py index b1f3617..b647eb0 100644 --- a/core/technology_switch.py +++ b/core/technology_switch.py @@ -1,6 +1,3 @@ -import ast - -from output_generators.logger import logger import technology_specific_extractors.database_connections.dbc_entry as dbc import technology_specific_extractors.docker_compose.dcm_entry as dcm import technology_specific_extractors.feign_client.fgn_entry as fgn @@ -11,48 +8,36 @@ import technology_specific_extractors.maven.mvn_entry as mvn import technology_specific_extractors.rabbitmq.rmq_entry as rmq import technology_specific_extractors.resttemplate.rst_entry as rst -import tmp.tmp as tmp +CONTAINER_TECH_LIST = {"Maven": mvn, "Gradle": grd, "DockerCompose": dcm} +COMMUNICATIONS_TECH_LIST = {"RabbitMQ": rmq, "Kafka": kfk, "RestTemplate": rst, "FeignClient": fgn, + "Implicit Connections": imp, "Database Connections": dbc, "HTML": html, + "Docker-Compose": dcm} -def get_microservices(dfd) -> dict: - """Calls get_microservices from correct container technology or returns existing list. - """ - if tmp.tmp_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) - else: - logger.info("Microservices not set yet, start extraction") +def set_microservices(dfd): + """Calls set_microservices from correct container technology or returns existing list. + """ - mvn.set_microservices(dfd) - grd.set_microservices(dfd) - dcm.set_microservices(dfd) - if tmp.tmp_config.has_option("DFD", "microservices"): - return ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) + for func in CONTAINER_TECH_LIST.values(): + func.set_microservices(dfd) -def get_information_flows(dfd) -> dict: +def set_information_flows(dfd) -> dict: """Calls get_information_flows from correct communication technology. """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - logger.info("Information flows not set yet, start extraction") - communication_techs_list = ast.literal_eval(tmp.tmp_config["Technology Profiles"]["communication_techs_list"]) - for com_tech in communication_techs_list: - eval(com_tech[1]).set_information_flows(dfd) + for func in COMMUNICATIONS_TECH_LIST.values(): + func.set_information_flows(dfd) - if tmp.tmp_config.has_option("DFD", "information_flows"): - return ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) + return dfd["information_flows"] def detect_microservice(file_path: str, dfd) -> str: """Calls detect_microservices from correct microservice detection technology. """ - microservice = mvn.detect_microservice(file_path, dfd) - if not microservice: - microservice = grd.detect_microservice(file_path, dfd) - if not microservice: - microservice = dcm.detect_microservice(file_path, dfd) - return microservice + for func in CONTAINER_TECH_LIST.values(): + microservice = func.detect_microservice(file_path, dfd) + if microservice: + return microservice diff --git a/output_generators/codeable_model.py b/output_generators/codeable_model.py index 716a62f..46739c3 100644 --- a/output_generators/codeable_model.py +++ b/output_generators/codeable_model.py @@ -1,15 +1,19 @@ import os from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config # the used metamodel is microservice_dfds_metamodel.py -def output_codeable_model(microservices, information_flows, external_components): +def output_codeable_model(dfd): """Entry function to creation of codeable models. Calls all necessary helper functions and outputs the codeable model""" - parts = Path(tmp.tmp_config["Analysis Settings"]["output_path"]).parts + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + + parts = Path(code2dfd_config["Analysis Settings"]["output_path"]).parts model_name = f"{parts[-2]}_{parts[-1]}" file_content = header() @@ -129,7 +133,7 @@ def create_file(model_name: str, content: str): """Writes content to file. """ model_name = model_name.replace("-", "_") - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] filename = f"{model_name}.py" output_path = os.path.join(output_path, filename) diff --git a/output_generators/codeable_models_to_plantuml.py b/output_generators/codeable_models_to_plantuml.py index 47d4dc9..a65d991 100644 --- a/output_generators/codeable_models_to_plantuml.py +++ b/output_generators/codeable_models_to_plantuml.py @@ -1,7 +1,7 @@ import os.path from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config plantuml_new = str() @@ -13,7 +13,7 @@ def convert(codeable_models_path: str) -> str: global plantuml_new - output_file_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_file_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_file_path).parts filename = f"{parts[-2]}--{parts[-1]}_uml.txt" output_file_path = os.path.join(output_file_path, filename) diff --git a/output_generators/json_architecture.py b/output_generators/json_architecture.py index 2f40459..093c8b4 100644 --- a/output_generators/json_architecture.py +++ b/output_generators/json_architecture.py @@ -2,18 +2,22 @@ import os from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config -def generate_json_architecture(microservices: dict, information_flows: dict, external_components: dict): +def generate_json_architecture(dfd): """Creates JSON file that contains the complete extracted architecture. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + full_dict = {"microservices": list(microservices.values()), "information_flows": list(information_flows.values()), "external_components": list(external_components.values())} - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_json_architecture.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/json_edges.py b/output_generators/json_edges.py index dcf5804..2a8dc6c 100644 --- a/output_generators/json_edges.py +++ b/output_generators/json_edges.py @@ -7,12 +7,13 @@ import os from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config -def generate_json_edges(information_flows: dict): +def generate_json_edges(dfd: dict): """Creates JSON file that contains a list of plain information flows, without annotations. """ + information_flows = dfd["information_flows"] edges_list = list() for i in information_flows.keys(): @@ -33,7 +34,7 @@ def write_to_file(architecture_dict): """Writes json architecture to file. """ - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_edges.json" output_path = os.path.join(output_path, filename) diff --git a/output_generators/plaintext.py b/output_generators/plaintext.py index 614bf7d..6ca8f1b 100644 --- a/output_generators/plaintext.py +++ b/output_generators/plaintext.py @@ -1,11 +1,15 @@ import os from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config -def write_plaintext(microservices, information_flows, external_components): - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] +def write_plaintext(dfd): + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_results.txt" output_path = os.path.join(output_path, filename) diff --git a/output_generators/traceability.py b/output_generators/traceability.py index bb1330d..5c439a9 100644 --- a/output_generators/traceability.py +++ b/output_generators/traceability.py @@ -2,7 +2,7 @@ import os from pathlib import Path -import tmp.tmp as tmp +from core.config import code2dfd_config traceability = dict() traceability["nodes"] = dict() @@ -154,7 +154,7 @@ def write_to_file(): """Writes tracebility info from dict to json file. """ - output_path = tmp.tmp_config["Analysis Settings"]["output_path"] + output_path = code2dfd_config["Analysis Settings"]["output_path"] parts = Path(output_path).parts filename = f"{parts[-2]}--{parts[-1]}_traceability.json" output_path = os.path.join(output_path, filename) diff --git a/technology_specific_extractors/apache_httpd/aph_entry.py b/technology_specific_extractors/apache_httpd/aph_entry.py index aa82a73..76ba34e 100644 --- a/technology_specific_extractors/apache_httpd/aph_entry.py +++ b/technology_specific_extractors/apache_httpd/aph_entry.py @@ -7,14 +7,20 @@ import output_generators.traceability as traceability -def detect_apachehttpd_webserver(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_apachehttpd_webserver(dfd): """Detects apachehttpd webservers and routes if possible. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + microservices, information_flows, external_components = detect_via_docker(microservices, information_flows, external_components, dfd) microservices, information_flows, external_components = detect_via_proxypass(microservices, information_flows, external_components, dfd) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def detect_via_docker(microservices: dict, information_flows: dict, external_components: dict, dfd): diff --git a/technology_specific_extractors/circuit_breaker/cbr_entry.py b/technology_specific_extractors/circuit_breaker/cbr_entry.py index 5007e59..3d31a07 100644 --- a/technology_specific_extractors/circuit_breaker/cbr_entry.py +++ b/technology_specific_extractors/circuit_breaker/cbr_entry.py @@ -2,10 +2,13 @@ import core.technology_switch as tech_sw -def detect_circuit_breakers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_circuit_breakers(dfd): """Find circuit breakers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableCircuitBreaker") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -50,11 +53,5 @@ def detect_circuit_breakers(microservices: dict, information_flows: dict, dfd) - else: information_flows[i]["tagged_values"] = [circuit_breaker_tuple] - return microservices, information_flows - - -def detect_circuit_breaker_tech(path): - - circuit_breaker_tuple = False - - return circuit_breaker_tuple + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/consul/cns_entry.py b/technology_specific_extractors/consul/cns_entry.py index fc78955..450d4b6 100644 --- a/technology_specific_extractors/consul/cns_entry.py +++ b/technology_specific_extractors/consul/cns_entry.py @@ -1,10 +1,12 @@ import output_generators.traceability as traceability -def detect_consul(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_consul(dfd): """Detects Consul server and clients (service discover, monitoring, configuration). """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] # Server consul_server = set() for m in microservices.keys(): @@ -43,4 +45,5 @@ def detect_consul(microservices: dict, information_flows: dict, dfd) -> dict: traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/database_connections/dbc_entry.py b/technology_specific_extractors/database_connections/dbc_entry.py index e9b8bca..51c183c 100644 --- a/technology_specific_extractors/database_connections/dbc_entry.py +++ b/technology_specific_extractors/database_connections/dbc_entry.py @@ -1,40 +1,19 @@ -import ast - import technology_specific_extractors.environment_variables as env -import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability -def set_information_flows(dfd) -> set: +def set_information_flows(dfd): """Goes through services and checks if there are connections to databases. """ - - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - if tmp.tmp_config.has_option("DFD", "external_components"): - external_components = ast.literal_eval(tmp.tmp_config["DFD"]["external_components"]) - else: - external_components = dict() - - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - return microservices, information_flows - - -def get_information_flows(microservices: dict, information_flows: dict, external_components: dict) -> dict: - - microservices, information_flows, external_components = check_properties(microservices, information_flows, external_components) - - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def check_properties(microservices: dict, information_flows: dict, external_components: dict) -> dict: @@ -220,8 +199,6 @@ def check_properties(microservices: dict, information_flows: dict, external_comp except: information_flows[id]["tagged_values"] = [("Username", username.strip())] - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - trace = dict() trace["item"] = "database-" + str(microservices[m]["name"]) + " -> " + microservices[m]["name"] trace["file"] = trace_info[0] @@ -233,10 +210,13 @@ def check_properties(microservices: dict, information_flows: dict, external_comp return microservices, information_flows, external_components -def clean_database_connections(microservices: dict, information_flows: dict): +def clean_database_connections(dfd: dict): """Removes database connections in wrong direction, which can occur from docker compose. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + for microservice in microservices.values(): if microservice["type"] == "database_component": to_purge = set() @@ -245,3 +225,6 @@ def clean_database_connections(microservices: dict, information_flows: dict): to_purge.add(i) for p in to_purge: del information_flows[p] + + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/databases/dbs_entry.py b/technology_specific_extractors/databases/dbs_entry.py index 21f078b..095422e 100644 --- a/technology_specific_extractors/databases/dbs_entry.py +++ b/technology_specific_extractors/databases/dbs_entry.py @@ -1,10 +1,12 @@ import core.file_interaction as fi import output_generators.traceability as traceability -def detect_databases(microservices: dict) -> dict: + +def detect_databases(dfd: dict): """Detects databases. """ + microservices = dfd["microservices"] for m in microservices.keys(): database = False if "image" in microservices[m]: @@ -34,7 +36,7 @@ def detect_databases(microservices: dict) -> dict: else: microservices = detect_via_docker(microservices, m) - return microservices + dfd["microservices"] = microservices def detect_via_docker(microservices: dict, m: int) -> dict: diff --git a/technology_specific_extractors/docker/dcr_entry.py b/technology_specific_extractors/docker/dcr_entry.py index 45ea022..ff5ab5e 100644 --- a/technology_specific_extractors/docker/dcr_entry.py +++ b/technology_specific_extractors/docker/dcr_entry.py @@ -1,6 +1,6 @@ import os -import tmp.tmp as tmp +from core.config import code2dfd_config def detect_port(path: str) -> int: @@ -8,7 +8,7 @@ def detect_port(path: str) -> int: """ port = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, os.path.dirname(path)))) diff --git a/technology_specific_extractors/docker_compose/dcm_entry.py b/technology_specific_extractors/docker_compose/dcm_entry.py index 37750ec..bd4e343 100644 --- a/technology_specific_extractors/docker_compose/dcm_entry.py +++ b/technology_specific_extractors/docker_compose/dcm_entry.py @@ -1,11 +1,9 @@ -import ast import os import core.file_interaction as fi from output_generators.logger import logger -import core.technology_switch as tech_sw import technology_specific_extractors.docker_compose.dcm_parser as dcm_parser -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability docker_compose_content = False @@ -27,23 +25,23 @@ def set_microservices(dfd) -> None: if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose*") if len(raw_files) == 0: - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return docker_compose_content = raw_files[0]["content"] - microservices_set, properties_dict = dcm_parser.extract_microservices(docker_compose_content, raw_files[0]["path"]) + microservices_set, properties_dict = dcm_parser.extract_microservices(docker_compose_content, raw_files[0]["path"], dfd) if not microservices_set: - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return - microservices = dictionarify(microservices_set, properties_dict) + microservices = dictionarify(microservices_set, properties_dict, dfd["microservices"]) microservices = clean_pom_names(microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices def clean_pom_names(microservices: dict) -> dict: @@ -56,15 +54,10 @@ def clean_pom_names(microservices: dict) -> dict: return microservices -def dictionarify(elements_set: set, properties_dict: dict) -> dict: +def dictionarify(elements_set: set, properties_dict: dict, microservices: dict) -> dict: """Turns set of services into dictionary. """ - if tmp.tmp_config.has_option("DFD", "microservices"): - elements = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) - else: - elements = dict() - for e in elements_set: try: properties = properties_dict[e[0]] @@ -87,17 +80,17 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: trace["span"] = e[3][3] traceability.add_trace(trace) try: - id = max(elements.keys()) + 1 + id = max(microservices.keys()) + 1 except: id = 0 - elements[id] = dict() + microservices[id] = dict() - elements[id]["name"] = e[0] - elements[id]["image"] = e[1] - elements[id]["type"] = e[2] - elements[id]["properties"] = properties - elements[id]["stereotype_instances"] = stereotypes - elements[id]["tagged_values"] = tagged_values + microservices[id]["name"] = e[0] + microservices[id]["image"] = e[1] + microservices[id]["type"] = e[2] + microservices[id]["properties"] = properties + microservices[id]["stereotype_instances"] = stereotypes + microservices[id]["tagged_values"] = tagged_values trace = dict() trace["item"] = e[0]#.replace("pom_", "") @@ -106,7 +99,7 @@ def dictionarify(elements_set: set, properties_dict: dict) -> dict: trace["span"] = e[4][2] traceability.add_trace(trace) - return elements + return microservices def set_information_flows(dfd): @@ -115,12 +108,8 @@ def set_information_flows(dfd): global docker_compose_content - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] # Download docker-compose file if not docker_compose_content: @@ -134,9 +123,7 @@ def set_information_flows(dfd): docker_compose_content = raw_files[0]["content"] information_flows = dcm_parser.extract_information_flows(docker_compose_content, microservices, information_flows) - - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def get_environment_variables(docker_compose_file_URL: str) -> set: @@ -172,11 +159,11 @@ def detect_microservice(file_path: str, dfd) -> str: """Detects, which service a file belongs to based on image given in docker-compose file and dockerfile belonging to file given as input. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] microservice = False dockerfile_path = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] # Find corresponding dockerfile dirs = list() diff --git a/technology_specific_extractors/docker_compose/dcm_parser.py b/technology_specific_extractors/docker_compose/dcm_parser.py index 7ec5e4b..5fae87a 100644 --- a/technology_specific_extractors/docker_compose/dcm_parser.py +++ b/technology_specific_extractors/docker_compose/dcm_parser.py @@ -1,12 +1,10 @@ -import ast import re from pathlib import Path import ruamel.yaml -import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env -import tmp.tmp as tmp +from core.config import code2dfd_config # The following is taken from ruamel.yaml's authro as a workaround for getting line count for str objects @@ -55,7 +53,7 @@ def construct_scalar(self, node): # end of external code -def extract_microservices(file_content, file_name) -> set: +def extract_microservices(file_content, file_name, dfd) -> set: """ Extracts the list of microservices from the docker-compose file autonomously, i.e. without asking for user-input in case of errors. """ @@ -67,10 +65,7 @@ def extract_microservices(file_content, file_name) -> set: image = False build = False - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices_dict = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) - else: - microservices_dict= dict() + microservices_dict = dfd["microservices"] microservices_set = set() properties_dict = dict() @@ -621,7 +616,7 @@ def extract_microservices(file_content, file_name) -> set: else: microservices_dict[correct_id]["properties"] = properties - tmp.tmp_config.set("DFD", "microservices", str(microservices_dict).replace("%", "%%")) + dfd["microservices"] = microservices_dict return microservices_set, properties_dict diff --git a/technology_specific_extractors/elasticsearch/ela_entry.py b/technology_specific_extractors/elasticsearch/ela_entry.py index 26e0763..30fde43 100644 --- a/technology_specific_extractors/elasticsearch/ela_entry.py +++ b/technology_specific_extractors/elasticsearch/ela_entry.py @@ -1,10 +1,13 @@ import output_generators.traceability as traceability -def detect_elasticsearch(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_elasticsearch(dfd): """Detects elasticsearch services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + elasticsearch = False for m in microservices.keys(): if "elasticsearch:" in microservices[m]["image"]: @@ -52,4 +55,5 @@ def detect_elasticsearch(microservices: dict, information_flows: dict, dfd) -> d information_flows.pop(p) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/eureka/eur_entry.py b/technology_specific_extractors/eureka/eur_entry.py index 7bdb895..6bcaf76 100644 --- a/technology_specific_extractors/eureka/eur_entry.py +++ b/technology_specific_extractors/eureka/eur_entry.py @@ -3,13 +3,15 @@ import output_generators.traceability as traceability -def detect_eureka(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_eureka(dfd): """Detects Eureka servers if there are any. """ # Server (/microservice classification) results = fi.search_keywords("@EnableEurekaServer") + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] eureka_server = False for r in results.keys(): eureka_server = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -78,7 +80,8 @@ def detect_eureka(microservices: dict, information_flows: dict, dfd) -> dict: traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def is_eureka(microservice: tuple) -> bool: @@ -95,8 +98,9 @@ def is_eureka(microservice: tuple) -> bool: return False -def detect_eureka_server_only(microservices: dict, dfd): +def detect_eureka_server_only(dfd: dict): + microservices = dfd["microservices"] results = fi.search_keywords("@EnableEurekaServer") eureka_servers = set() for r in results.keys(): @@ -114,4 +118,4 @@ def detect_eureka_server_only(microservices: dict, dfd): except: microservices[m]["tagged_values"] = ("Service Discovery", "Eureka") - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/feign_client/fgn_entry.py b/technology_specific_extractors/feign_client/fgn_entry.py index e6b0963..4ff262d 100644 --- a/technology_specific_extractors/feign_client/fgn_entry.py +++ b/technology_specific_extractors/feign_client/fgn_entry.py @@ -1,21 +1,14 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability -def set_information_flows(dfd) -> dict: +def set_information_flows(dfd): """Detects uses of Feign Client in the code. """ - microservices = tech_sw.get_microservices(dfd) - - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] # check for circuit breaker results = fi.search_keywords("@EnableFeignClients") # content, name, path @@ -94,8 +87,7 @@ def set_information_flows(dfd) -> dict: traceability.add_trace(trace) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def is_microservice(service: str, dfd) -> bool: @@ -105,14 +97,9 @@ def is_microservice(service: str, dfd) -> bool: if not service: return False is_microservice = False - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] for m in microservices.keys(): if service.casefold() == microservices[m]["name"].casefold(): is_microservice = True return is_microservice - - - - -# diff --git a/technology_specific_extractors/gradle/grd_entry.py b/technology_specific_extractors/gradle/grd_entry.py index f11b7d2..6d7580e 100644 --- a/technology_specific_extractors/gradle/grd_entry.py +++ b/technology_specific_extractors/gradle/grd_entry.py @@ -1,12 +1,9 @@ -import ast import os -from pathlib import Path import core.file_interaction as fi from output_generators.logger import logger import core.parse_files as parse -import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -17,10 +14,7 @@ def set_microservices(dfd) -> dict: if not used_in_application(): return False - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) - else: - microservices = dict() + microservices = dfd["microservices"] gradle_files = fi.get_file_as_lines("build.gradle") for gf in gradle_files.keys(): @@ -56,9 +50,7 @@ def set_microservices(dfd) -> dict: except: pass - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) - - return microservices + dfd["microservices"] = microservices def used_in_application() -> bool: @@ -89,7 +81,7 @@ def parse_properties_file(gradle_path: str): # find properties file path = os.path.dirname(gradle_path) - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -130,12 +122,12 @@ def detect_microservice(file_path, dfd): return False microservice = [False, False] - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] found_gradle = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(file_path) diff --git a/technology_specific_extractors/grafana/grf_entry.py b/technology_specific_extractors/grafana/grf_entry.py index bfda61c..c436fd0 100644 --- a/technology_specific_extractors/grafana/grf_entry.py +++ b/technology_specific_extractors/grafana/grf_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_grafana(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_grafana(dfd): """Detects grafana server and connections. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + grafana_server = str() results = fi.search_keywords("grafana/grafana") @@ -31,4 +34,5 @@ def detect_grafana(microservices: dict, information_flows: dict, dfd) -> dict: trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/html/html_entry.py b/technology_specific_extractors/html/html_entry.py index 418b981..42ad3d3 100644 --- a/technology_specific_extractors/html/html_entry.py +++ b/technology_specific_extractors/html/html_entry.py @@ -1,21 +1,14 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw import output_generators.traceability as traceability -import tmp.tmp as tmp def set_information_flows(dfd): """Looks for connections between services via html sites / href's. """ - microservices = tech_sw.get_microservices(dfd) - - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] results = fi.search_keywords("href") for r in results.keys(): @@ -52,5 +45,4 @@ def set_information_flows(dfd): except: pass - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/http_security/hts_entry.py b/technology_specific_extractors/http_security/hts_entry.py index ad1393e..c3722e0 100644 --- a/technology_specific_extractors/http_security/hts_entry.py +++ b/technology_specific_extractors/http_security/hts_entry.py @@ -3,14 +3,12 @@ import output_generators.traceability as traceability -def detect_authentication_scopes(microservices: dict, dfd) -> dict: +def detect_authentication_scopes(dfd: dict): """Detects authentication scopes via HttpSecurity configurations. """ configuration_tuples = detect_configurations(dfd) - microservices = interpret_configurations(microservices, configuration_tuples) - - return microservices + interpret_configurations(dfd, configuration_tuples) def detect_configurations(dfd): @@ -64,10 +62,11 @@ def detect_configurations(dfd): return configuration_tuples -def interpret_configurations(microservices: dict, configuration_tuples: list) -> dict: +def interpret_configurations(dfd: dict, configuration_tuples: list) -> dict: """Translates configurations into stereotypes and tagged values. """ + microservices = dfd["microservices"] for configuration_tuple in configuration_tuples: stereotypes, tagged_values = list(), list() # create stereotypes and tagged_values @@ -140,4 +139,4 @@ def interpret_configurations(microservices: dict, configuration_tuples: list) -> except: microservices[m]["tagged_values"] = tagged_values - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/hystrix/hsx_entry.py b/technology_specific_extractors/hystrix/hsx_entry.py index fbaccf0..1adc143 100644 --- a/technology_specific_extractors/hystrix/hsx_entry.py +++ b/technology_specific_extractors/hystrix/hsx_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_hystrix_dashboard(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_hystrix_dashboard(dfd): """Detects hystrix monitoring dashboards . """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableHystrixDashboard") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -31,13 +34,17 @@ def detect_hystrix_dashboard(microservices: dict, information_flows: dict, dfd) trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows -def detect_hystrix_circuit_breakers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_hystrix_circuit_breakers(dfd): """Detects HystrixCommand. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableHystrix") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -62,4 +69,5 @@ def detect_hystrix_circuit_breakers(microservices: dict, information_flows: dict trace["span"] = results[r]["span"] traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/implicit_connections/imp_entry.py b/technology_specific_extractors/implicit_connections/imp_entry.py index 8a89219..5ed2e93 100644 --- a/technology_specific_extractors/implicit_connections/imp_entry.py +++ b/technology_specific_extractors/implicit_connections/imp_entry.py @@ -1,25 +1,18 @@ -import ast import os import yaml import core.file_interaction as fi from output_generators.logger import logger -import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability -def set_information_flows(dfd) -> dict: +def set_information_flows(dfd): """Adds connections based on parsed config files. """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] # Weavescope new_information_flows = weavescope(microservices) @@ -40,8 +33,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def weavescope(microservices): diff --git a/technology_specific_extractors/kafka/kfk_entry.py b/technology_specific_extractors/kafka/kfk_entry.py index 2575569..2d40094 100644 --- a/technology_specific_extractors/kafka/kfk_entry.py +++ b/technology_specific_extractors/kafka/kfk_entry.py @@ -5,43 +5,36 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability from output_generators.logger import logger kafka_server = str() -def set_information_flows(dfd) -> set: +def set_information_flows(dfd): """Connects incoming endpoints, outgoing endpoints, and routings to information flows """ - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] incoming_endpoints = get_incoming_endpoints(dfd) outgoing_endpoints = get_outgoing_endpoints(dfd) - new_information_flows = match_incoming_to_outgoing_endpoints(microservices, incoming_endpoints, outgoing_endpoints) + new_information_flows = match_incoming_to_outgoing_endpoints(microservices, information_flows, incoming_endpoints, outgoing_endpoints) # merge old and new flows - for ni in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = new_information_flows[ni] + if new_information_flows is not information_flows: + for ni in new_information_flows.keys(): + try: + id = max(information_flows.keys()) + 1 + except: + id = 0 + information_flows[id] = new_information_flows[ni] information_flows = detect_stream_binders(microservices, information_flows, dfd) - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - - return information_flows + dfd["information_flows"] = information_flows def get_incoming_endpoints(dfd) -> set: @@ -178,17 +171,12 @@ def asset_is_input(variable: str, file, line_nr: int) -> bool: return False -def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints: set, outgoing_endpoints: set) -> dict: +def match_incoming_to_outgoing_endpoints(microservices: dict, information_flows: dict, incoming_endpoints: set, outgoing_endpoints: set) -> dict: """Finds information flows by regexing routing keys of outgoing endpoints to queues of incoming endpoints. """ # incoming: (topic, microservice, (file, line, span)) # outgoing: (topic, microservice, asset, (file, line, span)) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - kafka_server = False for id in microservices.keys(): if ("Message Broker", "Kafka") in microservices[id]["tagged_values"]: @@ -310,12 +298,13 @@ def match_incoming_to_outgoing_endpoints(microservices: dict, incoming_endpoints return information_flows -def detect_kafka_server(microservices: dict) -> dict: +def detect_kafka_server(dfd: dict) -> dict: """Detects and marks kafka server. """ global kafka_server + microservices = dfd["microservices"] raw_files = fi.get_file_as_yaml("docker-compose.yml") if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose.yaml") @@ -379,7 +368,7 @@ def detect_kafka_server(microservices: dict) -> dict: traceability.add_trace(trace) except: pass - return microservices + dfd["microservices"] = microservices def detect_stream_binders(microservices: dict, information_flows: dict, dfd) -> dict: diff --git a/technology_specific_extractors/kibana/kib_entry.py b/technology_specific_extractors/kibana/kib_entry.py index 02e27d6..b873d07 100644 --- a/technology_specific_extractors/kibana/kib_entry.py +++ b/technology_specific_extractors/kibana/kib_entry.py @@ -1,9 +1,13 @@ import output_generators.traceability as traceability -def detect_kibana(microservices: dict, information_flows: dict, dfd) -> dict: + +def detect_kibana(dfd): """Detects logstash services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + for m in microservices.keys(): if "kibana:" in microservices[m]["image"]: try: @@ -22,4 +26,5 @@ def detect_kibana(microservices: dict, information_flows: dict, dfd) -> dict: trace["span"] = "heuristic, based on image" traceability.add_trace(trace) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/load_balancer/lob_entry.py b/technology_specific_extractors/load_balancer/lob_entry.py index 1670888..6029cb6 100644 --- a/technology_specific_extractors/load_balancer/lob_entry.py +++ b/technology_specific_extractors/load_balancer/lob_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_load_balancers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_load_balancers(dfd): """Find load balancers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@LoadBalanced") # content, name, path for r in results.keys(): microservice = tech_sw.detect_microservice(results[r]["path"], dfd) @@ -50,4 +53,5 @@ def detect_load_balancers(microservices: dict, information_flows: dict, dfd) -> else: information_flows[i]["tagged_values"] = [('Load Balancer', "Spring Cloud")] - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/local_logging/llo_entry.py b/technology_specific_extractors/local_logging/llo_entry.py index 5e508e7..09e3151 100644 --- a/technology_specific_extractors/local_logging/llo_entry.py +++ b/technology_specific_extractors/local_logging/llo_entry.py @@ -3,14 +3,18 @@ import output_generators.traceability as traceability -def detect_local_logging(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_local_logging(dfd): """Detects if a service performs local logging. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices = detect_loggerfactory(microservices, dfd) microservices = detect_lombok(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_loggerfactory(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/logstash/log_entry.py b/technology_specific_extractors/logstash/log_entry.py index 67d5a74..b02c808 100644 --- a/technology_specific_extractors/logstash/log_entry.py +++ b/technology_specific_extractors/logstash/log_entry.py @@ -1,10 +1,14 @@ import output_generators.traceability as traceability -def detect_logstash(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_logstash(dfd): """Detects logstash services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + # Service logstash = False trace_info = False @@ -155,4 +159,6 @@ def detect_logstash(microservices: dict, information_flows: dict, external_compo traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/maven/mvn_entry.py b/technology_specific_extractors/maven/mvn_entry.py index 81c2d7a..beaed7a 100644 --- a/technology_specific_extractors/maven/mvn_entry.py +++ b/technology_specific_extractors/maven/mvn_entry.py @@ -1,13 +1,11 @@ -import ast import os import re import core.file_interaction as fi from output_generators.logger import logger import core.parse_files as parse -import core.technology_switch as tech_sw import technology_specific_extractors.docker.dcr_entry as dcr -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability try: @@ -23,10 +21,7 @@ def set_microservices(dfd) -> dict: """Extracts the list of services from pom.xml files and sets the variable in the tmp-file. """ - if tmp.tmp_config.has_option("DFD", "microservices"): - microservices = ast.literal_eval(tmp.tmp_config["DFD"]["microservices"]) - else: - microservices = dict() + microservices = dfd["microservices"] microservices_set = set() pom_files = fi.get_file_as_lines("pom.xml") @@ -71,9 +66,7 @@ def set_microservices(dfd) -> dict: nested_microservices = check_nested_modules(module_dict) microservices_set.update(nested_microservices) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) # Need to escape single percentage signs for ConfigParser - - return microservices + dfd["microservices"] = microservices def extract_dependencies(properties: set, pom_file) -> set: @@ -81,7 +74,7 @@ def extract_dependencies(properties: set, pom_file) -> set: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -100,7 +93,7 @@ def extract_modules(pom_file: dict) -> list: """ file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -146,7 +139,7 @@ def parse_properties_file(pom_path: str): # find properties file path = os.path.dirname(pom_path) - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() dirs.append(os.scandir(os.path.join(local_repo_path, path))) @@ -185,7 +178,7 @@ def extract_servicename_pom_file(pom_file) -> str: microservice = [False, False] file_name = pom_file["path"] - pom_path = os.path.join(tmp.tmp_config.get("Repository", "local_path"), file_name) + pom_path = os.path.join(code2dfd_config.get("Repository", "local_path"), file_name) tree = etree.parse(pom_path) root = tree.getroot() @@ -219,12 +212,12 @@ def detect_microservice(file_path, dfd): """ microservice = [False, False] - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] path = file_path found_pom = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() path = os.path.dirname(path) diff --git a/technology_specific_extractors/nginx/ngn_entry.py b/technology_specific_extractors/nginx/ngn_entry.py index 30b5494..f40a299 100644 --- a/technology_specific_extractors/nginx/ngn_entry.py +++ b/technology_specific_extractors/nginx/ngn_entry.py @@ -4,7 +4,7 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability @@ -54,10 +54,14 @@ def construct_scalar(self, node): # end of external code -def detect_nginx(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_nginx(dfd): """Detects nginx web applications. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + web_app = False port = False correct_id = False @@ -101,7 +105,7 @@ def detect_nginx(microservices: dict, information_flows: dict, external_componen id = 0 microservices[id] = dict() - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] docker_path = os.path.dirname(results[r]["path"]) if docker_path and local_repo_path: docker_path = os.path.relpath(docker_path, start=local_repo_path) @@ -279,4 +283,6 @@ def detect_nginx(microservices: dict, information_flows: dict, external_componen # Add flows between web_app and user information_flows = ext.add_user_connections(information_flows, web_app) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/plaintext_credentials/plc_entry.py b/technology_specific_extractors/plaintext_credentials/plc_entry.py index 923ad92..53e3829 100644 --- a/technology_specific_extractors/plaintext_credentials/plc_entry.py +++ b/technology_specific_extractors/plaintext_credentials/plc_entry.py @@ -1,12 +1,13 @@ - import output_generators.traceability as traceability import technology_specific_extractors.environment_variables as env -def set_plaintext_credentials(microservices: dict) -> dict: +def set_plaintext_credentials(dfd: dict): """Goes through properties and sets stereotype and tagged values, if plaintext credentials are found. """ + microservices = dfd["microservices"] + for m in microservices.keys(): plaintext_credentials = False tagged_values = set() @@ -63,4 +64,4 @@ def set_plaintext_credentials(microservices: dict) -> dict: else: microservices[m]["tagged_values"] = tagged_values - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/prometheus/prm_entry.py b/technology_specific_extractors/prometheus/prm_entry.py index fcbf306..db2d39c 100644 --- a/technology_specific_extractors/prometheus/prm_entry.py +++ b/technology_specific_extractors/prometheus/prm_entry.py @@ -2,17 +2,21 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability -def detect_prometheus_server(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_prometheus_server(dfd): """Detects prometheus server and adds information flows. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices, information_flows = detect_server_docker(microservices, information_flows, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_server_docker(microservices: dict, information_flows: dict, dfd) -> dict: @@ -59,7 +63,7 @@ def detect_connections(microservices: dict, information_flows: dict, dockerfile, """Parses config file to find connections to prometheus. """ - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] for line in dockerfile["content"]: if "ADD" in line: diff --git a/technology_specific_extractors/rabbitmq/rmq_entry.py b/technology_specific_extractors/rabbitmq/rmq_entry.py index ab4d86b..be71b9f 100644 --- a/technology_specific_extractors/rabbitmq/rmq_entry.py +++ b/technology_specific_extractors/rabbitmq/rmq_entry.py @@ -1,11 +1,9 @@ -import ast import re import yaml import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability from output_generators.logger import logger @@ -17,28 +15,24 @@ def set_information_flows(dfd) -> set: if not used_in_application(): return - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - + information_flows = dfd["information_flows"] new_information_flows = dict() routings = get_routings() incoming_endpoints = get_incoming_endpoints(dfd) outgoing_endpoints = get_outgoing_endpoints(routings, dfd) - new_information_flows = match_incoming_to_outgoing_endpoints(incoming_endpoints, outgoing_endpoints, dfd) + new_information_flows = match_incoming_to_outgoing_endpoints(information_flows, incoming_endpoints, outgoing_endpoints, dfd) # merge old and new flows - for ni in new_information_flows.keys(): - try: - id = max(information_flows.keys()) + 1 - except: - id = 0 - information_flows[id] = new_information_flows[ni] - - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + if new_information_flows is not information_flows: + for flow in new_information_flows.values(): + try: + id = max(information_flows.keys()) + 1 + except: + id = 0 + information_flows[id] = flow + + dfd["information_flows"] = information_flows def used_in_application(): @@ -148,19 +142,14 @@ def get_outgoing_endpoints(routings: set, dfd) -> set: return outgoing_endpoints -def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpoints: set, dfd) -> dict: +def match_incoming_to_outgoing_endpoints(information_flows: dict, incoming_endpoints: set, outgoing_endpoints: set, dfd): """Finds information flows by regexing routing keys of outgoing endpoints to queues of incoming endpoints. """ # outgoing: (exchange, routingkey, microservice, (file, line, span)) # incoming: (queue, microservice, (file, line, span)) - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() - - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] rabbit_server = False for id in microservices.keys(): if ("Message Broker", "RabbitMQ") in microservices[id]["tagged_values"]: @@ -304,14 +293,15 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: set, outgoing_endpo traceability.add_trace(trace) - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices return information_flows -def detect_rabbitmq_server(microservices: dict) -> dict: +def detect_rabbitmq_server(dfd: dict) -> dict: """Detects RabbitMQ server. """ + microservices = dfd["microservices"] raw_files = fi.get_file_as_yaml("docker-compose.yml") if len(raw_files) == 0: raw_files = fi.get_file_as_yaml("docker-compose.yaml") @@ -395,4 +385,4 @@ def detect_rabbitmq_server(microservices: dict) -> dict: except: pass - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/repository_rest_resource/rrr_entry.py b/technology_specific_extractors/repository_rest_resource/rrr_entry.py index b283dcb..af1e57d 100644 --- a/technology_specific_extractors/repository_rest_resource/rrr_entry.py +++ b/technology_specific_extractors/repository_rest_resource/rrr_entry.py @@ -2,10 +2,12 @@ import core.technology_switch as tech_sw -def detect_endpoints(microservices: dict, dfd) -> dict: +def detect_endpoints(dfd): """Detects endpoints offered via @RepositoryRestResource """ + microservices = dfd["microservices"] + results = fi.search_keywords("@RepositoryRestResource") for r in results.keys(): endpoints = set() @@ -22,4 +24,4 @@ def detect_endpoints(microservices: dict, dfd) -> dict: microservices[m]["tagged_values"].append(("Endpoints", list(endpoints))) except: microservices[m]["tagged_values"] = [("Endpoints", list(endpoints))] - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/resttemplate/rst_entry.py b/technology_specific_extractors/resttemplate/rst_entry.py index 3e7fc65..7285645 100644 --- a/technology_specific_extractors/resttemplate/rst_entry.py +++ b/technology_specific_extractors/resttemplate/rst_entry.py @@ -1,10 +1,7 @@ -import ast - import core.file_interaction as fi import core.technology_switch as tech_sw from output_generators.logger import logger import output_generators.traceability as traceability -import tmp.tmp as tmp def set_information_flows(dfd) -> dict: @@ -14,10 +11,7 @@ def set_information_flows(dfd) -> dict: if not used_in_application(): return - if tmp.tmp_config.has_option("DFD", "information_flows"): - information_flows = ast.literal_eval(tmp.tmp_config["DFD"]["information_flows"]) - else: - information_flows = dict() + information_flows = dfd["information_flows"] incoming_endpoints = get_incoming_endpoints(dfd) add_endpoints_tagged_values(incoming_endpoints, dfd) @@ -31,8 +25,7 @@ def set_information_flows(dfd) -> dict: id = 0 information_flows[id] = new_information_flows[ni] - tmp.tmp_config.set("DFD", "information_flows", str(information_flows).replace("%", "%%")) - return information_flows + dfd["information_flows"] = information_flows def used_in_application(): @@ -91,7 +84,6 @@ def get_incoming_endpoints(dfd) -> list: bracket_count = adjust_bracket_count(bracket_count, line) - tmp.tmp_config.set("DFD", "endpoints", str(endpoints)) return endpoints @@ -119,7 +111,7 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): """Adds tagged values containing the endpoitns to the microservices. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] ordered_endpoints = dict() for endpoint_tuple in endpoint_tuples: @@ -135,14 +127,14 @@ def add_endpoints_tagged_values(endpoint_tuples: list, dfd): microservices[m]["tagged_values"].append(('Endpoints', list(ordered_endpoints[endpoint]))) else: microservices[m]["tagged_values"] = [('Endpoints', list(ordered_endpoints[endpoint]))] - tmp.tmp_config.set("DFD", "microservices", str(microservices).replace("%", "%%")) + dfd["microservices"] = microservices def get_outgoing_endpoints(information_flows: dict, dfd) -> set: """Finds API calls from one service to another if restTemplate.exchange() is used in the application. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] if microservices != None: microservices = [microservices[x]["name"] for x in microservices.keys()] @@ -186,7 +178,7 @@ def get_outgoing_endpoints(information_flows: dict, dfd) -> set: def find_rst_variable(parameter: str, file: dict, line_nr: int, information_flows: dict, microservice: str, dfd, count=0): # check if service name in parameter, if yes, add flow directly - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] for m in microservices.keys(): if microservices[m]["name"] in parameter: try: @@ -311,7 +303,7 @@ def match_incoming_to_outgoing_endpoints(incoming_endpoints: list, outgoing_endp """ Find information flows by matching incomgin to outgoing endpoints. Returns list of flows. """ - microservices = tech_sw.get_microservices(dfd) + microservices = dfd["microservices"] information_flows_set = set() information_flows = dict() for o in outgoing_endpoints: diff --git a/technology_specific_extractors/ribbon/rib_entry.py b/technology_specific_extractors/ribbon/rib_entry.py index ac80c86..c5a9ee2 100644 --- a/technology_specific_extractors/ribbon/rib_entry.py +++ b/technology_specific_extractors/ribbon/rib_entry.py @@ -2,13 +2,15 @@ import core.technology_switch as tech_sw -def detect_ribbon_load_balancers(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_ribbon_load_balancers(dfd): """Detects load balancing via Ribbon. """ + microservices = dfd["microservices"] + microservices = detect_client_side(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices def detect_client_side(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/service_functionality_classification/itf_entry.py b/technology_specific_extractors/service_functionality_classification/itf_entry.py index 6a89228..55c06eb 100644 --- a/technology_specific_extractors/service_functionality_classification/itf_entry.py +++ b/technology_specific_extractors/service_functionality_classification/itf_entry.py @@ -1,6 +1,7 @@ import output_generators.traceability as traceability -def classify_internal_infrastructural(microservices: dict) -> dict: + +def classify_internal_infrastructural(dfd: dict): """Classifies processes as either internal or infrastructural. The latter if they are marked as one of the known infrastructural technologies. """ @@ -26,6 +27,8 @@ def classify_internal_infrastructural(microservices: dict) -> dict: "proxy" ] + microservices = dfd["microservices"] + for m in microservices.keys(): infrastructural = False if not "database" in microservices[m]["stereotype_instances"]: @@ -60,5 +63,4 @@ def classify_internal_infrastructural(microservices: dict) -> dict: trace["span"] = "heuristic" traceability.add_trace(trace) - - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/spring_admin/sad_entry.py b/technology_specific_extractors/spring_admin/sad_entry.py index 0e2c276..8068323 100644 --- a/technology_specific_extractors/spring_admin/sad_entry.py +++ b/technology_specific_extractors/spring_admin/sad_entry.py @@ -3,10 +3,13 @@ import output_generators.traceability as traceability -def detect_spring_admin_server(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_admin_server(dfd): """Detects Spring Admin Servers. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + results = fi.search_keywords("@EnableAdminServer") admin_server = False for r in results.keys(): @@ -141,5 +144,5 @@ def detect_spring_admin_server(microservices: dict, information_flows: dict, dfd traceability.add_trace(trace) - - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/spring_config/cnf_entry.py b/technology_specific_extractors/spring_config/cnf_entry.py index fa9508e..874ae31 100644 --- a/technology_specific_extractors/spring_config/cnf_entry.py +++ b/technology_specific_extractors/spring_config/cnf_entry.py @@ -4,21 +4,25 @@ import core.parse_files as parse import technology_specific_extractors.environment_variables as env import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability -def detect_spring_config(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_spring_config(dfd: dict): """Detects Spring Cloud Config server and connections to it. And parses config files. """ - + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] config_server, config_path = False, False microservices, config_server, config_path, config_file_path, config_repo_uri, config_server_ports, config_file_path_local = detect_config_server(microservices, dfd) if config_file_path or config_repo_uri or config_file_path_local: microservices, information_flows, external_components = parse_config_files(config_server, config_file_path, config_file_path_local, config_repo_uri, microservices, information_flows, external_components) microservices, information_flows = detect_config_clients(microservices, information_flows, config_server, config_server_ports) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components def detect_config_server(microservices: dict, dfd): @@ -229,7 +233,7 @@ def parse_config_files(config_server: str, config_file_path: str, config_file_pa # external (other github repository) didn't work, look locally if config_file_path_local: - local_path = tmp.tmp_config["Repository"]["local_path"] + local_path = code2dfd_config["Repository"]["local_path"] config_file_path_local = os.path.relpath(config_file_path_local, start=local_path) new_contents = fi.get_repo_contents_local(config_file_path_local) diff --git a/technology_specific_extractors/spring_encryption/enc_entry.py b/technology_specific_extractors/spring_encryption/enc_entry.py index 1ba796a..1e81a6a 100644 --- a/technology_specific_extractors/spring_encryption/enc_entry.py +++ b/technology_specific_extractors/spring_encryption/enc_entry.py @@ -2,15 +2,19 @@ import core.technology_switch as tech_sw -def detect_spring_encryption(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_encryption(dfd): """Detects use of Spring's crypto module encryption functions. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices = detect_passwordEncoder(microservices, dfd) microservices = detect_bytesEncryptor(microservices, dfd) microservices = detect_keyGenerator(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_passwordEncoder(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/spring_gateway/sgt_entry.py b/technology_specific_extractors/spring_gateway/sgt_entry.py index a31d27c..b2bd8d1 100644 --- a/technology_specific_extractors/spring_gateway/sgt_entry.py +++ b/technology_specific_extractors/spring_gateway/sgt_entry.py @@ -4,10 +4,14 @@ import output_generators.traceability as traceability -def detect_spring_cloud_gateway(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_spring_cloud_gateway(dfd): """Detetcs Spring Cloud Gateway. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + server = False results = fi.search_keywords("spring-cloud-starter-gateway") for r in results.keys(): @@ -84,4 +88,6 @@ def detect_spring_cloud_gateway(microservices: dict, information_flows: dict, ex trace["span"] = prop[2][2] traceability.add_trace(trace) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/technology_specific_extractors/spring_oauth/soa_entry.py b/technology_specific_extractors/spring_oauth/soa_entry.py index bc5484e..7b89b72 100644 --- a/technology_specific_extractors/spring_oauth/soa_entry.py +++ b/technology_specific_extractors/spring_oauth/soa_entry.py @@ -4,16 +4,20 @@ import output_generators.traceability as traceability -def detect_spring_oauth(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_spring_oauth(dfd): """Detect Spring OAuth Server and connections to it. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + microservices = detect_authorization_server(microservices, dfd) microservices = detect_resource_servers(microservices, dfd) microservices, information_flows = detect_token_server(microservices, information_flows, dfd) microservices = detect_preauthorized_methods(microservices, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_authorization_server(microservices: dict, dfd) -> dict: diff --git a/technology_specific_extractors/ssl/ssl_entry.py b/technology_specific_extractors/ssl/ssl_entry.py index 5009d5f..d51b668 100644 --- a/technology_specific_extractors/ssl/ssl_entry.py +++ b/technology_specific_extractors/ssl/ssl_entry.py @@ -1,8 +1,9 @@ -def detect_ssl_services(microservices: dict) -> dict: +def detect_ssl_services(dfd: dict): """Checks if services have ssl enabled. """ + microservices = dfd["microservices"] for m in microservices.keys(): for prop in microservices[m]["properties"]: if prop[0] == "ssl_enabled": @@ -23,4 +24,4 @@ def detect_ssl_services(microservices: dict) -> dict: except: microservices[m]["tagged_values"] = [("SSL Protocol", prop[1])] - return microservices + dfd["microservices"] = microservices diff --git a/technology_specific_extractors/turbine/trb_entry.py b/technology_specific_extractors/turbine/trb_entry.py index 6538dac..cf5689e 100644 --- a/technology_specific_extractors/turbine/trb_entry.py +++ b/technology_specific_extractors/turbine/trb_entry.py @@ -2,19 +2,22 @@ import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp +from core.config import code2dfd_config import output_generators.traceability as traceability -def detect_turbine(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_turbine(dfd): """Detects turbine server. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] microservices = detect_turbine_server(microservices, dfd) microservices, information_flows = detect_turbineamqp(microservices, information_flows, dfd) microservices, information_flows = detect_turbine_stream(microservices, information_flows, dfd) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows def detect_turbine_server(microservices: dict, dfd) -> dict: @@ -155,7 +158,7 @@ def detect_turbine_stream(microservices: dict, information_flows: dict, dfd) -> found_pom = False - local_repo_path = tmp.tmp_config["Repository"]["local_path"] + local_repo_path = code2dfd_config["Repository"]["local_path"] dirs = list() diff --git a/technology_specific_extractors/zipkin/zip_entry.py b/technology_specific_extractors/zipkin/zip_entry.py index 900b723..d93eeb7 100644 --- a/technology_specific_extractors/zipkin/zip_entry.py +++ b/technology_specific_extractors/zipkin/zip_entry.py @@ -1,10 +1,13 @@ import output_generators.traceability as traceability -def detect_zipkin_server(microservices: dict, information_flows: dict, iterative=False) -> dict: +def detect_zipkin_server(dfd: dict, iterative=False): """Detects zipkin server and connections to it. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + zipkin_server_exists, connections_exist = False, False for m in microservices.keys(): @@ -98,8 +101,10 @@ def detect_zipkin_server(microservices: dict, information_flows: dict, iterative microservices[id_]["stereotype_instances"] = ["tracing_server"] microservices[id_]["tagged_values"] = [("Tracing Server", "Zipkin")] - + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows if not iterative: - microservices, information_flows = detect_zipkin_server(microservices, information_flows, True) + detect_zipkin_server(dfd, True) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/zookeeper/zoo_entry.py b/technology_specific_extractors/zookeeper/zoo_entry.py index d080305..74389ec 100644 --- a/technology_specific_extractors/zookeeper/zoo_entry.py +++ b/technology_specific_extractors/zookeeper/zoo_entry.py @@ -1,10 +1,10 @@ -import output_generators.traceability as traceability - - -def detect_zookeeper(microservices: dict, information_flows: dict, dfd) -> dict: +def detect_zookeeper(dfd): """Detects ZooKeeper config services. """ + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + zookeeper_service = False # Service @@ -45,4 +45,5 @@ def detect_zookeeper(microservices: dict, information_flows: dict, dfd) -> dict: for p in to_purge: information_flows.pop(p) - return microservices, information_flows + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows diff --git a/technology_specific_extractors/zuul/zul_entry.py b/technology_specific_extractors/zuul/zul_entry.py index 6b4fcee..f36d479 100644 --- a/technology_specific_extractors/zuul/zul_entry.py +++ b/technology_specific_extractors/zuul/zul_entry.py @@ -1,11 +1,10 @@ import core.external_components as ext import core.file_interaction as fi import core.technology_switch as tech_sw -import tmp.tmp as tmp import output_generators.traceability as traceability -def detect_zuul(microservices: dict, information_flows: dict, external_components: dict, dfd) -> dict: +def detect_zuul(dfd): """Detects Zuul gateway if there is one. """ @@ -13,6 +12,10 @@ def detect_zuul(microservices: dict, information_flows: dict, external_component results = fi.search_keywords("@EnableZuulServer") new_results = fi.search_keywords("@EnableZuulProxy") + microservices = dfd["microservices"] + information_flows = dfd["information_flows"] + external_components = dfd["external_components"] + for r in new_results.keys(): try: id = max(results.keys()) + 1 @@ -112,5 +115,6 @@ def detect_zuul(microservices: dict, information_flows: dict, external_component except: information_flows[id]["tagged_values"] = [("Load Balancer", load_balancer)] - tmp.tmp_config.set("DFD", "external_components", str(external_components).replace("%", "%%")) - return microservices, information_flows, external_components + dfd["microservices"] = microservices + dfd["information_flows"] = information_flows + dfd["external_components"] = external_components diff --git a/tmp/tmp.py b/tmp/tmp.py deleted file mode 100644 index e07827c..0000000 --- a/tmp/tmp.py +++ /dev/null @@ -1,3 +0,0 @@ -from configparser import ConfigParser - -tmp_config = ConfigParser()