diff --git a/app/directory_generators/terraform_generator.py b/app/directory_generators/terraform_generator.py index f0c511a9..d6024788 100644 --- a/app/directory_generators/terraform_generator.py +++ b/app/directory_generators/terraform_generator.py @@ -1,196 +1,210 @@ import os project_name = "app/media/MyTerraform" modules_dir = os.path.join(project_name, "modules") -docker_dir = os.path.join(modules_dir, "docker") +argocd_dir = os.path.join(modules_dir, "argocd") # Create project directories -os.makedirs(docker_dir, exist_ok=True) +os.makedirs(argocd_dir, exist_ok=True) # Create main.tf with open(os.path.join(project_name, "main.tf"), "w") as main_file: - main_file.write('''provider "docker" { - host = "unix:///var/run/docker.sock" -} - -module "docker" { - source = "./modules/docker" - - create_image = var.create_image - image_name = var.image_name - image_force_remove = var.image_force_remove - image_build = var.image_build - - create_container = var.create_container - container_image = var.container_image - container_name = var.container_name - container_hostname = var.container_hostname - container_restart = var.container_restart + main_file.write(''' +provider "argocd" { + server_addr = var.argocd_instance_info["server_addr"] + username = var.argocd_instance_info["username"] + password = var.argocd_instance_info["password"] + insecure = var.argocd_instance_info["insecure"] +} + +module "argocd" { + source = "./modules/argocd" + + repository_create = var.repository_create + argocd_repository_info = var.argocd_repository_info + application_create = var.application_create + argocd_application = var.argocd_application + argocd_sync_options = var.argocd_sync_options } ''') # Create variables.tf -with open(os.path.join(project_name, "variables.tf"), "w") as variables_file: - variables_file.write('''variable "create_image" { - type = bool -} - -variable "image_name" { - type = string -} - -variable "image_force_remove" { - type = bool -} - -variable "image_build" { +with open(os.path.join(project_name, "variables.tf"), "w") as vars_file: + vars_file.write(''' +variable "argocd_instance_info" { type = object({ - context = string - tag = list(string) + server_addr = string + username = string + password = string + insecure = bool }) } -variable "create_container" { +variable "repository_create" { type = bool } -variable "container_image" { - type = string +variable "argocd_repository_info" { + type = map(string) } -variable "container_name" { - type = string +variable "application_create" { + type = bool } -variable "container_hostname" { - type = string +variable "argocd_application" { + type = map(string) } -variable "container_restart" { - type = string +variable "argocd_sync_options" { + type = list(string) } ''') # Create terraform.tfvars with open(os.path.join(project_name, "terraform.tfvars"), "w") as tfvars_file: - tfvars_file.write('''create_image = true -image_name = "my-image" -image_force_remove = true -image_build = { - context = "./" - tag = ["my-image:latest"] -} - -create_container = false -container_image = "my-image" -container_name = "my-container" -container_hostname = "my-host" -container_restart = "always" + tfvars_file.write(''' +argocd_instance_info = { + server_addr = "ARGOCD_DOMAIN" + username = "admin" + password = "ARGOCD_ADMIN_PASS" + insecure = true +} + +repository_create = true +argocd_repository_info = { + repo = "https://YOUR_REPO.git" + username = "USERNAME" + password = "CHANGE_ME_WITH_TOKEN" +} + +application_create = true +argocd_application = { + name = "APPLICATION_NAME" + destination_server = "https://kubernetes.default.svc" + destination_namespace = "DESTINATION_NAMESPACE" + source_repo_url = "https://YOUR_REPO.git" + source_path = "SOURCE_PATH" + source_target_revision = "SOURCE_TARGET_REVISION" +} + +argocd_sync_options = ["CreateNamespace=true", "ApplyOutOfSyncOnly=true", "FailOnSharedResource=true"] ''') # Create versions.tf -with open(os.path.join(project_name, "versions.tf"), "w") as versions_file: - versions_file.write('''terraform { +with open(os.path.join(project_name, "versions.tf"), "w") as version_file: + version_file.write(''' +terraform { required_version = ">= 1.0" required_providers { - docker = { - source = "kreuzwerker/docker" - version = ">= 2.8.0" + argocd = { + source = "oboukili/argocd" + version = ">= 6.0.2" } } } ''') # Create module main.tf -with open(os.path.join(docker_dir, "main.tf"), "w") as module_main_file: - module_main_file.write('''resource "docker_image" "image" { - count = var.create_image ? 1 : 0 - name = var.image_name - force_remove = var.image_force_remove - - build { - context = var.image_build.context - tag = var.image_build.tag +with open(os.path.join(argocd_dir, "main.tf"), "w") as module_main_file: + module_main_file.write(''' +resource "argocd_repository" "repository" { + count = var.repository_create ? 1 : 0 + repo = var.argocd_repository_info["repo"] + username = var.argocd_repository_info["username"] + password = var.argocd_repository_info["password"] +} + +resource "argocd_application" "application" { + count = var.application_create ? 1 : 0 + depends_on = [argocd_repository.repository] + + metadata { + name = var.argocd_application["name"] + namespace = "argocd" + labels = { + using_sync_policy_options = "true" + } } -} -resource "docker_container" "container" { - count = var.create_container ? 1 : 0 - image = var.container_image - name = var.container_name - hostname = var.container_hostname - restart = var.container_restart + spec { + destination { + server = var.argocd_application["destination_server"] + namespace = var.argocd_application["destination_namespace"] + } + source { + repo_url = var.argocd_application["source_repo_url"] + path = var.argocd_application["source_path"] + target_revision = var.argocd_application["source_target_revision"] + } + sync_policy { + automated { + prune = true + self_heal = true + } + sync_options = var.argocd_sync_options + } + } } ''') # Create module variables.tf -with open(os.path.join(docker_dir, "variables.tf"), "w") as module_variables_file: - module_variables_file.write('''variable "create_image" { - type = bool -} - -variable "image_name" { - type = string -} - -variable "image_force_remove" { +with open(os.path.join(argocd_dir, "variables.tf"), "w") as module_vars_file: + module_vars_file.write(''' +variable "repository_create" { type = bool } -variable "image_build" { - type = object({ - context = string - tag = list(string) - }) +variable "argocd_repository_info" { + type = map(string) } -variable "create_container" { +variable "application_create" { type = bool } -variable "container_image" { - type = string -} - -variable "container_name" { - type = string -} - -variable "container_hostname" { - type = string +variable "argocd_application" { + type = map(string) } -variable "container_restart" { - type = string +variable "argocd_sync_options" { + type = list(string) } ''') # Create module terraform.tfvars -with open(os.path.join(docker_dir, "terraform.tfvars"), "w") as module_tfvars_file: - module_tfvars_file.write('''create_image = true -image_name = "my-image" -image_force_remove = true -image_build = { - context = "./" - tag = ["my-image:latest"] -} - -create_container = false -container_image = "my-image" -container_name = "my-container" -container_hostname = "my-host" -container_restart = "always" +with open(os.path.join(argocd_dir, "terraform.tfvars"), "w") as module_tfvars_file: + module_tfvars_file.write(''' +repository_create = true +argocd_repository_info = { + repo = "https://YOUR_REPO.git" + username = "USERNAME" + password = "CHANGE_ME_WITH_TOKEN" +} + +application_create = true +argocd_application = { + name = "APPLICATION_NAME" + destination_server = "https://kubernetes.default.svc" + destination_namespace = "DESTINATION_NAMESPACE" + source_repo_url = "https://YOUR_REPO.git" + source_path = "SOURCE_PATH" + source_target_revision = "SOURCE_TARGET_REVISION" +} + +argocd_sync_options = ["CreateNamespace=true", "ApplyOutOfSyncOnly=true", "FailOnSharedResource=true"] ''') # Create module versions.tf -with open(os.path.join(docker_dir, "versions.tf"), "w") as module_versions_file: - module_versions_file.write('''terraform { +with open(os.path.join(argocd_dir, "versions.tf"), "w") as module_version_file: + module_version_file.write(''' +terraform { required_version = ">= 1.0" required_providers { - docker = { - source = "kreuzwerker/docker" - version = ">= 2.8.0" + argocd = { + source = "oboukili/argocd" + version = ">= 6.0.2" } } } diff --git a/app/main.py b/app/main.py index 1980475c..5068cfdf 100644 --- a/app/main.py +++ b/app/main.py @@ -5,4 +5,5 @@ from app.routes.jcasc import * from app.routes.docker import * from app.routes.jenkins import * -from app.routes.gitlab import * \ No newline at end of file +from app.routes.gitlab import * +from app.routes.grafana_data_sources import * \ No newline at end of file diff --git a/app/media/MyBash/bash.sh b/app/media/MyBash/bash.sh index a9bc2e7d..fcb3d783 100644 --- a/app/media/MyBash/bash.sh +++ b/app/media/MyBash/bash.sh @@ -1,6 +1,8 @@ +#!/bin/bash sudo apt update -y sudo apt install -y fontconfig openjdk-17-jre + sudo wget -O /usr/share/keyrings/jenkins-keyring.asc \ https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key echo "deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc]" \ @@ -8,4 +10,4 @@ echo "deb [signed-by=/usr/share/keyrings/jenkins-keyring.asc]" \ /etc/apt/sources.list.d/jenkins.list > /dev/null sudo apt-get update -y sudo apt-get install -y jenkins -sudo systemctl enable --now jenkins +sudo systemctl enable --now jenkins \ No newline at end of file diff --git a/app/media/MyCompose/docker-compose.yaml b/app/media/MyCompose/docker-compose.yaml index fe15d044..efc50a8b 100644 --- a/app/media/MyCompose/docker-compose.yaml +++ b/app/media/MyCompose/docker-compose.yaml @@ -1,31 +1,15 @@ -# sudo mkdir -p /srv/gitlab -# export GITLAB_HOME=/srv/gitlab - -version: '3.6' +version: '3.8' services: - gitlab: - image: gitlab/gitlab-ee:-ee.0 - container_name: gitlab - restart: always - hostname: 'gitlab.example.com' - environment: - GITLAB_OMNIBUS_CONFIG: | - # Add any other gitlab.rb configuration here, each on its own line - external_url 'https://gitlab.example.com' - - # you can also use custom HTTP and SSH port. if you you want to do that, follow the below syntax - - # external_url 'http://gitlab.example.com:8929' - # gitlab_rails['gitlab_shell_ssh_port'] = 2424 - + jenkins: + image: jenkins/jenkins:lts + privileged: true + user: root ports: - # - '8929:8929' # Custom HTTP Port - # - '2424:22' # Custom SSH Port - - '80:80' - - '443:443' - - '22:22' + - 8080:8080 + - 50000:50000 + container_name: jenkins volumes: - - '$GITLAB_HOME/config:/etc/gitlab' - - '$GITLAB_HOME/logs:/var/log/gitlab' - - '$GITLAB_HOME/data:/var/opt/gitlab' - shm_size: '256m' \ No newline at end of file + - /home/${myname}/jenkins_compose/jenkins_configuration:/var/jenkins_home + - /var/run/docker.sock:/var/run/docker.sock + +# Replace "/home/${myname}/jenkins_compose/jenkins_configuration" with the path you want to use to store your jenkins data \ No newline at end of file diff --git a/app/media/MyGrafana/alertmanager.yml b/app/media/MyGrafana/alertmanager.yml new file mode 100644 index 00000000..12ee81b0 --- /dev/null +++ b/app/media/MyGrafana/alertmanager.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +datasources: +- name: Alertmanager + uid: alertmanager + type: alertmanager + url: http://localhost:9093 + access: proxy + orgId: 1 + jsonData: + implementation: prometheus + handleGrafanaManagedAlerts: true + editable: true diff --git a/app/media/MyTerraform/main.tf b/app/media/MyTerraform/main.tf index 30e4c065..9a90ebb7 100644 --- a/app/media/MyTerraform/main.tf +++ b/app/media/MyTerraform/main.tf @@ -1,18 +1,17 @@ -provider "docker" { - host = "unix:///var/run/docker.sock" -} - -module "docker" { - source = "./modules/docker" - create_image = var.create_image - image_name = var.image_name - image_force_remove = var.image_force_remove - image_build = var.image_build +provider "argocd" { + server_addr = var.argocd_instance_info["server_addr"] + username = var.argocd_instance_info["username"] + password = var.argocd_instance_info["password"] + insecure = var.argocd_instance_info["insecure"] +} - create_container = var.create_container - container_image = var.container_image - container_name = var.container_name - container_hostname = var.container_hostname - container_restart = var.container_restart +module "argocd" { + source = "./modules/argocd" + + repository_create = var.repository_create + argocd_repository_info = var.argocd_repository_info + application_create = var.application_create + argocd_application = var.argocd_application + argocd_sync_options = var.argocd_sync_options } diff --git a/app/media/MyTerraform/modules/argocd/main.tf b/app/media/MyTerraform/modules/argocd/main.tf new file mode 100644 index 00000000..68cbe649 --- /dev/null +++ b/app/media/MyTerraform/modules/argocd/main.tf @@ -0,0 +1,39 @@ + +resource "argocd_repository" "repository" { + count = var.repository_create ? 1 : 0 + repo = var.argocd_repository_info["repo"] + username = var.argocd_repository_info["username"] + password = var.argocd_repository_info["password"] +} + +resource "argocd_application" "application" { + count = var.application_create ? 1 : 0 + depends_on = [argocd_repository.repository] + + metadata { + name = var.argocd_application["name"] + namespace = "argocd" + labels = { + using_sync_policy_options = "true" + } + } + + spec { + destination { + server = var.argocd_application["destination_server"] + namespace = var.argocd_application["destination_namespace"] + } + source { + repo_url = var.argocd_application["source_repo_url"] + path = var.argocd_application["source_path"] + target_revision = var.argocd_application["source_target_revision"] + } + sync_policy { + automated { + prune = true + self_heal = true + } + sync_options = var.argocd_sync_options + } + } +} diff --git a/app/media/MyTerraform/modules/argocd/terraform.tfvars b/app/media/MyTerraform/modules/argocd/terraform.tfvars new file mode 100644 index 00000000..d2e9357a --- /dev/null +++ b/app/media/MyTerraform/modules/argocd/terraform.tfvars @@ -0,0 +1,19 @@ + +repository_create = true +argocd_repository_info = { + repo = "https://YOUR_REPO.git" + username = "USERNAME" + password = "CHANGE_ME_WITH_TOKEN" +} + +application_create = true +argocd_application = { + name = "APPLICATION_NAME" + destination_server = "https://kubernetes.default.svc" + destination_namespace = "DESTINATION_NAMESPACE" + source_repo_url = "https://YOUR_REPO.git" + source_path = "SOURCE_PATH" + source_target_revision = "SOURCE_TARGET_REVISION" +} + +argocd_sync_options = ["CreateNamespace=true", "ApplyOutOfSyncOnly=true", "FailOnSharedResource=true"] diff --git a/app/media/MyTerraform/modules/argocd/variables.tf b/app/media/MyTerraform/modules/argocd/variables.tf new file mode 100644 index 00000000..5d76fdc1 --- /dev/null +++ b/app/media/MyTerraform/modules/argocd/variables.tf @@ -0,0 +1,20 @@ + +variable "repository_create" { + type = bool +} + +variable "argocd_repository_info" { + type = map(string) +} + +variable "application_create" { + type = bool +} + +variable "argocd_application" { + type = map(string) +} + +variable "argocd_sync_options" { + type = list(string) +} diff --git a/app/media/MyTerraform/modules/argocd/versions.tf b/app/media/MyTerraform/modules/argocd/versions.tf new file mode 100644 index 00000000..c2fa9111 --- /dev/null +++ b/app/media/MyTerraform/modules/argocd/versions.tf @@ -0,0 +1,11 @@ + +terraform { + required_version = ">= 1.0" + + required_providers { + argocd = { + source = "oboukili/argocd" + version = ">= 6.0.2" + } + } +} diff --git a/app/media/MyTerraform/modules/docker/main.tf b/app/media/MyTerraform/modules/docker/main.tf deleted file mode 100644 index a312214a..00000000 --- a/app/media/MyTerraform/modules/docker/main.tf +++ /dev/null @@ -1,18 +0,0 @@ -resource "docker_image" "image" { - count = var.create_image ? 1 : 0 - name = var.image_name - force_remove = var.image_force_remove - - build { - context = var.image_build.context - tag = var.image_build.tag - } -} - -resource "docker_container" "container" { - count = var.create_container ? 1 : 0 - image = var.container_image - name = var.container_name - hostname = var.container_hostname - restart = var.container_restart -} diff --git a/app/media/MyTerraform/modules/docker/terraform.tfvars b/app/media/MyTerraform/modules/docker/terraform.tfvars deleted file mode 100644 index 033c86d8..00000000 --- a/app/media/MyTerraform/modules/docker/terraform.tfvars +++ /dev/null @@ -1,13 +0,0 @@ -create_image = true -image_name = "my-image" -image_force_remove = true -image_build = { - context = "./" - tag = ["my-image:latest"] -} - -create_container = false -container_image = "my-image" -container_name = "my-container" -container_hostname = "my-host" -container_restart = "always" diff --git a/app/media/MyTerraform/modules/docker/variables.tf b/app/media/MyTerraform/modules/docker/variables.tf deleted file mode 100644 index 5cf1cd69..00000000 --- a/app/media/MyTerraform/modules/docker/variables.tf +++ /dev/null @@ -1,38 +0,0 @@ -variable "create_image" { - type = bool -} - -variable "image_name" { - type = string -} - -variable "image_force_remove" { - type = bool -} - -variable "image_build" { - type = object({ - context = string - tag = list(string) - }) -} - -variable "create_container" { - type = bool -} - -variable "container_image" { - type = string -} - -variable "container_name" { - type = string -} - -variable "container_hostname" { - type = string -} - -variable "container_restart" { - type = string -} diff --git a/app/media/MyTerraform/modules/docker/versions.tf b/app/media/MyTerraform/modules/docker/versions.tf deleted file mode 100644 index 255624a3..00000000 --- a/app/media/MyTerraform/modules/docker/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - docker = { - source = "kreuzwerker/docker" - version = ">= 2.8.0" - } - } -} diff --git a/app/media/MyTerraform/terraform.tfvars b/app/media/MyTerraform/terraform.tfvars index 033c86d8..6c40888d 100644 --- a/app/media/MyTerraform/terraform.tfvars +++ b/app/media/MyTerraform/terraform.tfvars @@ -1,13 +1,26 @@ -create_image = true -image_name = "my-image" -image_force_remove = true -image_build = { - context = "./" - tag = ["my-image:latest"] + +argocd_instance_info = { + server_addr = "ARGOCD_DOMAIN" + username = "admin" + password = "ARGOCD_ADMIN_PASS" + insecure = true +} + +repository_create = true +argocd_repository_info = { + repo = "https://YOUR_REPO.git" + username = "USERNAME" + password = "CHANGE_ME_WITH_TOKEN" +} + +application_create = true +argocd_application = { + name = "APPLICATION_NAME" + destination_server = "https://kubernetes.default.svc" + destination_namespace = "DESTINATION_NAMESPACE" + source_repo_url = "https://YOUR_REPO.git" + source_path = "SOURCE_PATH" + source_target_revision = "SOURCE_TARGET_REVISION" } -create_container = false -container_image = "my-image" -container_name = "my-container" -container_hostname = "my-host" -container_restart = "always" +argocd_sync_options = ["CreateNamespace=true", "ApplyOutOfSyncOnly=true", "FailOnSharedResource=true"] diff --git a/app/media/MyTerraform/variables.tf b/app/media/MyTerraform/variables.tf index 5cf1cd69..6cd53c28 100644 --- a/app/media/MyTerraform/variables.tf +++ b/app/media/MyTerraform/variables.tf @@ -1,38 +1,29 @@ -variable "create_image" { - type = bool -} - -variable "image_name" { - type = string -} -variable "image_force_remove" { - type = bool -} - -variable "image_build" { +variable "argocd_instance_info" { type = object({ - context = string - tag = list(string) + server_addr = string + username = string + password = string + insecure = bool }) } -variable "create_container" { +variable "repository_create" { type = bool } -variable "container_image" { - type = string +variable "argocd_repository_info" { + type = map(string) } -variable "container_name" { - type = string +variable "application_create" { + type = bool } -variable "container_hostname" { - type = string +variable "argocd_application" { + type = map(string) } -variable "container_restart" { - type = string +variable "argocd_sync_options" { + type = list(string) } diff --git a/app/media/MyTerraform/versions.tf b/app/media/MyTerraform/versions.tf index 255624a3..c2fa9111 100644 --- a/app/media/MyTerraform/versions.tf +++ b/app/media/MyTerraform/versions.tf @@ -1,10 +1,11 @@ + terraform { required_version = ">= 1.0" required_providers { - docker = { - source = "kreuzwerker/docker" - version = ">= 2.8.0" + argocd = { + source = "oboukili/argocd" + version = ">= 6.0.2" } } } diff --git a/app/media/grafana_datasources/alertmanager.yml b/app/media/grafana_datasources/alertmanager.yml new file mode 100644 index 00000000..7835d885 --- /dev/null +++ b/app/media/grafana_datasources/alertmanager.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Alertmanager + uid: alertmanager + type: alertmanager + url: http://localhost:9093 + access: proxy + orgId: 1 + jsonData: + # Valid options for implementation include mimir, cortex and prometheus + implementation: [prometheus|cortex|mimir] + # Whether or not Grafana should send alert instances to this Alertmanager + handleGrafanaManagedAlerts: [false|true] + + editable: [true|false] + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/elasticsearch.yml b/app/media/grafana_datasources/elasticsearch.yml new file mode 100644 index 00000000..91a45312 --- /dev/null +++ b/app/media/grafana_datasources/elasticsearch.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: elasticsearch-v7-filebeat + type: elasticsearch + access: proxy + url: http://localhost:9200 + editable: [true|false] + jsonData: + index: '[filebeat-]YYYY.MM.DD' + interval: Daily + timeField: '@timestamp' + logMessageField: message + logLevelField: fields.level diff --git a/app/media/grafana_datasources/loki.yml b/app/media/grafana_datasources/loki.yml new file mode 100644 index 00000000..0dc90212 --- /dev/null +++ b/app/media/grafana_datasources/loki.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + orgId: 1 + access: proxy + editable: [true|false] + url: http://localhost:3100 + jsonData: + timeout: 60 + maxLines: 1000 + + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/mimir.yml b/app/media/grafana_datasources/mimir.yml new file mode 100644 index 00000000..ebbb9c8c --- /dev/null +++ b/app/media/grafana_datasources/mimir.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Mimir + uid: mimir + type: prometheus + access: proxy + orgId: 1 + url: http://mimir-nginx.mimir.svc.cluster.local/prometheus + editable: [true|false] + version: 1 + jsonData: + httpHeaderName1: "X-Scope-OrgID" + alertmanagerUid: "alertmanager" + secureJsonData: + httpHeaderValue1: "pods" diff --git a/app/media/grafana_datasources/mysql.yml b/app/media/grafana_datasources/mysql.yml new file mode 100644 index 00000000..fc5a6fe5 --- /dev/null +++ b/app/media/grafana_datasources/mysql.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: MySQL + type: mysql + url: localhost:3306 + user: grafana + editable: [true|false] + jsonData: + tlsAuth: true + tlsSkipVerify: true + database: grafana + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + secureJsonData: + password: ${GRAFANA_MYSQL_PASSWORD} + tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT} + tlsCACert: ${GRAFANA_TLS_CA_CERT} diff --git a/app/media/grafana_datasources/postgresql.yml b/app/media/grafana_datasources/postgresql.yml new file mode 100644 index 00000000..bdd8b891 --- /dev/null +++ b/app/media/grafana_datasources/postgresql.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Postgres + type: postgres + url: localhost:5432 + user: grafana # Database user’s login/username + editable: [true|false] + secureJsonData: + password: 'Password!' + jsonData: + database: grafana + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false diff --git a/app/media/grafana_datasources/prometheus.yml b/app/media/grafana_datasources/prometheus.yml new file mode 100644 index 00000000..657a1c98 --- /dev/null +++ b/app/media/grafana_datasources/prometheus.yml @@ -0,0 +1,23 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + # Access mode - proxy (server in the UI) or direct (browser in the UI). + url: http://localhost:9090 + editable: [true|false] + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.44.0 + cacheLevel: 'High' + disableRecordingRules: false + incrementalQueryOverlapWindow: 10m + exemplarTraceIdDestinations: + # Field with internal link pointing to data source in Grafana. + # datasourceUid value can be anything, but it should be unique across all defined data source uids. + - datasourceUid: my_jaeger_uid + name: traceID diff --git a/app/media/grafana_datasources/tempo.yml b/app/media/grafana_datasources/tempo.yml new file mode 100644 index 00000000..cb8794ce --- /dev/null +++ b/app/media/grafana_datasources/tempo.yml @@ -0,0 +1,25 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo-query-frontend.tempo.svc.cluster.local:3100 + basicAuth: false + version: 1 + editable: true + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: # If you are going to link your tracing data with logs, configure <> + datasourceUid: 'loki' + spanStartTimeShift: '-2m' + spanEndTimeShift: '2m' + filterByTraceID: true + filterBySpanID: true + serviceMap: # If you are going to add serviceGraph feature to tempo, configure <> + datasourceUid: 'Mimir-OtelMetrics-Tenant' + nodeGraph: # If you are going to add nodeGraph feature to tempo, enable <> + enabled: true diff --git a/app/models/__init__.py b/app/models/__init__.py index a529a87f..a819a217 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -6,4 +6,5 @@ from .compose_models import * from .docker_installation_models import * from .jenkins import * -from .gitlab_models import * \ No newline at end of file +from .gitlab_models import * +from .alert_managers_models import * \ No newline at end of file diff --git a/app/models/alert_managers_models.py b/app/models/alert_managers_models.py new file mode 100644 index 00000000..29539ccc --- /dev/null +++ b/app/models/alert_managers_models.py @@ -0,0 +1,28 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + +class BasicAuth(BaseModel): + basicAuthUser:str + basicAuthPassword:str + + +class AlertManagerInput(BaseModel): + name:str + url:str + uid:str + implementation:str + + handleGrafanaManagedAlerts:bool = True + editable: bool = True + basic_auth:Optional[BasicAuth] + + @validator("implementation") + def validator_implementation(cls,value): + valid = ['prometheus','cortex','mimir'] + if value not in valid: + raise ValueError(f"implementation must be in {valid}") + return value + + + + diff --git a/app/routes/ansible.py b/app/routes/ansible.py index 367f543c..f3b70b7d 100644 --- a/app/routes/ansible.py +++ b/app/routes/ansible.py @@ -13,8 +13,7 @@ @app.post("/api/ansible-install/nginx/") async def ansible_install_generation_nginx(request:AnsibleInstallNginx) -> Output: - - + if os.environ.get("TEST"): return Output(output='output') diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py new file mode 100644 index 00000000..9a7c22a7 --- /dev/null +++ b/app/routes/grafana_data_sources.py @@ -0,0 +1,17 @@ +from app.app_instance import app +from app.models import (AlertManagerInput,Output) +from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template +import shutil +import os + +@app.post("/api/grafana/alertmanager") +async def alertmanager_template(request:AlertManagerInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + alert_manager_template(request) + + return Output(output='output') + diff --git a/app/template_generators/docker/compose.py b/app/template_generators/docker/compose.py index 7a11eb31..34cf3f35 100644 --- a/app/template_generators/docker/compose.py +++ b/app/template_generators/docker/compose.py @@ -1,6 +1,7 @@ import yaml from app.models.compose_models import DockerCompose import os + def remove_none_values(d): if isinstance(d, dict): return {k: remove_none_values(v) for k, v in d.items() if v is not None} diff --git a/app/template_generators/grafana_data_sources/alertmanager.py b/app/template_generators/grafana_data_sources/alertmanager.py new file mode 100644 index 00000000..694783e7 --- /dev/null +++ b/app/template_generators/grafana_data_sources/alertmanager.py @@ -0,0 +1,61 @@ +import yaml +import os + +def alert_manager_template(input): + if input.basic_auth is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + "basicAuth": True, + "basicAuthUser": input.basic_auth.basicAuthUser, + "secureJsonData": { + "basicAuthPassword": input.basic_auth.basicAuthPassword + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) diff --git a/crawl/crawled_data/Terraform Registry.txt b/crawl/crawled_data/Terraform Registry.txt new file mode 100644 index 00000000..f07d8aaa --- /dev/null +++ b/crawl/crawled_data/Terraform Registry.txt @@ -0,0 +1,3 @@ +Title: Terraform Registry + +Please enable Javascript to use this application diff --git a/crawl/main.py b/crawl/main.py index 3a4621e3..c7185dcd 100644 --- a/crawl/main.py +++ b/crawl/main.py @@ -1,11 +1,24 @@ -import requests -from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By import os +import time -# List of URLs to crawl +# Set up headless mode if you don't want to see the browser +chrome_options = Options() +# Uncomment the following line to run Chrome in headless mode +#chrome_options.add_argument("--headless") + +# Provide the path to your ChromeDriver +service = Service('/usr/bin/google-chrome') # Update this path + +# Create a new instance of the Chrome driver +driver = webdriver.Chrome(service=service, options=chrome_options) + +# # List of URLs to crawl urls = [ - "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", - "https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html#current-gen-instances" + "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs" ] # Directory to save the files @@ -14,30 +27,32 @@ def fetch_and_save(url): try: - response = requests.get(url) - response.raise_for_status() # Check if the request was successful - - # Parse the HTML content - soup = BeautifulSoup(response.text, 'html.parser') + driver.get(url) + + # Optionally wait for some time for JavaScript to render (you might need to adjust this) + time.sleep(5) - # For demonstration, we are fetching the page title and all paragraphs - title = soup.title.string if soup.title else "no_title" - paragraphs = soup.find_all('p') + # Fetch the title and all paragraphs + title = driver.title + paragraphs = driver.find_elements(By.TAG_NAME, 'p') # Prepare the file name file_name = os.path.join(save_dir, f"{title}.txt") - + # Write the content to the file with open(file_name, 'w', encoding='utf-8') as file: file.write(f"Title: {title}\n\n") for para in paragraphs: - file.write(para.get_text() + "\n") + file.write(para.text + "\n") print(f"Saved content from {url} to {file_name}") - except requests.RequestException as e: + except Exception as e: print(f"Failed to fetch {url}: {e}") # Fetch and save data from each URL for url in urls: fetch_and_save(url) + +# Close the driver +driver.quit() \ No newline at end of file