diff --git a/app/main.py b/app/main.py index 1980475c..5068cfdf 100644 --- a/app/main.py +++ b/app/main.py @@ -5,4 +5,5 @@ from app.routes.jcasc import * from app.routes.docker import * from app.routes.jenkins import * -from app.routes.gitlab import * \ No newline at end of file +from app.routes.gitlab import * +from app.routes.grafana_data_sources import * \ No newline at end of file diff --git a/app/media/MyGrafana/mimir.yml b/app/media/MyGrafana/mimir.yml new file mode 100644 index 00000000..4ff9e8c4 --- /dev/null +++ b/app/media/MyGrafana/mimir.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +datasources: +- name: Mimir + uid: mimir + type: prometheus + access: proxy + orgId: 1 + url: http://mimir-nginx.mimir.svc.cluster.local/prometheus + editable: true + version: 1 + jsonData: + alertmanagerUid: alertmanager diff --git a/app/media/grafana_datasources/alertmanager.yml b/app/media/grafana_datasources/alertmanager.yml new file mode 100644 index 00000000..7835d885 --- /dev/null +++ b/app/media/grafana_datasources/alertmanager.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Alertmanager + uid: alertmanager + type: alertmanager + url: http://localhost:9093 + access: proxy + orgId: 1 + jsonData: + # Valid options for implementation include mimir, cortex and prometheus + implementation: [prometheus|cortex|mimir] + # Whether or not Grafana should send alert instances to this Alertmanager + handleGrafanaManagedAlerts: [false|true] + + editable: [true|false] + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/elasticsearch.yml b/app/media/grafana_datasources/elasticsearch.yml new file mode 100644 index 00000000..91a45312 --- /dev/null +++ b/app/media/grafana_datasources/elasticsearch.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: elasticsearch-v7-filebeat + type: elasticsearch + access: proxy + url: http://localhost:9200 + editable: [true|false] + jsonData: + index: '[filebeat-]YYYY.MM.DD' + interval: Daily + timeField: '@timestamp' + logMessageField: message + logLevelField: fields.level diff --git a/app/media/grafana_datasources/loki.yml b/app/media/grafana_datasources/loki.yml new file mode 100644 index 00000000..0dc90212 --- /dev/null +++ b/app/media/grafana_datasources/loki.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + orgId: 1 + access: proxy + editable: [true|false] + url: http://localhost:3100 + jsonData: + timeout: 60 + maxLines: 1000 + + # optionally + basicAuth: true + basicAuthUser: my_user + secureJsonData: + basicAuthPassword: test_password diff --git a/app/media/grafana_datasources/mimir.yml b/app/media/grafana_datasources/mimir.yml new file mode 100644 index 00000000..ebbb9c8c --- /dev/null +++ b/app/media/grafana_datasources/mimir.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Mimir + uid: mimir + type: prometheus + access: proxy + orgId: 1 + url: http://mimir-nginx.mimir.svc.cluster.local/prometheus + editable: [true|false] + version: 1 + jsonData: + httpHeaderName1: "X-Scope-OrgID" + alertmanagerUid: "alertmanager" + secureJsonData: + httpHeaderValue1: "pods" diff --git a/app/media/grafana_datasources/mysql.yml b/app/media/grafana_datasources/mysql.yml new file mode 100644 index 00000000..fc5a6fe5 --- /dev/null +++ b/app/media/grafana_datasources/mysql.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: MySQL + type: mysql + url: localhost:3306 + user: grafana + editable: [true|false] + jsonData: + tlsAuth: true + tlsSkipVerify: true + database: grafana + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + secureJsonData: + password: ${GRAFANA_MYSQL_PASSWORD} + tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT} + tlsCACert: ${GRAFANA_TLS_CA_CERT} diff --git a/app/media/grafana_datasources/postgresql.yml b/app/media/grafana_datasources/postgresql.yml new file mode 100644 index 00000000..bdd8b891 --- /dev/null +++ b/app/media/grafana_datasources/postgresql.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - name: Postgres + type: postgres + url: localhost:5432 + user: grafana # Database user’s login/username + editable: [true|false] + secureJsonData: + password: 'Password!' + jsonData: + database: grafana + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false diff --git a/app/media/grafana_datasources/prometheus.yml b/app/media/grafana_datasources/prometheus.yml new file mode 100644 index 00000000..657a1c98 --- /dev/null +++ b/app/media/grafana_datasources/prometheus.yml @@ -0,0 +1,23 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + # Access mode - proxy (server in the UI) or direct (browser in the UI). + url: http://localhost:9090 + editable: [true|false] + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.44.0 + cacheLevel: 'High' + disableRecordingRules: false + incrementalQueryOverlapWindow: 10m + exemplarTraceIdDestinations: + # Field with internal link pointing to data source in Grafana. + # datasourceUid value can be anything, but it should be unique across all defined data source uids. + - datasourceUid: my_jaeger_uid + name: traceID diff --git a/app/media/grafana_datasources/tempo.yml b/app/media/grafana_datasources/tempo.yml new file mode 100644 index 00000000..cb8794ce --- /dev/null +++ b/app/media/grafana_datasources/tempo.yml @@ -0,0 +1,25 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo-query-frontend.tempo.svc.cluster.local:3100 + basicAuth: false + version: 1 + editable: true + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: # If you are going to link your tracing data with logs, configure <> + datasourceUid: 'loki' + spanStartTimeShift: '-2m' + spanEndTimeShift: '2m' + filterByTraceID: true + filterBySpanID: true + serviceMap: # If you are going to add serviceGraph feature to tempo, configure <> + datasourceUid: 'Mimir-OtelMetrics-Tenant' + nodeGraph: # If you are going to add nodeGraph feature to tempo, enable <> + enabled: true diff --git a/app/models/__init__.py b/app/models/__init__.py index a529a87f..d3b34264 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -6,4 +6,12 @@ from .compose_models import * from .docker_installation_models import * from .jenkins import * -from .gitlab_models import * \ No newline at end of file +from .gitlab_models import * +from app.models.grafana.alert_managers_models import * +from app.models.grafana.elastcsearch_models import * +from app.models.grafana.loki_models import * +from app.models.grafana.mimir_models import * +from app.models.grafana.mysql_models import * +from app.models.grafana.postgresql_models import * +from app.models.grafana.prometheus_models import * +from app.models.grafana.tempo_models import * \ No newline at end of file diff --git a/app/models/grafana/__init__.py b/app/models/grafana/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/grafana/alert_managers_models.py b/app/models/grafana/alert_managers_models.py new file mode 100644 index 00000000..595e99f6 --- /dev/null +++ b/app/models/grafana/alert_managers_models.py @@ -0,0 +1,28 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + +class BasicAuth(BaseModel): + basicAuthUser:str + basicAuthPassword:str + + +class AlertManagerInput(BaseModel): + name:str = "Alertmanager" + url:str = "http://localhost:9093" + uid:str = "alertmanager" + implementation:str + + handleGrafanaManagedAlerts:bool = True + editable: bool = True + basic_auth:Optional[BasicAuth] + + @validator("implementation") + def validator_implementation(cls,value): + valid = ['prometheus','cortex','mimir'] + if value not in valid: + raise ValueError(f"implementation must be in {valid}") + return value + + + + diff --git a/app/models/grafana/elastcsearch_models.py b/app/models/grafana/elastcsearch_models.py new file mode 100644 index 00000000..d8273b8c --- /dev/null +++ b/app/models/grafana/elastcsearch_models.py @@ -0,0 +1,21 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class ElasticSearchInput(BaseModel): + name:str = "elasticsearch-v7-filebeat" + url:str = "http://localhost:9200" + editable: bool = True + index:str = "[filebeat-]YYYY.MM.DD" + interval:str = "Daily" + timeField:str = "@timestamp" + logMessageField:str = "message" + logLevelField:str = "fields.level" + + + + + + + \ No newline at end of file diff --git a/app/models/grafana/loki_models.py b/app/models/grafana/loki_models.py new file mode 100644 index 00000000..43318c86 --- /dev/null +++ b/app/models/grafana/loki_models.py @@ -0,0 +1,20 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + +class BasicAuth(BaseModel): + basicAuthUser:str + basicAuthPassword:str + +class LokiInput(BaseModel): + name:str = "Loki" + uid:str = "loki" + url:str = "http://localhost:3100" + editable: bool = True + timeout:int = 60 + maxLines:int = 1000 + basic_auth:Optional[BasicAuth] + + + + + \ No newline at end of file diff --git a/app/models/grafana/mimir_models.py b/app/models/grafana/mimir_models.py new file mode 100644 index 00000000..7213961f --- /dev/null +++ b/app/models/grafana/mimir_models.py @@ -0,0 +1,17 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class MultiTenancy(BaseModel): + tenant_name:str = "pods" + httpHeaderName1:str = "X-Scope-OrgID" +class MimirInput(BaseModel): + name:str = "Mimir" + uid:str = "mimir" + url:str = "http://mimir-nginx.mimir.svc.cluster.local/prometheus" + editable: bool = True + alertmanagerUid:str = "alertmanager" + multi_tenancy:Optional[MultiTenancy] + + diff --git a/app/models/grafana/mysql_models.py b/app/models/grafana/mysql_models.py new file mode 100644 index 00000000..c3ac157b --- /dev/null +++ b/app/models/grafana/mysql_models.py @@ -0,0 +1,25 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + +class TLS(BaseModel): + tlsClientCert:str = "${GRAFANA_TLS_CLIENT_CERT}" + tlsCACert:str = "${GRAFANA_TLS_CA_CERT}" + tlsAuth:bool = True + tlsSkipVerify:bool = True + +class MysqlInput(BaseModel): + name:str = "MySQL" + url:str = "localhost:3306" + user:str = "grafana" + editable: bool = True + database:str = "grafana" + maxOpenConns:int = 100 + maxIdleConns:int = 100 + maxIdleConnsAuto:bool = True + connMaxLifetime:int = 14400 + password:str = "${GRAFANA_MYSQL_PASSWORD}" + tls :Optional[TLS] + + \ No newline at end of file diff --git a/app/models/grafana/postgresql_models.py b/app/models/grafana/postgresql_models.py new file mode 100644 index 00000000..54bf015c --- /dev/null +++ b/app/models/grafana/postgresql_models.py @@ -0,0 +1,22 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError + + + + +class PostgresInput(BaseModel): + name:str = "Postgres" + url:str = "localhost:5432" + user:str = "grafana" + editable: bool = True + database:str = "grafana" + sslmode:str = "'disable'" + password:str = "Password!" + maxOpenConns:int = 100 + maxIdleConns:int = 100 + maxIdleConnsAuto:bool = True + connMaxLifetime:int = 14400 + postgresVersion:int = 903 + timescaledb:bool = False + + \ No newline at end of file diff --git a/app/models/grafana/prometheus_models.py b/app/models/grafana/prometheus_models.py new file mode 100644 index 00000000..69ad98df --- /dev/null +++ b/app/models/grafana/prometheus_models.py @@ -0,0 +1,17 @@ +from typing import List, Optional +from pydantic import BaseModel, validator, ValidationError,field_validator + + + +class PrometheusInput(BaseModel): + name:str = "Prometheus" + url:str = "http://localhost:9090" + editable: bool = True + httpMethod:str = "POST" + manageAlerts:bool = True + prometheusType:str = "Prometheus" + prometheusVersion:str = "2.44.0" + cacheLevel:str = "High" + disableRecordingRules:bool = False + incrementalQueryOverlapWindow:str = "10m" + \ No newline at end of file diff --git a/app/models/grafana/tempo_models.py b/app/models/grafana/tempo_models.py new file mode 100644 index 00000000..8267d2f3 --- /dev/null +++ b/app/models/grafana/tempo_models.py @@ -0,0 +1,39 @@ +from typing import Optional, List +from pydantic import BaseModel,PrivateAttr,Field + +class TracesToLogsV2(BaseModel): + datasourceUid: str = 'loki' + spanStartTimeShift: str = '-2m' + spanEndTimeShift: str = '2m' + filterByTraceID: bool = True + filterBySpanID: bool = True + +class ServiceMap(BaseModel): + datasourceUid: str = 'Mimir-OtelMetrics-Tenant' + +class NodeGraph(BaseModel): + enabled: bool = True + +class JsonData(BaseModel): + httpMethod: str = 'GET' + tracesToLogsV2: Optional[TracesToLogsV2] = TracesToLogsV2() + serviceMap: Optional[ServiceMap] = ServiceMap() + nodeGraph: Optional[NodeGraph] = NodeGraph() + +class Datasource(BaseModel): + name: str = 'Tempo' + type: str = Field(default='tempo') + access: str = Field(default="proxy") + orgId: int = Field(default=1) + url: str = 'http://tempo-query-frontend.tempo.svc.cluster.local:3100' + basicAuth: bool = False + version: int = Field(default=1) + editable: bool = True + apiVersion: int = Field(default=1) + uid: str = Field(default="tempo") + jsonData: JsonData = JsonData() + +class TempoInput(BaseModel): + apiVersion: int = Field(default=1) + datasources: List[Datasource] = [Datasource()] + diff --git a/app/routes/ansible.py b/app/routes/ansible.py index 367f543c..78c6fe72 100644 --- a/app/routes/ansible.py +++ b/app/routes/ansible.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.routes.utils import add_files_to_folder from app.models import (AnsibleInstallNginx,AnsibleInstallDocker,Output,AnsibleInstallKuber) diff --git a/app/routes/grafana_data_sources.py b/app/routes/grafana_data_sources.py new file mode 100644 index 00000000..57eb7d16 --- /dev/null +++ b/app/routes/grafana_data_sources.py @@ -0,0 +1,103 @@ +from app.app_instance import app +from app.models import (AlertManagerInput,Output,ElasticSearchInput,LokiInput,MimirInput,MysqlInput,PostgresInput, + PrometheusInput,TempoInput) +from app.template_generators.grafana_data_sources.alertmanager import alert_manager_template +from app.template_generators.grafana_data_sources.elasticsearch import elasticsearch_template +from app.template_generators.grafana_data_sources.loki import loki_template +from app.template_generators.grafana_data_sources.mimir import mimir_template +from app.template_generators.grafana_data_sources.mysql import mysql_template +from app.template_generators.grafana_data_sources.postgresql import postgres_template +from app.template_generators.grafana_data_sources.prometheus import pormetheus_template +from app.template_generators.grafana_data_sources.tempo import tempo_template +import shutil +import os + +@app.post("/api/grafana/alertmanager") +async def alertmanager_template_route(request:AlertManagerInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + alert_manager_template(request) + + return Output(output='output') + +@app.post("/api/grafana/elasticsearch") +async def elastic_template_route(request:ElasticSearchInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + elasticsearch_template(request) + + return Output(output='output') + +@app.post("/api/grafana/loki") +async def loki_template_route(request:LokiInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + loki_template(request) + + return Output(output='output') + + +@app.post("/api/grafana/mimir") +async def mimir_template_route(request:MimirInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + mimir_template(request) + + return Output(output='output') + +@app.post("/api/grafana/mysql") +async def mysql_template_route(request:MysqlInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + mysql_template(request) + + return Output(output='output') + +@app.post("/api/grafana/postgres") +async def postgres_template_route(request:PostgresInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + postgres_template(request) + + return Output(output='output') + +@app.post("/api/grafana/prometheus") +async def prometheus_template_route(request:PrometheusInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + pormetheus_template(request) + + return Output(output='output') + + +@app.post("/api/grafana/tempo") +async def tempo_template_route(request:TempoInput) -> Output: + + dir = 'app/media/MyGrafana' + if os.path.exists(dir): + shutil.rmtree(dir) + + tempo_template(request) + + return Output(output='output') \ No newline at end of file diff --git a/app/routes/helm.py b/app/routes/helm.py index dde13d44..aff50423 100644 --- a/app/routes/helm.py +++ b/app/routes/helm.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.models import (HelmTemplateGeneration,Output) from app.prompt_generators import (helm_template_generator) import os diff --git a/app/routes/jcasc.py b/app/routes/jcasc.py index 5ad85ab6..937892d4 100644 --- a/app/routes/jcasc.py +++ b/app/routes/jcasc.py @@ -1,6 +1,6 @@ from app.app_instance import app from app.gpt_services import gpt_service -from app.services import (write_installation,edit_directory_generator,execute_pythonfile) +from app.services import (edit_directory_generator,execute_pythonfile) from app.models import (Jcasc,Output) from app.template_generators.jenkins.jcasc import jcasc_template_generator import os diff --git a/app/routes/terraform.py b/app/routes/terraform.py index 20ebbe03..19aeaf0a 100644 --- a/app/routes/terraform.py +++ b/app/routes/terraform.py @@ -1,7 +1,7 @@ from app.app_instance import app from app.gpt_services import gpt_service from app.services import ( - write_installation, + edit_directory_generator,execute_pythonfile) from app.models import (IaCBasicInput, IaCBugfixInput, diff --git a/app/services.py b/app/services.py index c7e3164a..b21e7304 100644 --- a/app/services.py +++ b/app/services.py @@ -1,37 +1,8 @@ -from .utils import save_to_mongo + import os import shutil from fastapi import HTTPException -def write_basic(request,output): - - data = { - 'question':request.question, - 'output':output - } - - save_to_mongo(data, index='question', collection = 'qa') - - -def write_bugfix(request,output): - data = { - 'bug_description':request.bug_description, - 'service':request.service, - 'output':output - } - - save_to_mongo(data, index=['bug_description','service'], collection = 'bugfix') - - -def write_installation(request,output): - - data = { - 'os':request.os, - 'service':request.service, - 'output':output - } - - save_to_mongo(data, index=['os','service'], collection = 'installation') def edit_directory_generator(gen_file,python_code): diff --git a/app/template_generators/grafana_data_sources/alertmanager.py b/app/template_generators/grafana_data_sources/alertmanager.py new file mode 100644 index 00000000..694783e7 --- /dev/null +++ b/app/template_generators/grafana_data_sources/alertmanager.py @@ -0,0 +1,61 @@ +import yaml +import os + +def alert_manager_template(input): + if input.basic_auth is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "alertmanager", + "url": input.url, + "access": "proxy", + "orgId": 1, + "jsonData": { + "implementation": input.implementation, + "handleGrafanaManagedAlerts": input.handleGrafanaManagedAlerts + }, + "editable": input.editable, + "basicAuth": True, + "basicAuthUser": input.basic_auth.basicAuthUser, + "secureJsonData": { + "basicAuthPassword": input.basic_auth.basicAuthPassword + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'alertmanager.yml') + + file=open("app/media/MyGrafana/alertmanager.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) diff --git a/app/template_generators/grafana_data_sources/elasticsearch.py b/app/template_generators/grafana_data_sources/elasticsearch.py new file mode 100644 index 00000000..92518455 --- /dev/null +++ b/app/template_generators/grafana_data_sources/elasticsearch.py @@ -0,0 +1,36 @@ +import yaml +import os + + +def elasticsearch_template(input): + + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "elasticsearch", + "url": input.url, + "access": "proxy", + + "jsonData": { + "index": input.index, + "interval": input.interval, + "timeField": input.timeField, + "logMessageField": input.logMessageField, + "logLevelField": input.logLevelField, + + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'elasticsearch.yml') + + file=open("app/media/MyGrafana/elasticsearch.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/loki.py b/app/template_generators/grafana_data_sources/loki.py new file mode 100644 index 00000000..0e5ab794 --- /dev/null +++ b/app/template_generators/grafana_data_sources/loki.py @@ -0,0 +1,64 @@ +import yaml +import os + +def loki_template(input): + if input.basic_auth is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "loki", + "orgId": 1, + "url": input.url, + "access": "proxy", + + "jsonData": { + "timeout": input.timeout, + "maxLines": input.maxLines + }, + "editable": input.editable, + + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'loki.yml') + + file=open("app/media/MyGrafana/loki.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "loki", + "url": input.url, + "access": "proxy", + "orgId": 1, + + "jsonData": { + "timeout": input.timeout, + "maxLines": input.maxLines + }, + "editable": input.editable, + "basicAuth": True, + "basicAuthUser": input.basic_auth.basicAuthUser, + "secureJsonData": { + "basicAuthPassword": input.basic_auth.basicAuthPassword + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'loki.yml') + + file=open("app/media/MyGrafana/loki.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + diff --git a/app/template_generators/grafana_data_sources/mimir.py b/app/template_generators/grafana_data_sources/mimir.py new file mode 100644 index 00000000..1237dccd --- /dev/null +++ b/app/template_generators/grafana_data_sources/mimir.py @@ -0,0 +1,67 @@ +import yaml +import os + +def mimir_template(input): + if input.multi_tenancy is not None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "prometheus", + "access": "proxy", + "orgId": 1, + "url": input.url, + "editable": input.editable, + "version": 1, + "jsonData": { + "httpHeaderName1": input.multi_tenancy.httpHeaderName1, + "alertmanagerUid": input.alertmanagerUid + }, + "secureJsonData": { + "httpHeaderValue1": input.multi_tenancy.tenant_name + } + } + ] + } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mimir.yml') + + file=open("app/media/MyGrafana/mimir.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": input.uid, + "type": "prometheus", + "access": "proxy", + "orgId": 1, + "url": input.url, + "editable": input.editable, + "version": 1, + "jsonData": { + "alertmanagerUid": input.alertmanagerUid + }, + + } + ] + } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mimir.yml') + + file=open("app/media/MyGrafana/mimir.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/mysql.py b/app/template_generators/grafana_data_sources/mysql.py new file mode 100644 index 00000000..4a736ffc --- /dev/null +++ b/app/template_generators/grafana_data_sources/mysql.py @@ -0,0 +1,75 @@ +import yaml +import os + +def mysql_template(input): + + if input.tls is None: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "mysql", + "url": input.url, + "user": input.user, + "editable": input.editable, + "jsonData": { + + "database": input.database, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns":input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime + }, + "secureJsonData": { + "password": input.password, + + } + } + ] + } + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mysql.yml') + + file=open("app/media/MyGrafana/mysql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + else: + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "mysql", + "url": input.url, + "user": input.user, + "editable": input.editable, + "jsonData": { + "tlsAuth": input.tls.tlsAuth, + "tlsSkipVerify": input.tls.tlsSkipVerify, + "database": input.database, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns":input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime + }, + "secureJsonData": { + + "password": input.password, + "tlsClientCert": input.tls.tlsClientCert, + "tlsCACert": input.tls.tlsCACert + } + } + ] + } + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'mysql.yml') + + file=open("app/media/MyGrafana/mysql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/postgresql.py b/app/template_generators/grafana_data_sources/postgresql.py new file mode 100644 index 00000000..2e2de924 --- /dev/null +++ b/app/template_generators/grafana_data_sources/postgresql.py @@ -0,0 +1,38 @@ +import yaml +import os + +def postgres_template(input): + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "type": "postgres", + "url": input.url, + "user": input.user, + "editable": input.editable, + "secureJsonData": { + "password": input.password + }, + "jsonData": { + "database": input.database, + "sslmode": input.sslmode, + "maxOpenConns": input.maxOpenConns, + "maxIdleConns": input.maxIdleConns, + "maxIdleConnsAuto": input.maxIdleConnsAuto, + "connMaxLifetime": input.connMaxLifetime, + "postgresVersion": input.postgresVersion, + "timescaledb": input.timescaledb + } + } + ] + } + + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'postgresql.yml') + + file=open("app/media/MyGrafana/postgresql.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/prometheus.py b/app/template_generators/grafana_data_sources/prometheus.py new file mode 100644 index 00000000..dcff822a --- /dev/null +++ b/app/template_generators/grafana_data_sources/prometheus.py @@ -0,0 +1,46 @@ +import yaml +import os +import ruamel.yaml +from ruamel.yaml.scalarstring import ScalarString +import re + +class SingleQuotedScalarString(ScalarString): + def __new__(cls, value): + return ScalarString.__new__(cls, value) + +def pormetheus_template(input): + + + json_template = { + "apiVersion": 1, + "datasources": [ + { + "name": input.name, + "uid": "prometheus", + "type": "prometheus", + "access": "proxy", + "url": input.url, + "editable": input.editable, + "jsonData": { + "httpMethod": input.httpMethod, + "manageAlerts": input.manageAlerts, + "prometheusType": input.prometheusType, + "prometheusVersion": input.prometheusVersion, + "cacheLevel": input.cacheLevel, + "disableRecordingRules": input.disableRecordingRules, + "incrementalQueryOverlapWindow": input.incrementalQueryOverlapWindow, + + } + } + ] + } + + + dir = "app/media/MyGrafana" + os.makedirs(dir) + os.path.join(dir, 'prometheus.yml') + file=open("app/media/MyGrafana/prometheus.yml","w") + yaml.dump(json_template,file,default_flow_style=False,sort_keys=False) + + + \ No newline at end of file diff --git a/app/template_generators/grafana_data_sources/tempo.py b/app/template_generators/grafana_data_sources/tempo.py new file mode 100644 index 00000000..e962f146 --- /dev/null +++ b/app/template_generators/grafana_data_sources/tempo.py @@ -0,0 +1,23 @@ +import yaml +import os + +def remove_none_values(d): + if isinstance(d, dict): + return {k: remove_none_values(v) for k, v in d.items() if v is not None} + elif isinstance(d, list): + return [remove_none_values(i) for i in d if i is not None] + return d + +def tempo_template(input): + dir = 'app/media/MyGrafana' + compose_total = input.dict(exclude_none=True) + + + os.makedirs(dir) + os.path.join(dir, 'tempo.yaml') + + file=open("app/media/MyGrafana/tempo.yaml","w") + yaml.dump(compose_total,file,default_flow_style=False, sort_keys=False) + file.close() + + \ No newline at end of file diff --git a/app/utils.py b/app/utils.py index 39dfdb8e..8b137891 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,38 +1 @@ -import os -from fastapi import HTTPException -from pymongo import MongoClient,ASCENDING,errors - -def get_mongo_client(): - - client = MongoClient(host=os.environ.get('MONGO_HOST'), - port=int(os.environ.get('MONGO_PORT')), - username=os.environ.get('MONGO_INITDB_ROOT_USERNAME'), - password=os.environ.get('MONGO_INITDB_ROOT_PASSWORD')) - - return client - - - -def get_mongo_collection(col:str): - - client = get_mongo_client() - col = client[os.environ.get('MONGO_INITDB_DATABASE')][col] - return col - - -def save_to_mongo(data:dict,index:str,collection:str) -> None: - - try: - - col = get_mongo_collection(collection) - - col.create_index(index, unique=True) - - col.insert_one(data) - - except errors.DuplicateKeyError: - - pass - - diff --git a/requirements.txt b/requirements.txt index 5d269e0e..48e73996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ +ruamel.yaml<0.18.0 fastapi[standard]>=0.113.0,<0.114.0 pydantic>=2.7.0,<3.0.0 openai pytest -pymongo>=4.10.1 +