Skip to content

Commit 38604e5

Browse files
neomatamuneneomatamune
committed
CSM Data Module: Implement CSM data module structure and command implementations
1 parent a8073bd commit 38604e5

32 files changed

+2260
-0
lines changed

‎cosmotech/csm_data/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.

‎cosmotech/csm_data/commands/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.

‎cosmotech/csm_data/commands/adx_send_runnerdata.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.
7+
8+
from cosmotech.csm_data.utils.click import click
9+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
10+
from cosmotech.orchestrator.utils.translate import T
11+
12+
13+
@click.command()
14+
@web_help("csm-data/adx-send-runner-data")
15+
@translate_help("csm-data.commands.storage.adx_send_runnerdata.description")
16+
@click.option(
17+
"--dataset-absolute-path",
18+
envvar="CSM_DATASET_ABSOLUTE_PATH",
19+
show_envvar=True,
20+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.dataset_absolute_path"),
21+
metavar="PATH",
22+
required=True,
23+
)
24+
@click.option(
25+
"--parameters-absolute-path",
26+
envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
27+
metavar="PATH",
28+
show_envvar=True,
29+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.parameters_absolute_path"),
30+
required=True,
31+
)
32+
@click.option(
33+
"--runner-id",
34+
envvar="CSM_RUNNER_ID",
35+
show_envvar=True,
36+
required=True,
37+
metavar="UUID",
38+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.runner_id"),
39+
)
40+
@click.option(
41+
"--adx-uri",
42+
envvar="AZURE_DATA_EXPLORER_RESOURCE_URI",
43+
show_envvar=True,
44+
required=True,
45+
metavar="URI",
46+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.adx_uri"),
47+
)
48+
@click.option(
49+
"--adx-ingest-uri",
50+
envvar="AZURE_DATA_EXPLORER_RESOURCE_INGEST_URI",
51+
show_envvar=True,
52+
required=True,
53+
metavar="URI",
54+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.adx_ingest_uri"),
55+
)
56+
@click.option(
57+
"--database-name",
58+
envvar="AZURE_DATA_EXPLORER_DATABASE_NAME",
59+
show_envvar=True,
60+
required=True,
61+
metavar="NAME",
62+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.database_name"),
63+
)
64+
@click.option(
65+
"--send-parameters/--no-send-parameters",
66+
type=bool,
67+
envvar="CSM_SEND_DATAWAREHOUSE_PARAMETERS",
68+
show_envvar=True,
69+
default=False,
70+
show_default=True,
71+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.send_parameters"),
72+
)
73+
@click.option(
74+
"--send-datasets/--no-send-datasets",
75+
type=bool,
76+
envvar="CSM_SEND_DATAWAREHOUSE_DATASETS",
77+
show_envvar=True,
78+
default=False,
79+
show_default=True,
80+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.send_datasets"),
81+
)
82+
@click.option(
83+
"--wait/--no-wait",
84+
envvar="WAIT_FOR_INGESTION",
85+
show_envvar=True,
86+
default=False,
87+
show_default=True,
88+
help=T("csm-data.commands.storage.adx_send_runnerdata.parameters.wait"),
89+
)
90+
def adx_send_runnerdata(
91+
send_parameters: bool,
92+
send_datasets: bool,
93+
dataset_absolute_path: str,
94+
parameters_absolute_path: str,
95+
runner_id: str,
96+
adx_uri: str,
97+
adx_ingest_uri: str,
98+
database_name: str,
99+
wait: bool,
100+
):
101+
# Import the function at the start of the command
102+
from cosmotech.coal.azure.adx.runner import send_runner_data
103+
104+
# Send runner data to ADX
105+
send_runner_data(
106+
dataset_absolute_path=dataset_absolute_path,
107+
parameters_absolute_path=parameters_absolute_path,
108+
runner_id=runner_id,
109+
adx_uri=adx_uri,
110+
adx_ingest_uri=adx_ingest_uri,
111+
database_name=database_name,
112+
send_parameters=send_parameters,
113+
send_datasets=send_datasets,
114+
wait=wait,
115+
)
116+
117+
118+
if __name__ == "__main__":
119+
adx_send_runnerdata()

‎cosmotech/csm_data/commands/api/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.

‎cosmotech/csm_data/commands/api/api.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.
7+
from cosmotech.csm_data.commands.api.postgres_send_runner_metadata import (
8+
postgres_send_runner_metadata,
9+
)
10+
from cosmotech.csm_data.commands.api.rds_load_csv import rds_load_csv
11+
from cosmotech.csm_data.commands.api.rds_send_csv import rds_send_csv
12+
from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
13+
from cosmotech.csm_data.commands.api.run_load_data import run_load_data
14+
from cosmotech.csm_data.commands.api.runtemplate_load_handler import (
15+
runtemplate_load_handler,
16+
)
17+
from cosmotech.csm_data.commands.api.tdl_load_files import tdl_load_files
18+
from cosmotech.csm_data.commands.api.tdl_send_files import tdl_send_files
19+
from cosmotech.csm_data.commands.api.wsf_load_file import wsf_load_file
20+
from cosmotech.csm_data.commands.api.wsf_send_file import wsf_send_file
21+
from cosmotech.csm_data.utils.click import click
22+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
23+
from cosmotech.coal.cosmotech_api.connection import get_api_client
24+
from cosmotech.coal.utils.logger import LOGGER
25+
from cosmotech.orchestrator.utils.translate import T
26+
27+
28+
@click.group(invoke_without_command=True)
29+
@web_help("csm-data/api")
30+
@click.pass_context
31+
@translate_help("csm-data.commands.api.description")
32+
def api(ctx: click.Context):
33+
if ctx.invoked_subcommand is None:
34+
try:
35+
api_client, description = get_api_client()
36+
LOGGER.info(T("coal.logs.connection.found_valid").format(type=description))
37+
except EnvironmentError:
38+
raise click.Abort()
39+
40+
41+
api.add_command(rds_send_csv, "rds-send-csv")
42+
api.add_command(rds_send_store, "rds-send-store")
43+
api.add_command(rds_load_csv, "rds-load-csv")
44+
api.add_command(wsf_send_file, "wsf-send-file")
45+
api.add_command(wsf_load_file, "wsf-load-file")
46+
api.add_command(tdl_send_files, "tdl-send-files")
47+
api.add_command(tdl_load_files, "tdl-load-files")
48+
api.add_command(runtemplate_load_handler, "runtemplate-load-handler")
49+
api.add_command(run_load_data, "run-load-data")
50+
api.add_command(postgres_send_runner_metadata, "postgres-send-runner-metadata")

‎cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
2+
# This document and all information contained herein is the exclusive property -
3+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
4+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
5+
# etc., to any person is prohibited unless it has been previously and
6+
# specifically authorized by written means by Cosmo Tech.
7+
8+
from cosmotech.csm_data.utils.click import click
9+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
10+
from cosmotech.orchestrator.utils.translate import T
11+
12+
13+
@click.command()
14+
@web_help("csm-data/api/postgres-send-runner-metadata")
15+
@translate_help("csm-data.commands.api.postgres_send_runner_metadata.description")
16+
@click.option(
17+
"--organization-id",
18+
envvar="CSM_ORGANIZATION_ID",
19+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.organization_id"),
20+
metavar="o-XXXXXXXX",
21+
type=str,
22+
show_envvar=True,
23+
required=True,
24+
)
25+
@click.option(
26+
"--workspace-id",
27+
envvar="CSM_WORKSPACE_ID",
28+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.workspace_id"),
29+
metavar="w-XXXXXXXX",
30+
type=str,
31+
show_envvar=True,
32+
required=True,
33+
)
34+
@click.option(
35+
"--runner-id",
36+
envvar="CSM_RUNNER_ID",
37+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.runner_id"),
38+
metavar="r-XXXXXXXX",
39+
type=str,
40+
show_envvar=True,
41+
required=True,
42+
)
43+
@click.option(
44+
"--table-prefix",
45+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.table_prefix"),
46+
metavar="PREFIX",
47+
type=str,
48+
default="Cosmotech_",
49+
)
50+
@click.option(
51+
"--postgres-host",
52+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_host"),
53+
envvar="POSTGRES_HOST_URI",
54+
show_envvar=True,
55+
required=True,
56+
)
57+
@click.option(
58+
"--postgres-port",
59+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_port"),
60+
envvar="POSTGRES_HOST_PORT",
61+
show_envvar=True,
62+
required=False,
63+
default=5432,
64+
)
65+
@click.option(
66+
"--postgres-db",
67+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_db"),
68+
envvar="POSTGRES_DB_NAME",
69+
show_envvar=True,
70+
required=True,
71+
)
72+
@click.option(
73+
"--postgres-schema",
74+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_schema"),
75+
envvar="POSTGRES_DB_SCHEMA",
76+
show_envvar=True,
77+
required=True,
78+
)
79+
@click.option(
80+
"--postgres-user",
81+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_user"),
82+
envvar="POSTGRES_USER_NAME",
83+
show_envvar=True,
84+
required=True,
85+
)
86+
@click.option(
87+
"--postgres-password",
88+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_password"),
89+
envvar="POSTGRES_USER_PASSWORD",
90+
show_envvar=True,
91+
required=True,
92+
)
93+
def postgres_send_runner_metadata(
94+
organization_id,
95+
workspace_id,
96+
runner_id,
97+
table_prefix: str,
98+
postgres_host,
99+
postgres_port,
100+
postgres_db,
101+
postgres_schema,
102+
postgres_user,
103+
postgres_password,
104+
):
105+
# Import the function at the start of the command
106+
from cosmotech.coal.postgresql import send_runner_metadata_to_postgresql
107+
108+
send_runner_metadata_to_postgresql(
109+
organization_id=organization_id,
110+
workspace_id=workspace_id,
111+
runner_id=runner_id,
112+
table_prefix=table_prefix,
113+
postgres_host=postgres_host,
114+
postgres_port=postgres_port,
115+
postgres_db=postgres_db,
116+
postgres_schema=postgres_schema,
117+
postgres_user=postgres_user,
118+
postgres_password=postgres_password,
119+
)

0 commit comments

Comments
 (0)