- Added updating paths to onboarding template under cli

- Added readme instructions for using cli with example

Author: ravi-databricks

README.md

@@ -69,9 +69,94 @@ With this framework you need to record the source and target metadata in an onbo
 ## Getting Started
 Refer to the [Getting Started](https://databrickslabs.github.io/dlt-meta/getting_started)
 ### Databricks Labs DLT-META CLI lets you run onboard and deploy in interactive python terminal
+- ``` git clone dlt-meta ```
+- ``` cd dlt-meta ```
 - ```databricks labs dlt-meta onboard``` 
+- - Above command will prompt you to provide onboarding details. If you have cloned dlt-meta git repo then accept defaults which will launch config from demo folder
+ ```Provide onboarding file path (default: demo/conf/onboarding.template): 
+    Provide onboarding files local directory (default: demo/): 
+    Provide dbfs path (default: dbfs:/dlt-meta_cli_demo): 
+    Provide databricks runtime version (default: 14.2.x-scala2.12): 
+    Run onboarding with unity catalog enabled?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide unity catalog name: ravi_dlt_meta_uc
+    Provide dlt meta schema name (default: dlt_meta_dataflowspecs_203b9da04bdc49f78cdc6c379d1c9ead): 
+    Provide dlt meta bronze layer schema name (default: dltmeta_bronze_cf5956873137432294892fbb2dc34fdb): 
+    Provide dlt meta silver layer schema name (default: dltmeta_silver_5afa2184543342f98f87b30d92b8c76f): 
+    Provide dlt meta layer
+    [0] bronze
+    [1] bronze_silver
+    [2] silver
+    Enter a number between 0 and 2: 1
+    Provide bronze dataflow spec table name (default: bronze_dataflowspec): 
+    Provide silver dataflow spec table name (default: silver_dataflowspec): 
+    Overwrite dataflow spec?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide dataflow spec version (default: v1): 
+    Provide environment name (default: prod): prod
+    Provide import author name (default: ravi.gawai): 
+    Provide cloud provider name
+    [0] aws
+    [1] azure
+    [2] gcp
+    Enter a number between 0 and 2: 0
+    Do you want to update ws paths, catalog, schema details to your onboarding file?
+    [0] False
+    [1] True
+```
+- Goto your databricks workspace and located onboarding job under: Workflow->Jobs runs
+- Once onboarding jobs is finished deploy `bronze` and `silver` DLT using below command
 - ```databricks labs dlt-meta deploy```
+- - Above command will prompt you to provide dlt details. Please provide respective details for schema which you provided in above steps
+- - Bronze DLT
+```
+    Deploy DLT-META with unity catalog enabled?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide unity catalog name: ravi_dlt_meta_uc
+    Deploy DLT-META with serverless?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide dlt meta layer
+    [0] bronze
+    [1] silver
+    Enter a number between 0 and 1: 0
+    Provide dlt meta onboard group: A1  
+    Provide dlt_meta dataflowspec schema name: dlt_meta_dataflowspecs_203b9da04bdc49f78cdc6c379d1c9ead
+    Provide bronze dataflowspec table name (default: bronze_dataflowspec): 
+    Provide dlt meta pipeline name (default: dlt_meta_bronze_pipeline_2aee3eb837f3439899eef61b76b80d53): 
+    Provide dlt target schema name: dltmeta_bronze_cf5956873137432294892fbb2dc34fdb
+```
 
+- Silver DLT
+- - ```databricks labs dlt-meta deploy```
+- - Above command will prompt you to provide dlt details. Please provide respective details for schema which you provided in above steps
+```
+    Deploy DLT-META with unity catalog enabled?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide unity catalog name: ravi_dlt_meta_uc
+    Deploy DLT-META with serverless?
+    [0] False
+    [1] True
+    Enter a number between 0 and 1: 1
+    Provide dlt meta layer
+    [0] bronze
+    [1] silver
+    Enter a number between 0 and 1: 1
+    Provide dlt meta onboard group: A1
+    Provide dlt_meta dataflowspec schema name: dlt_meta_dataflowspecs_203b9da04bdc49f78cdc6c379d1c9ead
+    Provide silver dataflowspec table name (default: silver_dataflowspec): 
+    Provide dlt meta pipeline name (default: dlt_meta_silver_pipeline_2147545f9b6b4a8a834f62e873fa1364): 
+    Provide dlt target schema name: dltmeta_silver_5afa2184543342f98f87b30d92b8c76f
+```
 ## More questions
 Refer to the [FAQ](https://databrickslabs.github.io/dlt-meta/faq)
 and DLT-META [documentation](https://databrickslabs.github.io/dlt-meta/)

src/cli.py

@@ -43,13 +43,16 @@ class OnboardCommand:
     version: str
     cloud: str
     dlt_meta_schema: str
+    bronze_schema: str = None
+    silver_schema: str = None
     uc_enabled: bool = False
     uc_catalog_name: str = None
     overwrite: bool = True
     bronze_dataflowspec_table: str = "bronze_dataflowspec"
     silver_dataflowspec_table: str = "silver_dataflowspec"
     bronze_dataflowspec_path: str = None
     silver_dataflowspec_path: str = None
+    update_paths: bool = True
 
     def __post_init__(self):
         if not self.onboarding_file_path or self.onboarding_file_path == "":
@@ -147,6 +150,7 @@ def _my_username(self):
 
     def onboard(self, cmd: OnboardCommand):
         """Perform the onboarding process."""
+        self.update_ws_onboarding_paths(cmd)
         if not self._ws.dbfs.exists(cmd.dbfs_path + "/dltmeta_conf/"):
             self._ws.dbfs.create(path=cmd.dbfs_path + "/dltmeta_conf/", overwrite=True)
         ob_file = open(cmd.onboarding_file_path, "rb")
@@ -302,9 +306,9 @@ def deploy(self, cmd: DeployCommand):
     def _load_onboard_config(self) -> OnboardCommand:
         onboard_cmd_dict = {}
         onboard_cmd_dict["onboarding_file_path"] = self._wsi._question(
-            "Provide onboarding file path", default='cli_demo/conf/onboarding.json')
+            "Provide onboarding file path", default='demo/conf/onboarding.template')
         onboarding_files_dir_path = self._wsi._question(
-            "Provide onboarding files local directory", default='cli_demo/conf')
+            "Provide onboarding files local directory", default='demo/')
         onboard_cmd_dict["onboarding_files_dir_path"] = f"file:./{onboarding_files_dir_path}"
         onboard_cmd_dict["dbfs_path"] = self._wsi._question(
             "Provide dbfs path", default="dbfs:/dlt-meta_cli_demo")
@@ -318,6 +322,10 @@ def _load_onboard_config(self) -> OnboardCommand:
                 "Provide unity catalog name")
         onboard_cmd_dict["dlt_meta_schema"] = self._wsi._question(
             "Provide dlt meta schema name", default=f'dlt_meta_dataflowspecs_{uuid.uuid4().hex}')
+        onboard_cmd_dict["bronze_schema"] = self._wsi._question(
+            "Provide dlt meta bronze layer schema name", default=f'dltmeta_bronze_{uuid.uuid4().hex}')
+        onboard_cmd_dict["silver_schema"] = self._wsi._question(
+            "Provide dlt meta silver layer schema name", default=f'dltmeta_silver_{uuid.uuid4().hex}')
         onboard_cmd_dict["onboard_layer"] = self._wsi._choice(
             "Provide dlt meta layer", ['bronze', 'silver', 'bronze_silver'])
         if onboard_cmd_dict["onboard_layer"] == "bronze":
@@ -352,7 +360,11 @@ def _load_onboard_config(self) -> OnboardCommand:
             "Provide import author name", default=self._wsi._short_name)
         onboard_cmd_dict["cloud"] = self._wsi._choice(
             "Provide cloud provider name", ['aws', 'azure', 'gcp'])
+        onboard_cmd_dict["update_paths"] = self._wsi._choice(
+            "Update workspace/dbfs paths, unity catalog name, bronze/silver schema names in onboarding file?",
+            ['True', 'False'])
         cmd = OnboardCommand(**onboard_cmd_dict)
+
         return cmd
 
     def _load_deploy_config(self) -> DeployCommand:
@@ -400,6 +412,48 @@ def _load_deploy_config(self) -> DeployCommand:
             "Provide dlt target schema name")
         return DeployCommand(**deploy_cmd_dict)
 
+    def update_ws_onboarding_paths(self, cmd: OnboardCommand):
+        """Create onboarding file for cloudfiles as source."""
+        with open(f"{cmd.onboarding_file_path}") as f:
+            onboard_obj = json.load(f)
+
+        for data_flow in onboard_obj:
+            for key, value in data_flow.items():
+                if key == "source_details":
+                    for source_key, source_value in value.items():
+                        if 'dbfs_path' in source_value:
+                            data_flow[key][source_key] = source_value.format(dbfs_path=f"{cmd.dbfs_path}/dltmeta_conf/")
+                if 'dbfs_path' in value:
+                    data_flow[key] = value.format(dbfs_path=f"{cmd.dbfs_path}/dltmeta_conf/")
+                elif 'uc_catalog_name' in value and 'bronze_schema' in value:
+                    if cmd.uc_catalog_name:
+                        data_flow[key] = value.format(
+                            uc_catalog_name=cmd.uc_catalog_name,
+                            bronze_schema=cmd.bronze_schema
+                        )
+                    else:
+                        data_flow[key] = value.format(
+                            uc_catalog_name=cmd.bronze_schema,
+                            bronze_schema=""
+                        ).replace(".", "")
+
+                elif 'uc_catalog_name' in value and 'silver_schema' in value:
+                    if cmd.uc_catalog_name:
+                        data_flow[key] = value.format(
+                            uc_catalog_name=cmd.uc_catalog_name,
+                            silver_schema=cmd.silver_schema
+                        )
+                    else:
+                        data_flow[key] = value.format(
+                            uc_catalog_name=cmd.silver_schema,
+                            silver_schema=""
+                        ).replace(".", "")
+        onboarding_filename = os.path.basename(cmd.onboarding_file_path)
+        updated_ob_file_path = cmd.onboarding_file_path.replace(onboarding_filename, "onboarding.json")
+        with open(updated_ob_file_path, "w") as onboarding_file:
+            json.dump(onboard_obj, onboarding_file)
+        cmd.onboarding_file_path = updated_ob_file_path
+
 
 def onboard(dltmeta: DLTMeta):
     logger.info("Please answer a couple of questions to for launching DLT META onboarding job")