Added databricks asset bundle files

Author: cloudpaldev

databricks.yml

@@ -5,8 +5,8 @@ bundle:
 include:
   - "resources/**/*.yml"
   - "targets/dev.yml"
-  - "targets/test.yml"  
-  - "targets/prod.yml"
+  # - "targets/test.yml"  
+  # - "targets/prod.yml"
 
 variables:
   catalog:
@@ -16,9 +16,15 @@ variables:
   environment:
     description: "Deployment environment"
     default: "dev"
+    
+  trigger_pause_status:
+    description: "Default pause status for scheduled jobs"
+    default: "PAUSED"
 
 # Global permissions
 permissions:
+  - user_name: "pambati@centrilogic.com"
+    level: "CAN_MANAGE"
   - level: "CAN_MANAGE"
     group_name: "iot-data-engineers"
   - level: "CAN_VIEW"

resources/clusters/shared_cluster.yml

@@ -36,16 +36,8 @@ resources:
         dbfs:
           destination: "dbfs:/cluster-logs/${var.environment}"
 
-      # Install required libraries
-      libraries:
-        - pypi:
-            package: "requests>=2.28.0"
-        - pypi:
-            package: "pandas>=1.5.0"
-        - pypi:
-            package: "numpy>=1.24.0"
-        - pypi:
-            package: "delta-spark>=2.4.0"
+      # Libraries should be installed via job configuration or init scripts
+      # libraries field is not supported in clusters resource
 
       # Custom tags for cost tracking
       custom_tags:

resources/jobs/config_job.yml

@@ -34,7 +34,7 @@ resources:
           description: "Create configuration tables for the environment"
           job_cluster_key: "config_cluster"
           notebook_task:
-            notebook_path: "notebooks/config/create_config_tables"
+            notebook_path: "notebooks/config/create_config_tables.py"
             source: "WORKSPACE"
             base_parameters:
               catalog: "${var.catalog}"
@@ -52,12 +52,8 @@ resources:
       # Performance optimization
       performance_target: "STANDARD"
 
-      # Access control
-      access_control_list:
-        - user_name: "odiakonova@seaspancorp.com"
-          permission_level: "IS_OWNER"
-        - group_name: "iot-data-engineers"
-          permission_level: "CAN_MANAGE_RUN"
+      # Access control should be managed via permissions in bundle configuration
+      # access_control_list field is not supported in job resources
 
       # Tags for cost tracking and organization
       tags:

resources/jobs/iot_ingestion_job.yml

@@ -19,11 +19,11 @@ resources:
         on_success:
           - id: "slack-webhook"
 
-      # Schedule configuration
+      # Schedule configuration (environment-specific)
       schedule:
         quartz_cron_expression: "0 0/2 * * * ?"
         timezone_id: "UTC"
-        pause_status: "${var.trigger_pause_status}"
+        pause_status: "PAUSED"  # Default to paused, override in target configs
 
       # Job cluster configuration
       job_clusters:
@@ -53,7 +53,7 @@ resources:
           description: "Load data from IoT API endpoints"
           job_cluster_key: "iot_processing_cluster"
           notebook_task:
-            notebook_path: "notebooks/iot_ingestion/api_ingestion_framework"
+            notebook_path: "notebooks/iot_ingestion/api_ingestion_framework.py"
             source: "WORKSPACE"
             base_parameters:
               catalog: "${var.catalog}"
@@ -70,7 +70,7 @@ resources:
             - task_key: "load_api_data"
           job_cluster_key: "iot_processing_cluster"
           notebook_task:
-            notebook_path: "notebooks/iot_ingestion/load_data_into_bronze"
+            notebook_path: "notebooks/iot_ingestion/load_data_into_bronze.py"
             source: "WORKSPACE"
             base_parameters:
               catalog: "${var.catalog}"
@@ -88,7 +88,7 @@ resources:
             - task_key: "load_to_bronze"
           job_cluster_key: "iot_processing_cluster" 
           notebook_task:
-            notebook_path: "notebooks/iot_ingestion/create_final_bronze_table"
+            notebook_path: "notebooks/iot_ingestion/create_final_bronze_table.py"
             source: "WORKSPACE"
             base_parameters:
               catalog: "${var.catalog}"
@@ -101,14 +101,8 @@ resources:
           max_retries: 1
           min_retry_interval_millis: 60000
 
-      # Access control
-      access_control_list:
-        - user_name: "odiakonova@seaspancorp.com"
-          permission_level: "IS_OWNER"
-        - group_name: "iot-data-engineers"
-          permission_level: "CAN_MANAGE_RUN"
-        - group_name: "iot-analysts"
-          permission_level: "CAN_VIEW"
+      # Access control should be managed via permissions in bundle configuration
+      # access_control_list field is not supported in job resources
 
       # Tags for cost tracking and organization
       tags:

targets/dev.yml

@@ -1,34 +1,28 @@
-# Development Environment Configuration
-variables:
-  catalog: "dev_test"
-  environment: "dev"
+targets:
+  dev:
+    # Development Environment Configuration
+    variables:
+      catalog:
+        default: "dev_test"
+      environment:
+        default: "dev"
 
-workspace:
-  host: "https://adb-dev-88474350318135.15.azuredatabricks.net"  # Dev workspace
-  root_path: "/Users/${workspace.current_user}/.bundle/${bundle.name}/${bundle.target}"
+    workspace:
+      host: "https://adb-787779827200774.14.azuredatabricks.net/"  # Dev workspace
+      root_path: "/Workspace/Shared/.bundle/${bundle.name}/${bundle.target}"
+    
+    sync:
+      include:
+        - "notebooks/**"
 
-mode: development
-default: true
+    presets:
+      name_prefix: "dev-"
+      trigger_pause_status: "PAUSED"
+      jobs_max_concurrent_runs: 1
+      tags:
+        environment: "dev"
+        cost_center: "analytics"
+        project: "iot-ingestion"
 
-presets:
-  name_prefix: "dev-"
-  trigger_pause_status: "PAUSED"
-  jobs_max_concurrent_runs: 1
-  tags:
-    environment: "dev"
-    cost_center: "analytics"
-    project: "iot-ingestion"
-
-# Override job settings for development
-resources:
-  jobs:
-    iot_ingestion_job:
-      schedule:
-        pause_status: "PAUSED"
-      job_clusters:
-        - job_cluster_key: "iot_processing_cluster"
-          new_cluster:
-            num_workers: 1
-            autoscale:
-              min_workers: 1
-              max_workers: 2 
+# Job overrides should be handled via variables or presets
+# Remove duplicate job definitions to avoid conflicts 

targets/prod.yml

@@ -1,13 +1,15 @@
 # Production Environment Configuration
 variables:
-  catalog: "prod"
-  environment: "prod"
+  catalog:
+    default: "prod"
+  environment:
+    default: "prod"
 
 workspace:
   host: "https://adb-prod-88474350318135.15.azuredatabricks.net"  # Production workspace
-  root_path: "/Shared/bundles/${bundle.name}/${bundle.target}"
+  root_path: "/Workspace/Shared/.bundle/${bundle.name}/${bundle.target}"
 
-mode: production
+# Databricks Asset Bundle target configuration
 
 presets:
   name_prefix: "prod-"
@@ -29,24 +31,5 @@ permissions:
   - level: "CAN_VIEW"
     group_name: "business-analysts"
 
-# Override job settings for production
-resources:
-  jobs:
-    iot_ingestion_job:
-      schedule:
-        quartz_cron_expression: "0 0/2 * * * ?"
-        timezone_id: "UTC"
-        pause_status: "UNPAUSED"
-      job_clusters:
-        - job_cluster_key: "iot_processing_cluster"
-          new_cluster:
-            num_workers: 4
-            autoscale:
-              min_workers: 2
-              max_workers: 8
-            enable_elastic_disk: true
-            
-    config_tables_job:
-      # Config job typically runs on-demand in production
-      schedule:
-        pause_status: "PAUSED" 
+# Job overrides should be handled via variables or presets
+# Remove duplicate job definitions to avoid conflicts 

targets/test.yml

@@ -1,13 +1,15 @@
 # Test Environment Configuration
 variables:
-  catalog: "iot_test"
-  environment: "test"
+  catalog:
+    default: "iot_test"
+  environment:
+    default: "test"
 
 workspace:
   host: "https://adb-test-88474350318135.15.azuredatabricks.net"  # Test workspace
-  root_path: "/Shared/bundles/${bundle.name}/${bundle.target}"
+  root_path: "/Workspace/Shared/.bundle/${bundle.name}/${bundle.target}"
 
-mode: development
+# Databricks Asset Bundle target configuration
 
 presets:
   name_prefix: "test-"
@@ -22,20 +24,5 @@ presets:
 run_as:
   service_principal_name: "12345678-1234-1234-1234-123456789012"  # Shared service principal client ID
 
-# Override job settings for test environment
-resources:
-  jobs:
-    iot_ingestion_job:
-      schedule:
-        pause_status: "PAUSED"
-      job_clusters:
-        - job_cluster_key: "iot_processing_cluster"
-          new_cluster:
-            num_workers: 2
-            autoscale:
-              min_workers: 1
-              max_workers: 3
-            
-    config_tables_job:
-      schedule:
-        pause_status: "PAUSED" 
+# Job overrides should be handled via variables or presets
+# Remove duplicate job definitions to avoid conflicts