Merge pull request #51 from linux-kdevops/cel/fixes

Fixes for recent incomplete patches

Author: chucklever

playbooks/roles/volume_group/tasks/terraform/aws.yml

@@ -10,39 +10,19 @@
 # are not fixed and cannot be depended upon to locate specific
 # devices or their content.
 #
-# The block device name attached to each volume is fixed by
-# terraform but not exposed to Ansible or the guest. It has to be
-# extracted from terraform state and matched to a volume ID, which
-# appears as the device serial number.
+# Instead, rely on a directory created by a udev rule to find the
+# EBS block devices attached on behalf of kdevops. The rule
+# guarantees the symlinks names always point to the same block
+# device.
 #
-# The root EBS device is not included in terraform's block device
-# map, so is easily avoided.
-#
-
-- name: Retrieve the block_device_map from terraform
-  delegate_to: localhost
-  run_once: true
-  ansible.builtin.command:
-    chdir: "{{ topdir_path }}/terraform/aws/"
-    cmd: "terraform output -json block_device_map"
-  register: terraform_output
-  changed_when: false
 
-- name: Exclude the device that will house the /data file system
-  vars:
-    block_device_dict: "{{ terraform_output.stdout | from_json }}"
-    local_map: "{{ block_device_dict[inventory_hostname] }}"
-  ansible.builtin.set_fact:
-    ebs_volume_ids: "{{ ebs_volume_ids + ['nvme-Amazon_Elastic_Block_Store_' + item.value | regex_replace('-', '')] }}"
-  when:
-    item.key != data_device
-  with_dict: "{{ local_map }}"
+- name: Retrieve list of symlinks residing under /dev/disk/kdevops
+  ansible.builtin.find:
+    file_type: link
+    paths: "/dev/disk/kdevops"
+    excludes: "{{ data_device | basename }}"
+  register: find_output
 
-- name: Add unused EBS volumes to the volume list
+- name: Extract device paths from find output
   ansible.builtin.set_fact:
-    physical_volumes: "{{ physical_volumes + ['/dev/' + item.key] }}"
-  when:
-    - item.value.links.ids | intersect(ebs_volume_ids) | list | count > 0
-  loop_control:
-    label: "Adding block device: /dev/{{ item.key }}"
-  with_dict: "{{ ansible_devices }}"
+    physical_volumes: "{{ find_output.files | map(attribute='path') | list }}"

scripts/detect_indentation_issues.py

@@ -34,9 +34,11 @@ def check_file_indentation(file_path):
 
         # Special rules for certain file types
         file_ext = Path(file_path).suffix.lower()
+        file_name = Path(file_path).name
         is_yaml = file_ext in [".yml", ".yaml"]
-        is_makefile = "Makefile" in Path(file_path).name or file_ext == ".mk"
+        is_makefile = "Makefile" in file_name or file_ext == ".mk"
         is_python = file_ext == ".py"
+        is_kconfig = file_name.startswith("Kconfig")
 
         # Check each line for issues
         for line_num, line in enumerate(lines, 1):
@@ -69,6 +71,11 @@ def check_file_indentation(file_path):
                     issues.append(
                         f"Line {line_num}: Tab character in Python file (PEP 8 recommends spaces)"
                     )
+            elif is_kconfig:
+                # Kconfig files use tabs for indentation and tab+spaces for help text
+                # The pattern "\t  " (tab followed by 2 spaces) is valid for help text
+                # So we don't flag this as mixed indentation
+                pass
             else:
                 # For other files, check for mixed indentation
                 if leading_ws:

scripts/dynamic-cloud-kconfig.Makefile

@@ -12,30 +12,51 @@ LAMBDALABS_KCONFIG_IMAGES := $(LAMBDALABS_KCONFIG_DIR)/Kconfig.images.generated
 
 LAMBDALABS_KCONFIGS := $(LAMBDALABS_KCONFIG_COMPUTE) $(LAMBDALABS_KCONFIG_LOCATION) $(LAMBDALABS_KCONFIG_IMAGES)
 
-# Add Lambda Labs generated files to mrproper clean list
-KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS)
+# AWS dynamic configuration
+AWS_KCONFIG_DIR := terraform/aws/kconfigs
+AWS_KCONFIG_AMI := $(AWS_KCONFIG_DIR)/Kconfig.ami
+AWS_KCONFIG_INSTANCE := $(AWS_KCONFIG_DIR)/Kconfig.instance
+AWS_KCONFIG_LOCATION := $(AWS_KCONFIG_DIR)/Kconfig.location
+
+AWS_KCONFIGS := $(AWS_KCONFIG_AMI) $(AWS_KCONFIG_INSTANCE) $(AWS_KCONFIG_LOCATION)
+
+# Add generated files to mrproper clean list
+KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS) $(AWS_KCONFIGS)
 
 # Touch Lambda Labs generated files so Kconfig can source them
 # This ensures the files exist (even if empty) before Kconfig runs
 dynamic_lambdalabs_kconfig_touch:
 	$(Q)touch $(LAMBDALABS_KCONFIGS)
 
-DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch
+# Touch AWS generated files so Kconfig can source them
+dynamic_aws_kconfig_touch:
+	$(Q)touch $(AWS_KCONFIGS)
+
+DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch dynamic_aws_kconfig_touch
 
-# Individual Lambda Labs targets are now handled by generate_cloud_configs.py
+# Lambda Labs targets use --provider argument for efficiency
 cloud-config-lambdalabs:
-	$(Q)python3 scripts/generate_cloud_configs.py
+	$(Q)python3 scripts/generate_cloud_configs.py --provider lambdalabs
+
+# AWS targets use --provider argument for efficiency
+cloud-config-aws:
+	$(Q)python3 scripts/generate_cloud_configs.py --provider aws
 
 # Clean Lambda Labs generated files
 clean-cloud-config-lambdalabs:
 	$(Q)rm -f $(LAMBDALABS_KCONFIGS)
 
-DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs
+# Clean AWS generated files
+clean-cloud-config-aws:
+	$(Q)rm -f $(AWS_KCONFIGS)
+
+DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs cloud-config-aws
 
 cloud-config-help:
 	@echo "Cloud-specific dynamic kconfig targets:"
 	@echo "cloud-config            - generates all cloud provider dynamic kconfig content"
 	@echo "cloud-config-lambdalabs - generates Lambda Labs dynamic kconfig content"
+	@echo "cloud-config-aws        - generates AWS dynamic kconfig content"
 	@echo "clean-cloud-config      - removes all generated cloud kconfig files"
 	@echo "cloud-list-all          - list all cloud instances for configured provider"
 
@@ -44,11 +65,11 @@ HELP_TARGETS += cloud-config-help
 cloud-config:
 	$(Q)python3 scripts/generate_cloud_configs.py
 
-clean-cloud-config: clean-cloud-config-lambdalabs
+clean-cloud-config: clean-cloud-config-lambdalabs clean-cloud-config-aws
 	$(Q)echo "Cleaned all cloud provider dynamic Kconfig files."
 
 cloud-list-all:
 	$(Q)chmod +x scripts/cloud_list_all.sh
 	$(Q)scripts/cloud_list_all.sh
 
-PHONY += cloud-config cloud-config-lambdalabs clean-cloud-config clean-cloud-config-lambdalabs cloud-config-help cloud-list-all
+PHONY += cloud-config cloud-config-lambdalabs cloud-config-aws clean-cloud-config clean-cloud-config-lambdalabs clean-cloud-config-aws cloud-config-help cloud-list-all

scripts/generate_cloud_configs.py

@@ -10,6 +10,7 @@
 import sys
 import subprocess
 import json
+import argparse
 
 
 def generate_lambdalabs_kconfig() -> bool:
@@ -146,16 +147,12 @@ def generate_aws_kconfig() -> bool:
     return all_success
 
 
-def main():
-    """Main function to generate cloud configurations."""
-    print("Cloud Provider Configuration Summary")
-    print("=" * 60)
-    print()
-
-    # Lambda Labs - Generate Kconfig files first
+def process_lambdalabs():
+    """Process Lambda Labs configuration."""
+    # Generate Kconfig files first
     kconfig_generated = generate_lambdalabs_kconfig()
 
-    # Lambda Labs - Get summary
+    # Get summary
     success, summary = get_lambdalabs_summary()
     if success:
         print(f"✓ {summary}")
@@ -167,23 +164,84 @@ def main():
         print(f"⚠ {summary}")
     print()
 
-    # AWS - Generate Kconfig files
+
+def process_aws():
+    """Process AWS configuration."""
     kconfig_generated = generate_aws_kconfig()
     if kconfig_generated:
         print("✓ AWS: Kconfig files generated successfully")
     else:
         print("⚠ AWS: Failed to generate Kconfig files - using defaults")
     print()
 
-    # Azure (placeholder - not implemented)
+
+def process_azure():
+    """Process Azure configuration (placeholder)."""
     print("⚠ Azure: Dynamic configuration not yet implemented")
 
-    # GCE (placeholder - not implemented)
+
+def process_gce():
+    """Process GCE configuration (placeholder)."""
     print("⚠ GCE: Dynamic configuration not yet implemented")
 
-    # OCI (placeholder - not implemented)
+
+def process_oci():
+    """Process OCI configuration (placeholder)."""
     print("⚠ OCI: Dynamic configuration not yet implemented")
 
+
+def main():
+    """Main function to generate cloud configurations."""
+    parser = argparse.ArgumentParser(
+        description="Generate dynamic cloud configurations for supported providers",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s                      # Generate configs for all providers
+  %(prog)s --provider lambdalabs    # Generate configs for Lambda Labs only
+  %(prog)s --provider aws           # Generate configs for AWS only
+  %(prog)s --provider azure         # Generate configs for Azure only
+
+Supported providers: lambdalabs, aws, azure, gce, oci
+        """,
+    )
+
+    parser.add_argument(
+        "--provider",
+        choices=["lambdalabs", "aws", "azure", "gce", "oci"],
+        help="Generate configuration for a specific cloud provider only",
+    )
+
+    args = parser.parse_args()
+
+    # Provider dispatch table
+    providers = {
+        "lambdalabs": process_lambdalabs,
+        "aws": process_aws,
+        "azure": process_azure,
+        "gce": process_gce,
+        "oci": process_oci,
+    }
+
+    print("Cloud Provider Configuration Summary")
+    print("=" * 60)
+    print()
+
+    # If a specific provider is requested, only process that one
+    if args.provider:
+        if args.provider in providers:
+            providers[args.provider]()
+        else:
+            print(f"Error: Unknown provider '{args.provider}'", file=sys.stderr)
+            sys.exit(1)
+    else:
+        # Process all providers
+        process_lambdalabs()
+        process_aws()
+        process_azure()
+        process_gce()
+        process_oci()
+
     print()
     print("Note: Dynamic configurations query real-time availability")
     print("Run 'make menuconfig' to configure your cloud provider")