Azure enhancements (#8)

* Enhance Azure discovery: parallel subscriptions, checkpointing, resume, and retry logic

- Add parallel processing for multiple Azure subscriptions (--subscription-workers)
- Implement checkpointing to save progress every 50 subs or 15 mins
- Add resume functionality to continue from last checkpoint
- Include retry logic with exponential backoff for failed API calls
- Update README with new features and usage examples
- Improve consistency and reliability for large Azure environments

* Remove legacy resource_count output functionality

- Remove --include-counts argument from all providers
- Remove legacy resource_count file generation code
- Update README to remove references to legacy outputs
- Streamline output to focus on Universal DDI licensing files

Author: stefanriegel

README.md

@@ -25,6 +25,8 @@ A Python tool that discovers and counts DDI Objects and Active IPs across AWS, A
 - **Active IP Tracking**: Identify IP addresses assigned to running instances and services
 - **Flexible Output**: Support for JSON, CSV, and TXT output formats
 - **Parallel Processing**: Configurable worker threads for improved performance
+- **Checkpointing & Resume**: Save progress and resume interrupted discoveries (Azure)
+- **Retry Logic**: Automatic retries for failed API calls to ensure consistency
 - **Modular Design**: Clean separation between cloud providers and shared utilities
 
 ## Quick Start
@@ -60,11 +62,8 @@ source venv/bin/activate  # macOS/Linux
 
 # Run discovery (licensing outputs are generated by default)
 python main.py aws
-python main.py azure
+python main.py azure --subscription-workers 8  # For large Azure environments
 python main.py gcp
-
-# Optional: include legacy resource_count files
-python main.py aws --include-counts
 ```
 
 ## Installation
@@ -175,6 +174,12 @@ python main.py {provider} [options]
 **Options:**
 - `--format {json,csv,txt}` - Output format (default: txt)
 - `--workers <number>` - Number of parallel workers (default: 8)
+- `--subscription-workers <number>` - Parallel subscriptions for Azure (default: 4)
+- `--retry-attempts <number>` - Retry attempts for failed API calls (default: 3)
+- `--no-checkpoint` - Disable checkpointing and resume
+- `--resume` - Auto-resume from checkpoint without prompt
+- `--checkpoint-file <path>` - Custom checkpoint file path
+- `--checkpoint-interval <number>` - Save checkpoint every N subscriptions (default: 50)
 - `--full` - Save detailed resource data (default: summary only)
 
 ### Examples
@@ -186,8 +191,13 @@ python main.py aws
 # High-performance discovery
 python main.py aws --workers 12
 
-# Include legacy count file
-python main.py azure --include-counts
+# Azure with parallel subscriptions and retries
+python main.py azure --subscription-workers 8 --retry-attempts 5
+
+# Resume interrupted discovery
+python main.py azure --resume
+
+
 ```
 
 
@@ -202,7 +212,6 @@ Generated in the `output/` directory:
 - `{provider}_universal_ddi_licensing_{timestamp}.txt` - Human-readable summary
 - `{provider}_universal_ddi_proof_{timestamp}.json` - Audit manifest (scope, regions, hashes)
 - `{provider}_unknown_resources_{timestamp}.json` - Only when unknown types exist
-- Optional legacy: `*_resource_count_*.{format}` via `--include-counts`
 
 ### Output Structure
 

aws_discovery/aws_discovery.py

@@ -140,6 +140,10 @@ def _discover_region(self, region: str) -> List[Dict]:
             load_balancers = self._discover_load_balancers(region)
             region_resources.extend(load_balancers)
 
+            # Discover allocated Elastic IPs (including unattached)
+            elastic_ips = self._discover_elastic_ips(region)
+            region_resources.extend(elastic_ips)
+
         except Exception as e:
             self.logger.error(f"Error discovering region {region}: {e}")
 
@@ -168,6 +172,14 @@ def _discover_ec2_instances(self, region: str) -> List[Dict]:
                         private_ip = instance.get("PrivateIpAddress")
                         public_ip = instance.get("PublicIpAddress")
 
+                        # IPv6 addresses live on the network interfaces
+                        ipv6_ips = []
+                        for nic in instance.get("NetworkInterfaces", []) or []:
+                            for entry in nic.get("Ipv6Addresses", []) or []:
+                                ipv6 = entry.get("Ipv6Address")
+                                if ipv6:
+                                    ipv6_ips.append(ipv6)
+
                         # Get tags
                         tags = get_resource_tags(instance.get("Tags", []))
 
@@ -182,6 +194,7 @@ def _discover_ec2_instances(self, region: str) -> List[Dict]:
                             "state": instance_state,
                             "private_ip": private_ip,
                             "public_ip": public_ip,
+                            "ipv6_ips": ipv6_ips,
                             "vpc_id": instance.get("VpcId"),
                             "subnet_id": instance.get("SubnetId"),
                             "launch_time": instance.get("LaunchTime"),
@@ -288,6 +301,7 @@ def _discover_subnets(self, region: str) -> List[Dict]:
                     details = {
                         "subnet_id": subnet_id,
                         "cidr_block": cidr_block,
+                        "Ipv6CidrBlockAssociationSet": subnet.get("Ipv6CidrBlockAssociationSet", []),
                         "state": state,
                         "vpc_id": vpc_id,
                         "availability_zone": availability_zone,
@@ -381,55 +395,102 @@ def _discover_load_balancers(self, region: str) -> List[Dict]:
         try:
             elb = self.clients[region]["elb"]
 
-            response = elb.describe_load_balancers()
-            for lb in response.get("LoadBalancerDescriptions", []):
-                lb_name = lb.get("LoadBalancerName")
-                if not lb_name:
+            paginator = elb.get_paginator("describe_load_balancers")
+            for response in paginator.paginate():
+                for lb in response.get("LoadBalancerDescriptions", []):
+                    lb_name = lb.get("LoadBalancerName")
+                    if not lb_name:
+                        continue
+
+                    # Get load balancer details
+                    dns_name = lb.get("DNSName")
+                    state = "active" if dns_name else "inactive"
+
+                    # Get tags
+                    try:
+                        tags_response = elb.describe_tags(LoadBalancerNames=[lb_name])
+                        lb_tags = {}
+                        if tags_response.get("TagDescriptions"):
+                            lb_tags = get_resource_tags(tags_response["TagDescriptions"][0].get("Tags", []))
+                    except Exception as e:
+                        self.logger.warning(f"Could not describe tags for {lb_name}: {e}")
+                        lb_tags = {}
+
+                    # Determine if Management Token is required
+                    is_managed = self._is_managed_service(lb_tags)
+                    requires_token = not is_managed
+
+                    # Create resource details
+                    details = {
+                        "load_balancer_name": lb_name,
+                        "dns_name": dns_name,
+                        "state": state,
+                        "vpc_id": lb.get("VPCId"),
+                        "availability_zones": lb.get("AvailabilityZones", []),
+                        "security_groups": lb.get("SecurityGroups", []),
+                    }
+
+                    # Format resource
+                    formatted_resource = self._format_resource(
+                        resource_data=details,
+                        resource_type="classic-load-balancer",
+                        region=region,
+                        name=lb_name,
+                        requires_management_token=requires_token,
+                        state=state,
+                        tags=lb_tags,
+                    )
+
+                    resources.append(formatted_resource)
+
+        except Exception as e:
+            self.logger.warning(f"Error discovering Classic LB in {region}: {e}")
+
+        return resources
+
+    def _discover_elastic_ips(self, region: str) -> List[Dict]:
+        """Discover allocated Elastic IPs (including unattached) in a region."""
+        resources: List[Dict] = []
+        try:
+            ec2 = self.clients[region]["ec2"]
+            resp = ec2.describe_addresses()
+            for addr in resp.get("Addresses", []):
+                public_ip = addr.get("PublicIp")
+                allocation_id = addr.get("AllocationId") or public_ip
+                if not allocation_id:
                     continue
 
-                # Get load balancer details
-                dns_name = lb.get("DNSName")
-                state = "active" if dns_name else "inactive"
-
-                # Get tags
-                try:
-                    tags_response = elb.describe_tags(LoadBalancerNames=[lb_name])
-                    lb_tags = {}
-                    if tags_response.get("TagDescriptions"):
-                        lb_tags = get_resource_tags(tags_response["TagDescriptions"][0].get("Tags", []))
-                except Exception as e:
-                    self.logger.warning(f"Could not describe tags for {lb_name}: {e}")
-                    lb_tags = {}
-
-                # Determine if Management Token is required
-                is_managed = self._is_managed_service(lb_tags)
-                requires_token = not is_managed
-
-                # Create resource details
+                tags = get_resource_tags(addr.get("Tags", []))
+
+                # Elastic IPs are allocations. We record them under the allocated key
+                # so they show up clearly in active IP breakdowns.
                 details = {
-                    "load_balancer_name": lb_name,
-                    "dns_name": dns_name,
-                    "state": state,
-                    "vpc_id": lb.get("VPCId"),
-                    "availability_zones": lb.get("AvailabilityZones", []),
-                    "security_groups": lb.get("SecurityGroups", []),
+                    "allocation_id": addr.get("AllocationId"),
+                    "association_id": addr.get("AssociationId"),
+                    "domain": addr.get("Domain"),
+                    "instance_id": addr.get("InstanceId"),
+                    "network_interface_id": addr.get("NetworkInterfaceId"),
+                    "private_ip": addr.get("PrivateIpAddress"),
+                    "elastic_ip": public_ip,
+                    "public_ipv4_pool": addr.get("PublicIpv4Pool"),
                 }
 
-                # Format resource
-                formatted_resource = self._format_resource(
-                    resource_data=details,
-                    resource_type="classic-load-balancer",
-                    region=region,
-                    name=lb_name,
-                    requires_management_token=requires_token,
-                    state=state,
-                    tags=lb_tags,
-                )
+                state = "associated" if addr.get("AssociationId") or addr.get("InstanceId") else "unassociated"
 
-                resources.append(formatted_resource)
+                resources.append(
+                    self._format_resource(
+                        resource_data=details,
+                        resource_type="elastic-ip",
+                        region=region,
+                        name=str(allocation_id),
+                        requires_management_token=True,
+                        state=state,
+                        tags=tags,
+                    )
+                )
 
         except Exception as e:
-            self.logger.warning(f"Error discovering Classic LB in {region}: {e}")
+            self.logger.warning(f"Error discovering Elastic IPs in {region}: {e}")
 
         return resources
 
@@ -438,53 +499,55 @@ def _discover_route53_zones_and_records(self) -> List[Dict]:
         resources = []
         try:
             route53 = get_aws_client("route53", "us-east-1", self.aws_config)
-            zones_resp = route53.list_hosted_zones()
-            for zone in zones_resp.get("HostedZones", []):
-                zone_id = zone["Id"].split("/")[-1]
-                zone_name = zone["Name"].rstrip(".")
-                is_private = zone.get("Config", {}).get("PrivateZone", False)
-
-                # Add the zone as a resource
-                zone_resource = self._format_resource(
-                    resource_data={
-                        "zone_id": zone_id,
-                        "zone_name": zone_name,
-                        "private": is_private,
-                        "record_set_count": zone.get("ResourceRecordSetCount", 0),
-                    },
-                    resource_type="route53-zone",
-                    region="global",
-                    name=zone_name,
-                    requires_management_token=True,
-                    state="private" if is_private else "public",
-                    tags={},
-                )
-                resources.append(zone_resource)
-
-                # List all records in the zone
-                paginator = route53.get_paginator("list_resource_record_sets")
-                for page in paginator.paginate(HostedZoneId=zone["Id"]):
-                    for record in page.get("ResourceRecordSets", []):
-                        record_type = record.get("Type")
-                        record_name = record.get("Name", "").rstrip(".")
-
-                        record_resource = self._format_resource(
-                            resource_data={
-                                "zone_id": zone_id,
-                                "zone_name": zone_name,
-                                "record_type": record_type,
-                                "record_name": record_name,
-                                "ttl": record.get("TTL"),
-                                "resource_records": record.get("ResourceRecords", []),
-                            },
-                            resource_type="route53-record",
-                            region="global",
-                            name=record_name,
-                            requires_management_token=True,
-                            state=record_type,
-                            tags={},
-                        )
-                        resources.append(record_resource)
+
+            zones_paginator = route53.get_paginator("list_hosted_zones")
+            for zones_resp in zones_paginator.paginate():
+                for zone in zones_resp.get("HostedZones", []):
+                    zone_id = zone["Id"].split("/")[-1]
+                    zone_name = zone["Name"].rstrip(".")
+                    is_private = zone.get("Config", {}).get("PrivateZone", False)
+
+                    # Add the zone as a resource
+                    zone_resource = self._format_resource(
+                        resource_data={
+                            "zone_id": zone_id,
+                            "zone_name": zone_name,
+                            "private": is_private,
+                            "record_set_count": zone.get("ResourceRecordSetCount", 0),
+                        },
+                        resource_type="route53-zone",
+                        region="global",
+                        name=zone_name,
+                        requires_management_token=True,
+                        state="private" if is_private else "public",
+                        tags={},
+                    )
+                    resources.append(zone_resource)
+
+                    # List all records in the zone
+                    paginator = route53.get_paginator("list_resource_record_sets")
+                    for page in paginator.paginate(HostedZoneId=zone["Id"]):
+                        for record in page.get("ResourceRecordSets", []):
+                            record_type = record.get("Type")
+                            record_name = record.get("Name", "").rstrip(".")
+
+                            record_resource = self._format_resource(
+                                resource_data={
+                                    "zone_id": zone_id,
+                                    "zone_name": zone_name,
+                                    "record_type": record_type,
+                                    "record_name": record_name,
+                                    "ttl": record.get("TTL"),
+                                    "resource_records": record.get("ResourceRecords", []),
+                                },
+                                resource_type="route53-record",
+                                region="global",
+                                name=record_name,
+                                requires_management_token=True,
+                                state=record_type,
+                                tags={},
+                            )
+                            resources.append(record_resource)
 
         except Exception as e:
             self.logger.error(f"Error discovering Route 53 zones/records: {e}")

aws_discovery/discover.py

@@ -85,11 +85,7 @@ def main(args=None):
             action="store_true",
             help="Save/export full resource/object data (default: only summary and token calculation)",
         )
-        parser.add_argument(
-            "--include-counts",
-            action="store_true",
-            help="Also write legacy resource_count files alongside licensing outputs",
-        )
+
         args = parser.parse_args()
 
     print("AWS Cloud Discovery for Management Token Calculation")
@@ -204,24 +200,7 @@ def main(args=None):
             print("Results saved to:")
             for file_type, filepath in saved_files.items():
                 print(f"  {file_type}: {filepath}")
-        else:
-            # Save only legacy count file if requested
-            if args.include_counts:
-                output_dir = config.output_directory
-                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-                from shared.output_utils import save_resource_count_results
-
-                summary_files = save_resource_count_results(
-                    count_results,
-                    output_dir,
-                    args.format,
-                    timestamp,
-                    "aws",
-                    extra_info={"accounts": scanned_accounts},
-                )
-                print(f"Summary saved to: {summary_files['resource_count']}")
-            else:
-                print("Skipping legacy resource_count output (use --include-counts to enable)")
+
 
         print("\nDiscovery completed successfully!")