Initial changes

.gitignore

@@ -13,7 +13,6 @@ tmp/
 trusted_ips.png
 resource_stats.png
 data/
-private_commands/
 output/
 .vscode/
 Pipfile.lock

Dockerfile

@@ -1,4 +1,7 @@
-FROM python:3.7-slim as cloudmapper
+#FROM python:3.7-slim as cloudmapper
+#FROM alpine:3.17.2
+#FROM python:3.7-alpine3.17
+FROM python:3.9-slim as cloudmapper
 
 LABEL maintainer="https://github.com/0xdabbad00/"
 LABEL Project="https://github.com/duo-labs/cloudmapper"
@@ -10,8 +13,28 @@ ENV AWS_DEFAULT_REGION=us-east-1
 RUN apt-get update -y
 RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq awscli
 RUN apt-get install -y bash
+# RUN apk update && \
+#     apk upgrade && \
+#     apk add --virtual build-dependencies \
+#     build-base \
+#     gcc \
+#     wget \
+#     git 
+# RUN apk add --no-cache \
+#     alpine-sdk \
+#     autoconf \
+#     automake \
+#     bash \
+#     jq \
+#     libtool \
+#     python3-tkinter \
+#     python3-dev \
+#     py-pip && \
+#     pip install --upgrade pip && \
+#     pip install awscli
 
 COPY . /opt/cloudmapper
 RUN pip install -r requirements.txt
 
-RUN bash
+#RUN bash
+ENTRYPOINT "./entrypoint"

Dockerfile-local

@@ -0,0 +1,40 @@
+FROM python:3.9-slim as cloudmapper
+
+LABEL maintainer="https://github.com/0xdabbad00/"
+LABEL Project="https://github.com/duo-labs/cloudmapper"
+
+EXPOSE 8000
+WORKDIR /opt/cloudmapper
+ENV AWS_DEFAULT_REGION=us-east-1 
+
+RUN apt-get update -y
+RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq groff less
+RUN apt-get install -y bash curl unzip
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip 
+RUN ./aws/install
+# RUN apk update && \
+#     apk upgrade && \
+#     apk add --virtual build-dependencies \
+#     build-base \
+#     gcc \
+#     wget \
+#     git 
+# RUN apk add --no-cache \
+#     alpine-sdk \
+#     autoconf \
+#     automake \
+#     bash \
+#     jq \
+#     libtool \
+#     python3-tkinter \
+#     python3-dev \
+#     py-pip && \
+#     pip install --upgrade pip && \
+#     pip install awscli
+
+COPY . /opt/cloudmapper
+RUN pip install -r requirements.txt
+
+RUN bash
+#ENTRYPOINT "./entrypoint"

collect_commands.yaml

@@ -175,6 +175,9 @@
   Request: describe-cluster-subnet-groups
 - Service: sqs
   Request: list-queues
+  Parameters:
+  - Name: MaxResults
+    Value: 1000
 - Service: sqs
   Request: get-queue-attributes
   Parameters:

commands/collect.py

@@ -228,8 +228,8 @@ def collect(arguments):
         default_region = "us-gov-west-1"
     elif "cn-" in default_region:
         default_region = "cn-north-1"
-    else:
-        default_region = "us-east-1"
+    #else:
+    #    default_region = "us-east-1"
 
     regions_filter = None
     if len(arguments.regions_filter) > 0:
@@ -364,7 +364,7 @@ def collect(arguments):
                     parameters[parameter["Name"]] = parameter["Value"]
 
                     # Look for any dynamic values (ones that jq parse a file)
-                    if "|" in parameter["Value"]:
+                    if not isinstance(parameter["Value"], int) and "|" in parameter["Value"]:
                         dynamic_parameter = parameter["Name"]
 
             if runner.get("Custom_collection", False):

entrypoint

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Create the entire file via echo or create it beforehand?
+export AWS_REGION="us-west-2"
+export AWS_CONFIG_FILE=aws-config
+echo "[profile prod]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_PROD_ROLE_ARN" >> aws-config
+echo "" >> aws-config
+
+echo "[profile dev]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_DEV_ROLE_ARN" >> aws-config
+echo "" >> aws-config
+
+echo "[profile solutions-external]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_SOL_EXT_ROLE_ARN" >> aws-config
+echo "" >> aws-config
+
+# Configure config.json for CloudMapper
+python cloudmapper.py configure add-account --config-file config.json --name prod --id $PROD_ID
+python cloudmapper.py configure add-account --config-file config.json --name dev --id $DEV_ID
+python cloudmapper.py configure add-account --config-file config.json --name solutions-external --id $SOL_EXT_ID
+
+# Run CloudMapper
+python cloudmapper.py collect --account prod --profile prod --regions us-west-2 --clean
+python cloudmapper.py stats --accounts prod --stats_all_resources --no_output_image \
+    > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+python cloudmapper.py collect --account dev --profile dev --regions us-west-2 --clean
+python cloudmapper.py stats --accounts dev --stats_all_resources --no_output_image \
+    > dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+python cloudmapper.py collect --account solutions-external --profile solutions-external --regions us-west-2 --clean
+python cloudmapper.py stats --accounts solutions-external --stats_all_resources --no_output_image \
+    > solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+# Output stats results to S3
+aws s3 cp prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
+aws s3 cp dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
+aws s3 cp solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER

private_commands/sqs-delete-unused.py

@@ -0,0 +1,117 @@
+import boto3
+import json
+import logging
+import os
+import re
+import copy
+
+from botocore.exceptions import ClientError
+from commands.collect import get_filename_from_parameter
+from datetime import datetime, timedelta
+from shared.common import get_regions, parse_arguments, custom_serializer
+from shared.nodes import Account, Region
+from shared.query import query_aws, get_parameter_file
+
+def filter_for_unused_queues(queue_urls: list, queue_attributes_objs: list) -> list:
+    unused_queues = [queue_url for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs) \
+                     if queue_attributes_obj['Attributes']['Unused']] #and \
+                     #('Tags' not in queue_attributes_obj['Attributes'] or 'EngTeam' not in queue_attributes_obj['Attributes']['Tags'])]
+    print(f'Filtered {len(queue_urls)} queues down to {len(unused_queues)} unused queues')
+
+    return unused_queues
+
+
+def delete_queues(client: boto3.Session.client, queue_urls: list) -> None:
+    print(f'Deleting {len(queue_urls)} unused queues')
+    for queue_url in queue_urls:
+        try:
+            client.delete_queue(QueueUrl=queue_url)
+        except ClientError as e:
+            print(f'Error deleting queue {queue_url}: {e}')
+
+    return
+
+
+def clean_up_deleted_queue_files(unused_queues: list, saved_sqs_list: dict, account_name: str, profile_name: str, region_name: str) -> None:
+    print(f'Cleaning up {len(unused_queues)} deleted queue files')
+    original_queue_list_len = len(saved_sqs_list['QueueUrls'])
+    sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
+        profile_name, region_name, "sqs", "get-queue-attributes"
+    )
+
+    for queue_url in unused_queues:
+        saved_sqs_list['QueueUrls'].remove(queue_url)
+        filename = get_filename_from_parameter(queue_url)
+        sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
+        if os.path.exists(sqs_attributes_file):
+            os.remove(sqs_attributes_file)
+
+    print(f'Reduced queue list from {original_queue_list_len} to {len(saved_sqs_list["QueueUrls"])}')
+    sqs_queue_list_file = "account-data/{}/{}/{}.json".format(account_name, region_name, "sqs-list-queues")
+    with open(sqs_queue_list_file, "w+") as f:
+        f.write(
+            json.dumps(saved_sqs_list, indent=4, sort_keys=True, default=custom_serializer)
+        )
+
+    return
+
+
+def run(arguments: list) -> None:
+    args, accounts, config = parse_arguments(arguments)
+    logging.getLogger("botocore").setLevel(logging.WARN)
+
+    default_region = os.environ.get("AWS_REGION", "us-east-1")
+
+    session_data = {"region_name": default_region}
+
+    if args.profile:
+        session_data["profile_name"] = args.profile
+
+    session = boto3.Session(**session_data)
+
+    sts = session.client("sts")
+    try:
+        sts.get_caller_identity()
+        logging.debug("Using AWS account: {}".format(sts.get_caller_identity()["Account"]))
+    except ClientError as e:
+        if "InvalidClientTokenId" in str(e):
+            print(
+                "ERROR: sts.get_caller_identity failed with InvalidClientTokenId. Likely cause is no AWS credentials are set.",
+                flush=True,
+            )
+            exit(-1)
+        else:
+            print(
+                "ERROR: Unknown exception when trying to call sts.get_caller_identity: {}".format(
+                    e
+                ),
+                flush=True,
+            )
+            exit(-1)
+
+    sqs = session.client('sqs')
+    for account in accounts:
+        # get_regions reads the file at account-data/{profile}/describe-regions.json
+        for region_json in get_regions(Account(None, account)):
+            region = Region(Account(None, account), region_json)
+            print(f"Processing SQS Queues for Region {region.name}")
+
+            saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
+
+            if 'QueueUrls' not in saved_sqs_list:
+                print(f"No SQS queues found in {region.name}")
+                continue
+
+            queue_urls = saved_sqs_list['QueueUrls']
+            queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls]
+            unused_queues = filter_for_unused_queues(queue_urls, queue_details)
+            if not args.dry_run:
+                delete_queues(sqs, unused_queues)
+                clean_up_deleted_queue_files(unused_queues, saved_sqs_list, account['name'], args.profile, region.name)
+            else:
+                print(f"This operation would delete the following {len(unused_queues)} SQS queues:")
+                print(f"This included {len([queue for queue in unused_queues if 'celery-pidbox' in queue])} celery-pidbox queues.")
+                print(f"Non-celery-pidbox queues are:")
+                for queue in unused_queues:
+                    if 'celery-pidbox' not in queue:
+                        print(queue)