Merge pull request #361 from 0xdabbad00/tags_for_rds_and_elb

Tags for rds and elb

Author: 0xdabbad00

Pipfile

@@ -22,7 +22,7 @@ autoflake = "==0.7"
 nose = "==1.3.7"
 coverage = "==4.4.2"
 mock = "==2.0.0"
-pylint = "==1.8.1"
+pylint = "==2.3.1"
 
 [requires]
 python_version = "3.7"

cloudmapper.py

@@ -31,7 +31,7 @@
 import importlib 
 import commands
 
-__version__ = "2.5.2"
+__version__ = "2.5.3"
 
 def show_help(commands):
     print("CloudMapper {}".format(__version__))

collect_commands.yaml

@@ -109,10 +109,20 @@
   Parameters:
   - Name: DBSnapshotIdentifier
     Value: rds-describe-db-snapshots.json|.DBSnapshots[]?|.DBSnapshotIdentifier
+- Service: rds
+  Request: list-tags-for-resource
+  Parameters:
+  - Name: ResourceName
+    Value: rds-describe-db-instances.json|.DBInstances[]?|.DBInstanceArn
 - Service: elb
   Request: describe-load-balancers
 - Service: elb
   Request: describe-load-balancer-policies
+- Service: elb
+  Request: describe-tags
+  Parameters:
+  - Name: LoadBalancerNames
+    Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|[[.LoadBalancerName]]
 - Service: elbv2
   Request: describe-load-balancers
 - Service: elbv2
@@ -125,6 +135,11 @@
   Parameters:
   - Name: TargetGroupArn
     Value: elbv2-describe-target-groups/*|.TargetGroups[].TargetGroupArn
+- Service: elbv2
+  Request: describe-tags
+  Parameters:
+  - Name: ResourceArns
+    Value: elbv2-describe-load-balancers.json|.LoadBalancers[]?|[[.LoadBalancerArn]]
 - Service: redshift
   Request: describe-clusters
 - Service: sqs

commands/collect.py

@@ -31,7 +31,14 @@ def get_identifier_from_parameter(parameter):
 
 def get_filename_from_parameter(parameter):
     if isinstance(parameter, list):
-        filename = parameter[1]
+        if len(parameter) > 1:
+            filename = parameter[1]
+        elif isinstance(parameter[0], list):
+            # For elbv2:describe-tags we need ResourceArns as a list like `[Arn]`
+            # the yaml file specifies `[[.LoadBalancerArn]]` because just doing 
+            # `[.LoadBalancerArn]` presents other issues, so this extracts out the inner, inner value.
+            # Similar issue for elb:describe-tags
+            filename = parameter[0][0]
     else:
         filename = parameter
 

commands/prepare.py

@@ -28,7 +28,7 @@
 import argparse
 import pyjq
 from netaddr import IPNetwork, IPAddress
-from shared.common import get_account, query_aws, get_regions, is_external_cidr
+from shared.common import get_account, query_aws, get_parameter_file, get_regions, is_external_cidr
 from shared.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Rds, Cidr, Connection
 
 __description__ = "Generate network connection information file"
@@ -70,6 +70,7 @@ def get_subnets(az):
 
 
 def get_ec2s(subnet, outputfilter):
+    # Filter the EC2s by the `tags`
     tag_filter = ""
     tag_set_conditions = []
     for tag_set in outputfilter.get("tags", []):
@@ -88,7 +89,7 @@ def get_ec2s(subnet, outputfilter):
     return pyjq.all(resource_filter.format(subnet.local_id), instances)
 
 
-def get_elbs(subnet):
+def get_elbs(subnet, outputfilter):
     # ELBs
     elb_instances = query_aws(subnet.account, "elb-describe-load-balancers", subnet.region)
     elb_resource_filter = '.LoadBalancerDescriptions[] | select(.VPCId == "{}") | select(.Subnets[] == "{}")'
@@ -99,13 +100,71 @@ def get_elbs(subnet):
     alb_resource_filter = '.LoadBalancers[] | select(.VpcId == "{}") | select(.AvailabilityZones[].SubnetId == "{}")'
     albs = pyjq.all(alb_resource_filter.format(subnet.vpc.local_id, subnet.local_id), alb_instances)
 
-    return elbs + albs
+    if 'tags' not in outputfilter:
+        return elbs + albs
+
+    # There are tags requested, so we need to filter these
+    tag_filter = ""
+    tag_set_conditions = []
+    for tag_set in outputfilter.get("tags", []):
+        conditions = [c.split("=") for c in tag_set.split(",")]
+        condition_queries = []
+        for pair in conditions:
+            if len(pair) == 2:
+                condition_queries.append('.{} == "{}"'.format(pair[0], pair[1]))
+        tag_set_conditions.append('(' + ' and '.join(condition_queries) + ')')
+    tag_filter = 'select(.TagDescriptions[0].Tags | from_entries | ' + ' or '.join(tag_set_conditions) + ')'
+
+    filtered_elbs = []
+    for elb in elbs:
+        tags = get_parameter_file(subnet.region, 'elb', 'describe-tags', elb['LoadBalancerName'])
+        if tags is None:
+            continue
+
+        if pyjq.first(tag_filter, tags) is not None:
+            filtered_elbs.append(elb)
+
+    for elb in albs:
+        tags = get_parameter_file(subnet.region, 'elbv2', 'describe-tags', elb['LoadBalancerArn'])
+        if tags is None:
+            continue
+
+        if pyjq.first(tag_filter, tags) is not None:
+            filtered_elbs.append(elb)
+
+    return filtered_elbs
 
 
-def get_rds_instances(subnet):
+def get_rds_instances(subnet, outputfilter):
     instances = query_aws(subnet.account, "rds-describe-db-instances", subnet.region)
     resource_filter = '.DBInstances[] | select(.DBSubnetGroup.Subnets != null and .DBSubnetGroup.Subnets[].SubnetIdentifier  == "{}")'
-    return pyjq.all(resource_filter.format(subnet.local_id), instances)
+    rds_instances = pyjq.all(resource_filter.format(subnet.local_id), instances)
+
+    if 'tags' not in outputfilter:
+        return rds_instances
+
+    # There are tags requested, so we need to filter these
+    tag_filter = ""
+    tag_set_conditions = []
+    for tag_set in outputfilter.get("tags", []):
+        conditions = [c.split("=") for c in tag_set.split(",")]
+        condition_queries = []
+        for pair in conditions:
+            if len(pair) == 2:
+                condition_queries.append('.{} == "{}"'.format(pair[0], pair[1]))
+        tag_set_conditions.append('(' + ' and '.join(condition_queries) + ')')
+    tag_filter = 'select(.TagList | from_entries | ' + ' or '.join(tag_set_conditions) + ')'
+
+    filtered_instances = []
+    for rds in rds_instances:
+        tags = get_parameter_file(subnet.region, 'rds', 'list-tags-for-resource', rds['DBInstanceArn'])
+        if tags is None:
+            continue
+
+        if pyjq.first(tag_filter, tags) is not None:
+            filtered_instances.append(rds)
+
+    return filtered_instances
 
 
 def get_sgs(vpc):
@@ -254,14 +313,14 @@ def build_data_structure(account_data, config, outputfilter):
                         subnet.addChild(ec2)
 
                     # Get RDS's
-                    for rds_json in get_rds_instances(subnet):
+                    for rds_json in get_rds_instances(subnet, outputfilter):
                         rds = Rds(subnet, rds_json)
                         if not outputfilter["read_replicas"] and rds.node_type == "rds_rr":
                             continue
                         subnet.addChild(rds)
 
                     # Get ELB's
-                    for elb_json in get_elbs(subnet):
+                    for elb_json in get_elbs(subnet, outputfilter):
                         elb = Elb(subnet, elb_json)
                         subnet.addChild(elb)
 

docs/network_visualizations.md

@@ -14,6 +14,7 @@ There are a number of filtering options that can be applied here to reduce the n
 The most useful filtering options:
 * `--regions`: Restrict the diagram to a set regions, ex. `us-east-1,us-east-2`
 * `--vpc-ids` and `--vpc-names`: Restrict the diagram to a set of VPCs.
+* `--tags`: Filter by tags, for exmaple `--tags Env=Prod --tags Env=Test,Name=Bastion` will filter to all resources tagged with a key `Env` that has value `Prod`, or where `Env=Test` and `Name=Bastion`. In this way, a the tags in a set are AND'd, and the tag sets are OR'd.
 * `--collapse-by-tag`: This is very useful to provide a tag name, and all nodes with that tag will be reduced to a single displayed node.
 
 The other filtering options are: