added retriving enabled services, added graph search from root node

cache_service.py

@@ -107,3 +107,14 @@ def _get_data(self, **kwargs):
         return self.service.datasets().list(
             projectId=kwargs['project_id'],
             pageToken=kwargs.get('pageToken', '')).execute()
+
+
+class ServiceManagement(JsonCacheService):
+    def _get_filename(self, **kwargs):
+        return "cache/{0}/services_{1}.json" \
+            .format(kwargs['project_id'], kwargs.get('pageToken', ''))
+
+    def _get_data(self, **kwargs):
+        return self.service.services().list(
+            consumerId='project:' + kwargs['project_id'],
+            pageToken=kwargs.get('pageToken', '')).execute()

create_iam_graph.py

@@ -1,25 +1,26 @@
 import logging
+import sys
 
 from gcp_iam_iterator import GcpIamIterator
 from visualisation.graph import Node, Edge
 from visualisation import template_renderer
 
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-5s %(filename)-12s %(message)s',
+    level=logging.DEBUG, stream=sys.stdout)
 
-def create_project_nodes(iam_iterator):
-    nodes = []
-    for counter, project in enumerate(iam_iterator.list_projects()):
-        project_id = project['projectId']
-        logging.info("parsing project [{0}] projectId: {1}"
-                     .format(counter, project_id))
-        properties = {k: v for k, v in project.iteritems() if
-                      'projectNumber' in k or 'name' in k or 'createTime' in k}
-        node = Node("project", "p:" + project_id, project_id,
-                    properties=properties)
-        nodes.append(node)
-    return nodes
+
+def get_project_enabled_services(iam_iterator, project_id):
+    iterator = iam_iterator.list_enabled_services(project_id)
+    services = [v['serviceName'][:-15] for v in iterator if
+                v['serviceName'] in ['cloudresourcemanager.googleapis.com',
+                                     'appengine.googleapis.com',
+                                     'deploymentmanager.googleapis.com',
+                                     'servicemanagement.googleapis.com']]
+    return services
 
 
-def create_service_account_nodes(iam_iterator):
+def create_graph(iam_iterator):
     nodes = {}
     edges = []
     for counter, project in enumerate(iam_iterator.list_projects()):
@@ -29,6 +30,8 @@ def create_service_account_nodes(iam_iterator):
         project_properties = {k: v for k, v in project.iteritems() if
                               k in ['projectNumber', 'name', 'createTime',
                                     'projectId']}
+        services = get_project_enabled_services(iam_iterator, project_id)
+        project_properties['enabled_services'] = ", ".join(services)
         project_node = Node("project", "p:" + project_id, project_id,
                             properties=project_properties)
         nodes[project_node.id] = project_node
@@ -60,11 +63,38 @@ def create_service_account_nodes(iam_iterator):
             sa_node = Node("serviceAccount", sa_id, email,
                            properties=sa_properties)
             nodes[sa_node.id] = sa_node
-            edges.append(Edge(sa_node, project_node))
+            edges.append(Edge(project_node, sa_node))
     return nodes.values(), edges
 
 
+def dfs(edges_per_project, start):
+    visited_nodes = set()
+    stack = [start]
+    edges = []
+    while stack:
+        node = stack.pop()
+        if node not in visited_nodes:
+            visited_nodes.add(node)
+            for edge in edges_per_project[node.id]:
+                edges.append(edge)
+                if edge.node_to not in visited_nodes:
+                    stack.append(edge.node_to)
+    return visited_nodes, edges
+
 if __name__ == '__main__':
-    nodes, edges = create_service_account_nodes(GcpIamIterator())
+    nodes, edges = create_graph(GcpIamIterator())
+
+    initial_node = 'p:initial_node'
+
+    edges_per_project = {}
+    for edge in edges:
+        project_edges = edges_per_project.get(edge.node_from.id, [])
+        project_edges.append(edge)
+        edges_per_project[edge.node_from.id] = project_edges
+    nodes_dict = {}
+    for node in nodes:
+        nodes_dict[node.id] = node
+
+    nodes, edges = dfs(edges_per_project, nodes_dict[initial_node])
 
     template_renderer.render(nodes, edges, 'iam_graph.html')

gcp_iam_iterator.py

@@ -3,7 +3,8 @@
 from googleapiclient.errors import HttpError
 
 from cache_service import CRMProjects, CRMProjectIam, ServiceAccountService, \
-    ServiceAccountKeyService, BQDataset, BQDatasets, GCSBuckets, GCSBucketACL
+    ServiceAccountKeyService, BQDataset, BQDatasets, GCSBuckets, GCSBucketACL, \
+    ServiceManagement
 
 from oauth2client.client import GoogleCredentials
 
@@ -16,6 +17,8 @@ def __init__(self):
         google_iam_service = build('iam', 'v1', credentials=credentials)
         google_bq_service = build('bigquery', 'v2', credentials=credentials)
         google_gcs_service = build('storage', 'v1', credentials=credentials)
+        google_service_management = build('servicemanagement', 'v1',
+                                          credentials=credentials)
         self.crm_service = CRMProjects(google_crm_service)
         self.crm_iam_service = CRMProjectIam(google_crm_service)
         self.sa_service = ServiceAccountService(google_iam_service)
@@ -24,6 +27,7 @@ def __init__(self):
         self.dataset_iam_service = BQDataset(google_bq_service)
         self.gcs_service = GCSBuckets(google_gcs_service)
         self.gcs_acl_service = GCSBucketACL(google_gcs_service)
+        self.service_management = ServiceManagement(google_service_management)
 
     def list_projects(self):
         response = self.crm_service.get()
@@ -121,13 +125,13 @@ def list_buckets(self, project_id):
         except HttpError as e:
             if e.resp.status == 400:
                 logging.warning(
-                    "400 received during listing buckets for project_id: {0}, content: {1}"
-                    .format(project_id, e.content))
+                    "400 received during listing buckets for project_id: {0}, "
+                    "content: {1}".format(project_id, e.content))
                 return
             elif e.resp.status == 403:
                 logging.warning(
-                    "403 received during listing buckets for project_id: {0}, content: {1}"
-                    .format(project_id, e.content))
+                    "403 received during listing buckets for project_id: {0}, "
+                    "content: {1}".format(project_id, e.content))
                 return
             else:
                 raise e
@@ -157,3 +161,28 @@ def list_bucket_access(self, bucket_id):
 
         for item in response['items']:
             yield item
+
+    def list_enabled_services(self, project_id):
+        try:
+            response = self.service_management.get(project_id=project_id)
+        except HttpError as e:
+            if e.resp.status == 400 or e.resp.status == 403 \
+                    or e.resp.status == 404:
+                logging.warning(
+                    "{0} received during listing services for project_id: {1}, "
+                    "content: {2}".format(e.resp.status, project_id, e.content))
+                return
+            else:
+                raise e
+
+        while response:
+            if 'services' in response:
+                for service in response['services']:
+                    yield service
+
+            if 'nextPageToken' in response:
+                response = self.service_management.get(project_id=project_id,
+                                                       pageToken=response[
+                                                           'nextPageToken'])
+            else:
+                response = None

visualisation/graph.py

@@ -61,8 +61,8 @@ def __unicode__(self):
 
 class Edge:
     def __init__(self, node_from, node_to, label=None, title=None, role=None):
-        self.nodeFrom = node_from
-        self.nodeTo = node_to
+        self.node_from = node_from
+        self.node_to = node_to
         self.label = label
         self.title = title if title else role
         self.role = role

visualisation/template_renderer.py

@@ -85,8 +85,8 @@ def format_graph(nodes, edges, role_color_map):
 
     for edge in edges:
         value = {
-            'from': node_ids[edge.nodeFrom.id],
-            'to': node_ids[edge.nodeTo.id],
+            'from': node_ids[edge.node_from.id],
+            'to': node_ids[edge.node_to.id],
             'arrows': 'to',
         }
         if edge.label: