Skip to content

Commit f7381ac

Browse files
tyler-dunkeltyler-dunkel
authored
Merge pull request #169 from cloudgraphdev/aa-pages
fix: get all pages from ecs,efs,elb,redshift,ses
2 parents 19c75b0 + e63f664 commit f7381ac

File tree

8 files changed

+319
-256
lines changed

8 files changed

+319
-256
lines changed

‎src/services/ecsContainer/data.ts‎

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,30 +43,42 @@ export default async ({
4343
/**
4444
* Get the instance arns of all the containers in each cluster
4545
*/
46-
let containerInstanceArns: any = await Promise.all(
46+
const containerInstanceArns: any = await Promise.all(
4747
ecsClusters.map(
4848
async ({ clusterName: cluster, region }) =>
49-
new Promise(resolveEcsData =>
50-
new ECS({ ...config, region, endpoint }).listContainerInstances(
51-
{ cluster },
52-
(err, data) => {
53-
if (err) {
54-
errorLog.generateAwsErrorLog({
55-
functionName: 'ecs:listContainerInstances',
56-
err,
57-
})
58-
}
59-
60-
if (isEmpty(data)) {
61-
return resolveEcsData([])
62-
}
49+
new Promise(resolveEcsData => {
50+
const ecs = new ECS({ ...config, region, endpoint })
51+
const resources: ECS.StringList = []
52+
const listContainerInstances = (nextToken?: string): void => {
53+
ecs.listContainerInstances(
54+
{ cluster, nextToken },
55+
(err, data) => {
56+
if (err) {
57+
errorLog.generateAwsErrorLog({
58+
functionName: 'ecs:listContainerInstances',
59+
err,
60+
})
61+
}
6362

64-
const { containerInstanceArns: containerInstances = [] } = data
63+
if (isEmpty(data?.containerInstanceArns)) {
64+
return resolveEcsData([])
65+
}
66+
resources.push(...data.containerInstanceArns)
6567

66-
resolveEcsData({ cluster, containerInstances, region })
67-
}
68-
)
69-
)
68+
if (data.nextToken) {
69+
listContainerInstances(data.nextToken)
70+
} else {
71+
resolveEcsData({
72+
cluster,
73+
containerInstances: resources,
74+
region,
75+
})
76+
}
77+
}
78+
)
79+
}
80+
return listContainerInstances()
81+
})
7082
)
7183
)
7284
/**

‎src/services/ecsTask/data.ts‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -48,27 +48,32 @@ export default async ({
4848
let ecsTaskArns: any = await Promise.all(
4949
ecsClusters.map(
5050
async ({ clusterName: cluster, region }) =>
51-
new Promise(resolveEcsData =>
52-
new ECS({ ...config, region, endpoint }).listTasks(
53-
{ cluster },
54-
(err, data) => {
51+
new Promise(resolveEcsData => {
52+
const ecs = new ECS({ ...config, region, endpoint })
53+
const resources: ECS.StringList = []
54+
const listTasks = (nextToken?: string): void => {
55+
ecs.listTasks({ cluster, nextToken }, (err, data) => {
5556
if (err) {
5657
errorLog.generateAwsErrorLog({
5758
functionName: 'ecs:listTasks',
5859
err,
5960
})
6061
}
6162

62-
if (isEmpty(data)) {
63+
if (isEmpty(data?.taskArns)) {
6364
return resolveEcsData([])
6465
}
6566

66-
const { taskArns = [] } = data
67-
68-
resolveEcsData({ region, cluster, taskArns })
69-
}
70-
)
71-
)
67+
resources.push(...data.taskArns)
68+
if (data.nextToken) {
69+
listTasks(data.nextToken)
70+
} else {
71+
resolveEcsData({ region, cluster, taskArns: resources })
72+
}
73+
})
74+
}
75+
listTasks()
76+
})
7277
)
7378
)
7479

‎src/services/efsMountTarget/data.ts‎

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -46,41 +46,50 @@ export default async ({
4646

4747
regions.split(',').map(region => {
4848
efsFileSystems.map(efs => {
49-
const regionPromise = new Promise<void>(resolveMountTargets =>
50-
new EFS({
49+
const regionPromise = new Promise<void>(resolveMountTargets => {
50+
const efsClient = new EFS({
5151
...config,
5252
region: efs.region,
5353
endpoint,
54-
}).describeMountTargets(
55-
{ FileSystemId: efs.FileSystemId },
56-
async (err: AWSError, data: DescribeMountTargetsResponse) => {
57-
if (err) {
58-
errorLog.generateAwsErrorLog({
59-
functionName: 'efs:describeMountTargets',
60-
err,
61-
})
62-
}
54+
})
55+
const describeMountTargets = (nextToken?: string): void => {
56+
efsClient.describeMountTargets(
57+
{ FileSystemId: efs.FileSystemId, Marker: nextToken },
58+
async (err: AWSError, data: DescribeMountTargetsResponse) => {
59+
if (err) {
60+
errorLog.generateAwsErrorLog({
61+
functionName: 'efs:describeMountTargets',
62+
err,
63+
})
64+
}
6365

64-
if (isEmpty(data)) {
65-
return resolveMountTargets()
66-
}
66+
if (isEmpty(data)) {
67+
return resolveMountTargets()
68+
}
6769

68-
const { MountTargets: mountTargets = [] } = data || {}
70+
const { MountTargets: mountTargets = [] } = data || {}
6971

70-
logger.debug(lt.fetchedEfsMountTargets(mountTargets.length))
72+
logger.debug(lt.fetchedEfsMountTargets(mountTargets.length))
7173

72-
if (!isEmpty(mountTargets)) {
73-
efsMountTargets.push(
74-
...mountTargets.map(target => ({
75-
region,
76-
...target,
77-
}))
78-
)
74+
if (!isEmpty(mountTargets)) {
75+
efsMountTargets.push(
76+
...mountTargets.map(target => ({
77+
region,
78+
...target,
79+
}))
80+
)
81+
}
82+
if (data.NextMarker) {
83+
describeMountTargets(data.NextMarker)
84+
} else {
85+
resolveMountTargets()
86+
}
7987
}
80-
resolveMountTargets()
81-
}
82-
)
83-
)
88+
)
89+
}
90+
91+
describeMountTargets()
92+
})
8493
regionPromises.push(regionPromise)
8594
})
8695
})

‎src/services/elb/data.ts‎

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -54,33 +54,40 @@ const getElbTags = async (
5454
)
5555
})
5656

57-
const listElbData = async (
58-
elb: ELB
59-
): Promise<LoadBalancerDescription[]> =>
60-
new Promise<LoadBalancerDescription[]>(resolve => {
61-
let loadBalancerData: LoadBalancerDescription[] = []
62-
elb.describeLoadBalancers(
63-
(err: AWSError, data: DescribeAccessPointsOutput) => {
64-
if (err) {
65-
errorLog.generateAwsErrorLog({
66-
functionName: 'elb:describeLoadBalancers',
67-
err,
68-
})
69-
}
70-
if (!isEmpty(data)) {
71-
const { LoadBalancerDescriptions: loadBalancerDescriptions = [] } =
72-
data
73-
logger.debug(lt.fetchedElbs(loadBalancerDescriptions.length))
74-
loadBalancerData = loadBalancerDescriptions.map(lbDescription => ({
75-
...lbDescription,
76-
}))
77-
resolve(loadBalancerData)
57+
const listElbData = async (elb: ELB): Promise<LoadBalancerDescription[]> => {
58+
const loadBalancerData: LoadBalancerDescription[] = []
59+
return new Promise<LoadBalancerDescription[]>(resolve => {
60+
const listAllData = (nextToken?: string): void => {
61+
elb.describeLoadBalancers(
62+
{ Marker: nextToken },
63+
(err: AWSError, data: DescribeAccessPointsOutput) => {
64+
if (err) {
65+
errorLog.generateAwsErrorLog({
66+
functionName: 'elb:describeLoadBalancers',
67+
err,
68+
})
69+
}
70+
if (!isEmpty(data)) {
71+
const { LoadBalancerDescriptions: loadBalancerDescriptions = [] } =
72+
data
73+
logger.debug(lt.fetchedElbs(loadBalancerDescriptions.length))
74+
loadBalancerData.push(
75+
...loadBalancerDescriptions.map(lbDescription => ({
76+
...lbDescription,
77+
}))
78+
)
79+
}
80+
if (data.NextMarker) {
81+
listAllData(data.NextMarker)
82+
} else {
83+
resolve(loadBalancerData)
84+
}
7885
}
79-
80-
resolve(loadBalancerData)
81-
}
82-
)
86+
)
87+
}
88+
listAllData()
8389
})
90+
}
8491

8592
const listElbAttributes = async (
8693
elb: ELB,
@@ -127,7 +134,7 @@ export interface RawAwsElb extends LoadBalancerDescription {
127134
export default async ({
128135
regions,
129136
config,
130-
account
137+
account,
131138
}: {
132139
regions: string
133140
account: string
@@ -143,7 +150,9 @@ export default async ({
143150
return new Promise<void>(async resolveElbData => {
144151
// Get Load Balancer Data
145152
const elbDescriptionData = await listElbData(elbInstance)
146-
const elbNames: string[] = elbDescriptionData.map(elb => elb.LoadBalancerName)
153+
const elbNames: string[] = elbDescriptionData.map(
154+
elb => elb.LoadBalancerName
155+
)
147156

148157
if (!isEmpty(elbNames)) {
149158
// Get Tags

‎src/services/redshift/data.ts‎

Lines changed: 38 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -35,41 +35,48 @@ export default async ({
3535
const rsData: RawAwsRedshiftCluster[] = []
3636
const regionPromises = []
3737

38-
regions.split(',').map(region => {
39-
const regionPromise = new Promise<void>(resolveRegion =>
40-
new RS({ ...config, region, endpoint }).describeClusters(
41-
{},
42-
(err: AWSError, data: ClustersMessage) => {
43-
if (err) {
44-
errorLog.generateAwsErrorLog({
45-
functionName: 'redshift:describeClusters',
46-
err,
47-
})
48-
}
38+
regions.split(',').forEach(region => {
39+
const regionPromise = new Promise<void>(resolveRegion => {
40+
const rs = new RS({ ...config, region, endpoint })
41+
const listClusters = (nextToken?: string): void => {
42+
rs.describeClusters(
43+
{ Marker: nextToken },
44+
(err: AWSError, data: ClustersMessage) => {
45+
if (err) {
46+
errorLog.generateAwsErrorLog({
47+
functionName: 'redshift:describeClusters',
48+
err,
49+
})
50+
}
4951

50-
if (isEmpty(data)) {
51-
return resolveRegion()
52-
}
52+
if (isEmpty(data)) {
53+
return resolveRegion()
54+
}
5355

54-
const { Clusters: clusters = [] } = data
56+
const { Clusters: clusters = [] } = data
5557

56-
if (isEmpty(clusters)) {
57-
return resolveRegion()
58-
}
58+
if (isEmpty(clusters)) {
59+
return resolveRegion()
60+
}
5961

60-
logger.debug(lt.fetchedRedshiftClusters(clusters.length))
61-
rsData.push(
62-
...clusters.map(({ ...cluster }) => ({
63-
...cluster,
64-
region,
65-
Tags: convertAwsTagsToTagMap(cluster.Tags as AwsTag[]),
66-
}))
67-
)
68-
69-
resolveRegion()
70-
}
71-
)
72-
)
62+
logger.debug(lt.fetchedRedshiftClusters(clusters.length))
63+
rsData.push(
64+
...clusters.map(({ ...cluster }) => ({
65+
...cluster,
66+
region,
67+
Tags: convertAwsTagsToTagMap(cluster.Tags as AwsTag[]),
68+
}))
69+
)
70+
if (data.Marker) {
71+
listClusters(data.Marker)
72+
} else {
73+
resolveRegion()
74+
}
75+
}
76+
)
77+
}
78+
listClusters()
79+
})
7380
regionPromises.push(regionPromise)
7481
})
7582

0 commit comments

Comments
 (0)