Skip to content

Commit 75182f1

Browse files
m-pizarrom-pizarro
authored
Merge pull request #81 from cloudgraphdev/feature/EP-3219-add-missing-data-factory-services
feat(azure): Add missing dataFactory services
2 parents ba8036e + 2b9a48b commit 75182f1

File tree

4 files changed

+445
-3
lines changed

4 files changed

+445
-3
lines changed

‎src/services/dataFactory/data.ts‎

Lines changed: 239 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,13 @@
1-
import { DataFactoryManagementClient, Factory } from '@azure/arm-datafactory'
1+
import {
2+
DataFactoryManagementClient,
3+
DataFlowResource,
4+
DatasetResource,
5+
Factory,
6+
IntegrationRuntimeResource,
7+
LinkedServiceResource,
8+
PipelineResource,
9+
TriggerResource,
10+
} from '@azure/arm-datafactory'
211
import { PagedAsyncIterableIterator } from '@azure/core-paging'
312
import CloudGraph from '@cloudgraph/sdk'
413

@@ -19,6 +28,198 @@ export interface RawAzureDataFactory
1928
region: string
2029
resourceGroupId: string
2130
Tags: TagMap
31+
pipelines: PipelineResource[]
32+
dataFlows: DataFlowResource[]
33+
datasets: DatasetResource[]
34+
integrationRuntimes: IntegrationRuntimeResource[]
35+
linkedServices: LinkedServiceResource[]
36+
triggers: TriggerResource[]
37+
}
38+
39+
const getPipelines = async ({
40+
client,
41+
resourceGroupName,
42+
factoryName,
43+
}: {
44+
client: DataFactoryManagementClient
45+
resourceGroupName: string
46+
factoryName: string
47+
}): Promise<PipelineResource[]> => {
48+
const pipelines: PipelineResource[] = []
49+
const pipelinesIterable: PagedAsyncIterableIterator<PipelineResource> =
50+
client.pipelines.listByFactory(resourceGroupName, factoryName)
51+
52+
await tryCatchWrapper(
53+
async () => {
54+
for await (const pipeline of pipelinesIterable) {
55+
if (pipeline) {
56+
pipelines.push(pipeline)
57+
}
58+
}
59+
},
60+
{
61+
service: serviceName,
62+
client,
63+
scope: 'pipelines',
64+
operation: 'listByFactory',
65+
}
66+
)
67+
return pipelines
68+
}
69+
70+
const getDataFlows = async ({
71+
client,
72+
resourceGroupName,
73+
factoryName,
74+
}: {
75+
client: DataFactoryManagementClient
76+
resourceGroupName: string
77+
factoryName: string
78+
}): Promise<DataFlowResource[]> => {
79+
const flows: DataFlowResource[] = []
80+
const flowsIterable: PagedAsyncIterableIterator<DataFlowResource> =
81+
client.dataFlows.listByFactory(resourceGroupName, factoryName)
82+
83+
await tryCatchWrapper(
84+
async () => {
85+
for await (const flow of flowsIterable) {
86+
if (flow) {
87+
flows.push(flow)
88+
}
89+
}
90+
},
91+
{
92+
service: serviceName,
93+
client,
94+
scope: 'flows',
95+
operation: 'listByFactory',
96+
}
97+
)
98+
return flows
99+
}
100+
101+
const getDatasets = async ({
102+
client,
103+
resourceGroupName,
104+
factoryName,
105+
}: {
106+
client: DataFactoryManagementClient
107+
resourceGroupName: string
108+
factoryName: string
109+
}): Promise<DatasetResource[]> => {
110+
const datasets: DatasetResource[] = []
111+
const datasetsIterable: PagedAsyncIterableIterator<DatasetResource> =
112+
client.datasets.listByFactory(resourceGroupName, factoryName)
113+
114+
await tryCatchWrapper(
115+
async () => {
116+
for await (const dataset of datasetsIterable) {
117+
if (dataset) {
118+
datasets.push(dataset)
119+
}
120+
}
121+
},
122+
{
123+
service: serviceName,
124+
client,
125+
scope: 'datasets',
126+
operation: 'listByFactory',
127+
}
128+
)
129+
return datasets
130+
}
131+
132+
const getIntegrationRuntimes = async ({
133+
client,
134+
resourceGroupName,
135+
factoryName,
136+
}: {
137+
client: DataFactoryManagementClient
138+
resourceGroupName: string
139+
factoryName: string
140+
}): Promise<IntegrationRuntimeResource[]> => {
141+
const runtimes: IntegrationRuntimeResource[] = []
142+
const runtimesIterable: PagedAsyncIterableIterator<IntegrationRuntimeResource> =
143+
client.integrationRuntimes.listByFactory(resourceGroupName, factoryName)
144+
145+
await tryCatchWrapper(
146+
async () => {
147+
for await (const runtime of runtimesIterable) {
148+
if (runtime) {
149+
runtimes.push(runtime)
150+
}
151+
}
152+
},
153+
{
154+
service: serviceName,
155+
client,
156+
scope: 'integrationRuntimes',
157+
operation: 'listByFactory',
158+
}
159+
)
160+
return runtimes
161+
}
162+
163+
const getLinkedServices = async ({
164+
client,
165+
resourceGroupName,
166+
factoryName,
167+
}: {
168+
client: DataFactoryManagementClient
169+
resourceGroupName: string
170+
factoryName: string
171+
}): Promise<LinkedServiceResource[]> => {
172+
const linkedServices: LinkedServiceResource[] = []
173+
const linkedServicesIterable: PagedAsyncIterableIterator<LinkedServiceResource> =
174+
client.linkedServices.listByFactory(resourceGroupName, factoryName)
175+
176+
await tryCatchWrapper(
177+
async () => {
178+
for await (const linkedService of linkedServicesIterable) {
179+
if (linkedService) {
180+
linkedServices.push(linkedService)
181+
}
182+
}
183+
},
184+
{
185+
service: serviceName,
186+
client,
187+
scope: 'linkedServices',
188+
operation: 'listByFactory',
189+
}
190+
)
191+
return linkedServices
192+
}
193+
194+
const getTriggers = async ({
195+
client,
196+
resourceGroupName,
197+
factoryName,
198+
}: {
199+
client: DataFactoryManagementClient
200+
resourceGroupName: string
201+
factoryName: string
202+
}): Promise<TriggerResource[]> => {
203+
const triggers: TriggerResource[] = []
204+
const triggersIterable: PagedAsyncIterableIterator<TriggerResource> =
205+
client.triggers.listByFactory(resourceGroupName, factoryName)
206+
207+
await tryCatchWrapper(
208+
async () => {
209+
for await (const trigger of triggersIterable) {
210+
if (trigger) {
211+
triggers.push(trigger)
212+
}
213+
}
214+
},
215+
{
216+
service: serviceName,
217+
client,
218+
scope: 'triggers',
219+
operation: 'listByFactory',
220+
}
221+
)
222+
return triggers
22223
}
23224

24225
export default async ({
@@ -49,10 +250,46 @@ export default async ({
49250
if (factory) {
50251
const { location, tags, ...restOfFactory } = factory
51252
const resourceGroupId = getResourceGroupFromEntity(restOfFactory)
253+
const pipelines = await getPipelines({
254+
client,
255+
resourceGroupName: resourceGroupId,
256+
factoryName: factory.name,
257+
})
258+
const dataFlows = await getDataFlows({
259+
client,
260+
resourceGroupName: resourceGroupId,
261+
factoryName: factory.name,
262+
})
263+
const datasets = await getDatasets({
264+
client,
265+
resourceGroupName: resourceGroupId,
266+
factoryName: factory.name,
267+
})
268+
const integrationRuntimes = await getIntegrationRuntimes({
269+
client,
270+
resourceGroupName: resourceGroupId,
271+
factoryName: factory.name,
272+
})
273+
const linkedServices = await getLinkedServices({
274+
client,
275+
resourceGroupName: resourceGroupId,
276+
factoryName: factory.name,
277+
})
278+
const triggers = await getTriggers({
279+
client,
280+
resourceGroupName: resourceGroupId,
281+
factoryName: factory.name,
282+
})
52283
factories.push({
53284
region: lowerCaseLocation(location),
54285
Tags: tags || {},
55286
resourceGroupId,
287+
pipelines,
288+
dataFlows,
289+
datasets,
290+
integrationRuntimes,
291+
linkedServices,
292+
triggers,
56293
...restOfFactory,
57294
})
58295
}
@@ -68,7 +305,7 @@ export default async ({
68305
logger.debug(lt.foundDataFactory(factories.length))
69306

70307
const result: { [property: string]: RawAzureDataFactory[] } = {}
71-
factories.map(({ region, tags, ...rest }) => {
308+
factories.forEach(({ region, tags, ...rest }) => {
72309
if (regions.includes(region)) {
73310
if (!result[region]) {
74311
result[region] = []

‎src/services/dataFactory/format.ts‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
1-
import { AzureDataFactory } from '../../types/generated'
1+
import {
2+
AzureDataFactory,
3+
AzureDataFactoryDataFlow,
4+
AzureDataFactoryDataset,
5+
AzureDataFactoryIntegrationRuntime,
6+
AzureDataFactoryLinkedService,
7+
AzureDataFactoryPipeline,
8+
AzureDataFactoryTrigger,
9+
} from '../../types/generated'
210
import { formatTagsFromMap } from '../../utils/format'
311
import { RawAzureDataFactory } from './data'
412

@@ -23,6 +31,12 @@ export default ({
2331
createTime,
2432
version,
2533
publicNetworkAccess,
34+
pipelines = [],
35+
dataFlows = [],
36+
datasets = [],
37+
integrationRuntimes = [],
38+
linkedServices = [],
39+
triggers = [],
2640
} = service
2741

2842
return {
@@ -45,5 +59,77 @@ export default ({
4559
createTime,
4660
version,
4761
publicNetworkAccess,
62+
pipelines:
63+
pipelines?.map(
64+
({
65+
id: pipelineId,
66+
name: pipelineName,
67+
type: pipelineType,
68+
}): AzureDataFactoryPipeline => ({
69+
id: pipelineId,
70+
name: pipelineName,
71+
type: pipelineType,
72+
})
73+
) ?? [],
74+
dataFlows:
75+
dataFlows?.map(
76+
({
77+
id: dataFlowId,
78+
name: dataFlowName,
79+
type: dataFlowType,
80+
}): AzureDataFactoryDataFlow => ({
81+
id: dataFlowId,
82+
name: dataFlowName,
83+
type: dataFlowType,
84+
})
85+
) ?? [],
86+
datasets:
87+
datasets?.map(
88+
({
89+
id: datasetId,
90+
name: datasetName,
91+
properties: { type: datasetType },
92+
}): AzureDataFactoryDataset => ({
93+
id: datasetId,
94+
name: datasetName,
95+
type: datasetType,
96+
})
97+
) ?? [],
98+
integrationRuntimes:
99+
integrationRuntimes?.map(
100+
({
101+
id: runtimeId,
102+
name: runtimeName,
103+
properties: { type: runtimeType },
104+
}): AzureDataFactoryIntegrationRuntime => ({
105+
id: runtimeId,
106+
name: runtimeName,
107+
type: runtimeType,
108+
})
109+
) ?? [],
110+
linkedServices:
111+
linkedServices?.map(
112+
({
113+
id: runtimeId,
114+
name: runtimeName,
115+
properties: { type: runtimeType },
116+
}): AzureDataFactoryLinkedService => ({
117+
id: runtimeId,
118+
name: runtimeName,
119+
type: runtimeType,
120+
})
121+
) ?? [],
122+
triggers:
123+
triggers?.map(
124+
({
125+
id: triggerId,
126+
name: triggerName,
127+
properties: { type: triggerType },
128+
}): AzureDataFactoryTrigger => ({
129+
id: triggerId,
130+
name: triggerName,
131+
type: triggerType,
132+
})
133+
) ?? [],
48134
}
49135
}

0 commit comments

Comments
 (0)