Skip to content
This repository was archived by the owner on Feb 23, 2024. It is now read-only.

Commit a2fd25e

Browse files
authored
[Extractor]Fix API / Ops limit (#318)
1 parent 54c43b0 commit a2fd25e

File tree

5 files changed

+84
-67
lines changed

5 files changed

+84
-67
lines changed

example/demo/Input/valid.yml

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,34 @@
11
version: 0.0.1 # Required
22
apimServiceName: contosoapim-dev # Required, must match name of an apim service deployed in the specified resource group
33
apiVersionSet: # Optional
4-
id: myVersionSetID
4+
- id: myVersionSetID
55
displayName: myAPIVersionSet
66
description: a description
77
versioningScheme: Query
88
versionQueryName: versionQuery
99
versionHeaderName: versionHeader
1010
api:
11-
name: myAPI # Required
12-
openApiSpec: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/swaggerPetstore.json # Required, can be url or local file
13-
#openApiSpec: https://petstore.swagger.io/v2/swagger.json
14-
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/apiPolicyHeaders.xml # Optional, can be url or local file
15-
suffix: myAPIPet # Required
16-
apiVersion: v1 # Optional
17-
apiVersionDescription: My first version # Optional
18-
apiVersionSetId: myVersionSetID
19-
revision: 1 # Optional
20-
revisionDescription: My first revision # Optional
21-
operations: # Optional
22-
addPet: # Must match the operationId property of a path's operations
23-
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/operationRateLimit.xml # Optional, can be url or local file
24-
deletePet: # Must match the operationId property of a path's operations
25-
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/operationRateLimit.xml # Optional, can be url or local file
26-
authenticationSettings: # Optional
27-
subscriptionKeyRequired: false
28-
#oAuth2:
29-
# authorizationServerId: apimgmtaad
30-
# scope: scope
31-
products: starter # Optional, adds api to the specified products
11+
- name: myAPI # Required
12+
openApiSpec: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/swaggerPetstore.json # Required, can be url or local file
13+
#openApiSpec: https://petstore.swagger.io/v2/swagger.json
14+
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/apiPolicyHeaders.xml # Optional, can be url or local file
15+
suffix: myAPIPet # Required
16+
apiVersion: v1 # Optional
17+
apiVersionDescription: My first version # Optional
18+
apiVersionSetId: myVersionSetID
19+
revision: 1 # Optional
20+
revisionDescription: My first revision # Optional
21+
operations: # Optional
22+
addPet: # Must match the operationId property of a path's operations
23+
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/operationRateLimit.xml # Optional, can be url or local file
24+
deletePet: # Must match the operationId property of a path's operations
25+
policy: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Input/operationRateLimit.xml # Optional, can be url or local file
26+
authenticationSettings: # Optional
27+
subscriptionKeyRequired: false
28+
#oAuth2:
29+
# authorizationServerId: apimgmtaad
30+
# scope: scope
31+
products: starter # Optional, adds api to the specified products
3232
outputLocation: /Users/miaojiang/Work/azure-api-management-devops-example/example/demo/Output # Required, folder the creator will write the templates to
3333
linked: true # Optional
3434
linkedTemplatesBaseUrl : https://raw.githubusercontent.com/miaojiang/azure-api-management-devops-example/MVP/example/demo/Output # Required if 'linked' property is set to true

example/master.template.json

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
"contentVersion":"1.0.0.0"
5959
},
6060
"parameters": {
61-
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" },
61+
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" },
6262
"PublisherName": {"value": "[parameters('PublisherName')]" },
6363
"PublisherEmail": {"value": "[parameters('PublisherEmail')]" },
6464
"sku": {"value": "[parameters('sku')]" },
@@ -109,12 +109,11 @@
109109
"properties": {
110110
"mode": "Incremental",
111111
"templateLink": {
112-
"uri":"[concat(parameters('repoBaseUrl'), '/product-starter/product-starter.template.json')]",
112+
"uri":"[concat(parameters('repoBaseUrl'), '/products/product-starter.template.json')]",
113113
"contentVersion":"1.0.0.0"
114114
},
115115
"parameters": {
116-
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" },
117-
"repoBaseUrl": {"value": "[parameters('repoBaseUrl')]" }
116+
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" }
118117
}
119118
},
120119
"dependsOn": [
@@ -189,8 +188,7 @@
189188
"contentVersion":"1.0.0.0"
190189
},
191190
"parameters": {
192-
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" },
193-
"repoBaseUrl": {"value": "[parameters('repoBaseUrl')]" }
191+
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" }
194192
}
195193
},
196194
"dependsOn": [
@@ -209,8 +207,7 @@
209207
"contentVersion":"1.0.0.0"
210208
},
211209
"parameters": {
212-
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" },
213-
"repoBaseUrl": {"value": "[parameters('repoBaseUrl')]" }
210+
"ApimServiceName": {"value": "[parameters('ApimServiceName')]" }
214211
}
215212
},
216213
"dependsOn": [

src/APIM_ARMTemplate/apimtemplate/Common/Constants/GlobalConstants.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ internal static class GlobalConstants
1313

1414
public const string APIVersion = "2019-01-01";
1515
public const string LinkedAPIVersion = "2018-01-01";
16+
public const int NumOfRecords = 100;
1617

1718
public const string azAccessToken = "account get-access-token --query \"accessToken\" --output json";
1819
public const string azSubscriptionId = "account show --query id -o json";

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/APIExtractor.cs

Lines changed: 51 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,30 @@ public APIExtractor(FileWriter fileWriter)
1717
this.fileWriter = fileWriter;
1818
}
1919

20-
public async Task<string> GetAPIOperationsAsync(string ApiManagementName, string ResourceGroupName, string ApiName)
20+
private async Task<string[]> GetAllOperationNames(string ApiManagementName, string ResourceGroupName, string ApiName)
2121
{
22-
(string azToken, string azSubId) = await auth.GetAccessToken();
22+
JObject oOperations = new JObject();
23+
int numOfOps = 0;
24+
List<string> operationNames = new List<string>();
25+
do
26+
{
27+
(string azToken, string azSubId) = await auth.GetAccessToken();
2328

24-
string requestUrl = string.Format("{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis/{4}/operations?api-version={5}",
25-
baseUrl, azSubId, ResourceGroupName, ApiManagementName, ApiName, GlobalConstants.APIVersion);
29+
string requestUrl = string.Format("{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis/{4}/operations?$skip={5}&api-version={6}",
30+
baseUrl, azSubId, ResourceGroupName, ApiManagementName, ApiName, numOfOps, GlobalConstants.APIVersion);
31+
numOfOps += GlobalConstants.NumOfRecords;
2632

27-
return await CallApiManagementAsync(azToken, requestUrl);
33+
string operations = await CallApiManagementAsync(azToken, requestUrl);
34+
35+
oOperations = JObject.Parse(operations);
36+
37+
foreach (var item in oOperations["value"])
38+
{
39+
operationNames.Add(((JValue)item["name"]).Value.ToString());
40+
}
41+
}
42+
while (oOperations["nextLink"] != null);
43+
return operationNames.ToArray();
2844
}
2945

3046
public async Task<string> GetAPIOperationDetailsAsync(string ApiManagementName, string ResourceGroupName, string ApiName, string OperationName)
@@ -67,14 +83,30 @@ public async Task<string> GetAPIDetailsAsync(string ApiManagementName, string Re
6783
return await CallApiManagementAsync(azToken, requestUrl);
6884
}
6985

70-
public async Task<string> GetAPIsAsync(string ApiManagementName, string ResourceGroupName)
86+
public async Task<JToken[]> GetAllAPIObjsAsync(string ApiManagementName, string ResourceGroupName)
7187
{
72-
(string azToken, string azSubId) = await auth.GetAccessToken();
88+
JObject oApi = new JObject();
89+
int numOfApis = 0;
90+
List<JToken> apiObjs = new List<JToken>();
91+
do
92+
{
93+
(string azToken, string azSubId) = await auth.GetAccessToken();
7394

74-
string requestUrl = string.Format("{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?api-version={4}",
75-
baseUrl, azSubId, ResourceGroupName, ApiManagementName, GlobalConstants.APIVersion);
95+
string requestUrl = string.Format("{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?$skip={4}&api-version={5}",
96+
baseUrl, azSubId, ResourceGroupName, ApiManagementName, numOfApis, GlobalConstants.APIVersion);
97+
numOfApis += GlobalConstants.NumOfRecords;
7698

77-
return await CallApiManagementAsync(azToken, requestUrl);
99+
string apis = await CallApiManagementAsync(azToken, requestUrl);
100+
101+
oApi = JObject.Parse(apis);
102+
103+
foreach (var item in oApi["value"])
104+
{
105+
apiObjs.Add(item);
106+
}
107+
}
108+
while (oApi["nextLink"] != null);
109+
return apiObjs.ToArray();
78110
}
79111

80112
public async Task<string> GetAPIChangeLogAsync(string ApiManagementName, string ResourceGroupName, string ApiName)
@@ -199,12 +231,10 @@ public async Task<List<TemplateResource>> GenerateSingleAPIResourceAsync(string
199231
#region Operations
200232

201233
// pull api operations for service
202-
string operations = await GetAPIOperationsAsync(apimname, resourceGroup, apiName);
203-
JObject oOperations = JObject.Parse(operations);
234+
string[] operationNames = await GetAllOperationNames(apimname, resourceGroup, apiName);
204235

205-
foreach (var item in oOperations["value"])
236+
foreach (string operationName in operationNames)
206237
{
207-
string operationName = ((JValue)item["name"]).Value.ToString();
208238
string operationDetails = await GetAPIOperationDetailsAsync(apimname, resourceGroup, apiName, operationName);
209239

210240
Console.WriteLine("'{0}' Operation found", operationName);
@@ -422,7 +452,7 @@ public async Task<Template> GenerateAPIRevisionTemplateAsync(string currentRevis
422452
{
423453
// add other API revision resources to template
424454
apiResources = await GenerateSingleAPIResourceAsync(curApi, exc.sourceApimName, exc.resourceGroup, exc.fileFolder, exc.policyXMLBaseUrl);
425-
455+
426456
// make current API a dependency to other revisions, in case destination apim doesn't have the this API
427457
TemplateResource apiResource = apiResources.FirstOrDefault(resource => resource.type == ResourceTypeConstants.API) as TemplateResource;
428458
List<TemplateResource> newResourcesList = ExtractorUtils.removeResourceType(ResourceTypeConstants.API, apiResources);
@@ -483,12 +513,10 @@ public async Task<List<TemplateResource>> GenerateCurrentRevisionAPIResourceAsyn
483513
#region Operations
484514

485515
// pull api operations for service
486-
string operations = await GetAPIOperationsAsync(apimname, resourceGroup, apiName);
487-
JObject oOperations = JObject.Parse(operations);
516+
string[] operationNames = await GetAllOperationNames(apimname, resourceGroup, apiName);
488517

489-
foreach (var item in oOperations["value"])
518+
foreach (string operationName in operationNames)
490519
{
491-
string operationName = ((JValue)item["name"]).Value.ToString();
492520
string operationDetails = await GetAPIOperationDetailsAsync(apimname, resourceGroup, apiName, operationName);
493521

494522
Console.WriteLine("'{0}' Operation found", operationName);
@@ -715,12 +743,12 @@ public async Task<Template> GenerateAPIsARMTemplateAsync(string apimname, string
715743
// when extract all APIs and generate one master template
716744
else
717745
{
718-
JObject oApi = await GetAllAPIsFromAPIM(apimname, resourceGroup, policyXMLBaseUrl);
719-
Console.WriteLine("{0} APIs found ...", ((JContainer)oApi["value"]).Count.ToString());
746+
JToken[] oApis = await GetAllAPIObjsAsync(apimname, resourceGroup);
747+
Console.WriteLine("{0} APIs found ...", (oApis.Count().ToString()));
720748

721-
for (int i = 0; i < ((JContainer)oApi["value"]).Count; i++)
749+
foreach (JToken oApi in oApis)
722750
{
723-
string apiName = ((JValue)oApi["value"][i]["name"]).Value.ToString();
751+
string apiName = ((JValue)oApi["name"]).Value.ToString();
724752
templateResources.AddRange(await GenerateSingleAPIResourceAsync(apiName, apimname, resourceGroup, fileFolder, policyXMLBaseUrl));
725753
}
726754
}
@@ -761,14 +789,6 @@ private static bool CheckAPIExist(string singleApiName, JObject oApi)
761789
return false;
762790
}
763791

764-
public async Task<JObject> GetAllAPIsFromAPIM(string apimname, string resourceGroup, string policyXMLBaseUrl)
765-
{
766-
// pull all apis from service
767-
string apis = await GetAPIsAsync(apimname, resourceGroup);
768-
JObject oApi = JObject.Parse(apis);
769-
return oApi;
770-
}
771-
772792
private async Task<List<TemplateResource>> GenerateSchemasARMTemplate(string apimServiceName, string apiName, string resourceGroup, string fileFolder)
773793
{
774794
List<TemplateResource> templateResources = new List<TemplateResource>();

src/APIM_ARMTemplate/apimtemplate/Extractor/Utilities/ExtractorUtils.cs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -283,23 +283,22 @@ public static async Task<Dictionary<string, List<string>>> GetAllAPIsDictionary(
283283
{
284284
APIExtractor apiExtractor = new APIExtractor(fileWriter);
285285
// pull all apis from service
286-
string apis = await apiExtractor.GetAPIsAsync(sourceApim, resourceGroup);
287-
JObject oApi = JObject.Parse(apis);
286+
JToken[] apis = await apiExtractor.GetAllAPIObjsAsync(sourceApim, resourceGroup);
288287

289288
// Generate folders based on all apiversionset
290289
var apiDictionary = new Dictionary<string, List<string>>();
291-
for (int i = 0; i < ((JContainer)oApi["value"]).Count; i++)
290+
foreach (JToken oApi in apis)
292291
{
293-
string apiDisplayName = ((JValue)oApi["value"][i]["properties"]["displayName"]).Value.ToString();
292+
string apiDisplayName = ((JValue)oApi["properties"]["displayName"]).Value.ToString();
294293
if (!apiDictionary.ContainsKey(apiDisplayName))
295294
{
296295
List<string> apiVersionSet = new List<string>();
297-
apiVersionSet.Add(((JValue)oApi["value"][i]["name"]).Value.ToString());
296+
apiVersionSet.Add(((JValue)oApi["name"]).Value.ToString());
298297
apiDictionary[apiDisplayName] = apiVersionSet;
299298
}
300299
else
301300
{
302-
apiDictionary[apiDisplayName].Add(((JValue)oApi["value"][i]["name"]).Value.ToString());
301+
apiDictionary[apiDisplayName].Add(((JValue)oApi["name"]).Value.ToString());
303302
}
304303
}
305304
return apiDictionary;

0 commit comments

Comments
 (0)