Skip to content

Commit ea29e36

Browse files
DmitriiBobreshevMaxim Zaytsevivanduplenskikh
authored
[Courtesy push] Remove unused configs for courtesy push (#19322)
* [Courtesy push] Remove unused configs for courtesy push - Rewrited courtesy-push.js, added possibility to remove unused configs * [Courtesy push] Remove unused configs for courtesy push - formatting * Update ci/courtesy-push/courtesy-push.js Co-authored-by: Maxim Zaytsev <[email protected]> * Update ci/courtesy-push/courtesy-push.js Co-authored-by: Maxim Zaytsev <[email protected]> * Update ci/courtesy-push/courtesy-push.js Co-authored-by: Maxim Zaytsev <[email protected]> * [Courtesy push] Remove unused configs for courtesy push - Added xml2js package for xml convertation - Refactored getDeps to get rid of "_" - Refactore removeConfigsForTasks to identigicate configs properly * [Courtesy push] Remove unused configs for courtesy push - Restored cd * Add jsdoc for deps variable --------- Co-authored-by: Maxim Zaytsev <[email protected]> Co-authored-by: Ivan Duplenskikh <[email protected]>
1 parent 3c661a9 commit ea29e36

File tree

4 files changed

+218
-99
lines changed

4 files changed

+218
-99
lines changed

‎ci/courtesy-push.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ steps:
44
versionSpec: 6.0.0
55
displayName: Install nuget
66

7+
- powershell: |
8+
cd azure-pipelines-tasks/ci/courtesy-push
9+
npm install
10+
displayName: npm install
11+
712
- task: DownloadBuildArtifacts@0
813
inputs:
914
artifactName: IndividualNugetPackages
@@ -42,7 +47,6 @@ steps:
4247
git push origin $releaseBranch --force
4348
Write-Host "Creating Pull Request"
4449
cd ..\azure-pipelines-tasks\ci\courtesy-push
45-
npm install
4650
node open-courtesy-push-pull-request.js $releaseBranch
4751
4852
# Sleep 30 seconds to let PR be created

‎ci/courtesy-push/courtesy-push.js‎

Lines changed: 192 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,139 +1,234 @@
11
const fs = require('fs');
22
const path = require('path');
3+
const xml2js = require('xml2js');
34

45
const azureSourceFolder = process.argv[2];
56
const newDeps = process.argv[3];
67
const unifiedDepsPath = path.join(azureSourceFolder, '.nuget', 'externals', 'UnifiedDependencies.xml');
78
const tfsServerPath = path.join(azureSourceFolder, 'Tfs', 'Service', 'Deploy', 'components', 'TfsServer.Servicing.core.xml');
8-
const msPrefix = "Mseng.MS.TF.DistributedTask.Tasks.";
9-
const directoryTag = new RegExp('<Directory (.*)>');
9+
const msPrefix = 'Mseng.MS.TF.DistributedTask.Tasks.';
1010

11-
function formDirectoryString(nugetTaskName) {
11+
/**
12+
* Helper function to check if the value is included in the array but not equal to the value in the array
13+
* E.g. We compared generated task names such as Mseng.MS.TF.DistributedTask.Tasks.AppCenterDistributeV1_Node20
14+
* with basic Mseng.MS.TF.DistributedTask.Tasks.AppCenterDistributeV1
15+
* If the name is included in the list of dependencies but not equal to the name in the list, we assume its a config
16+
*/
17+
const isIncludeButNotEqual = (arr, value) => arr.reduce((acc, item) => acc || (value.includes(item) && value !== item), false);
18+
19+
function formDirectoryTag(nugetTaskName) {
1220
const taskName = nugetTaskName.replace(msPrefix, '');
13-
14-
return ` <Directory Path="[ServicingDir]Tasks\\Individual\\${taskName}\\">
15-
<File Origin="nuget://Mseng.MS.TF.DistributedTask.Tasks.${taskName}/content/*" />
16-
</Directory>`;
21+
return {
22+
$: {
23+
Path: `[ServicingDir]Tasks\\Individual\\${taskName}\\`
24+
},
25+
File: [
26+
{
27+
$: {
28+
Origin: `nuget://Mseng.MS.TF.DistributedTask.Tasks.${taskName}/content/*`
29+
}
30+
}
31+
]
32+
};
1733
}
1834

19-
function formatDeps(depArr) {
20-
const newDepsDict = {};
35+
/**
36+
* @typedef {Object} Dependencies
37+
* @property {string} name
38+
* @property {string} version
39+
* @property {string} depStr
40+
*/
2141

22-
depArr.forEach(newDep => {
23-
// add to dictionary
42+
/**
43+
* The function to form a dictionary of dependencies
44+
* @param {Array} depArr - array of dependencies
45+
* @returns {Dependencies} - dictionary of dependencies
46+
*/
47+
function getDeps(depArr) {
48+
/** @type {Record<key, Dependencies>} deps */
49+
const deps = {};
50+
const getDependantConfigs = (arrKeys, packageName) => arrKeys.filter(key => key.includes(packageName) && key !== packageName);
51+
52+
// first run we form structures
53+
for (let i = 0; i < depArr.length; i++) {
54+
const newDep = depArr[i];
2455
const depDetails = newDep.split('"');
56+
2557
console.log(JSON.stringify(depDetails));
2658
const name = depDetails[1];
2759
const version = depDetails[3];
2860
console.log(name + ' ' + version);
29-
newDepsDict[name] = version;
30-
});
3161

32-
return newDepsDict;
33-
}
62+
if (!deps[name]) deps[name] = {};
3463

35-
function findLastIndex(array, predicate) {
36-
let l = array.length;
64+
const dep = deps[name];
3765

38-
while (l--) {
39-
if (predicate(array[l], l, array)) {
40-
return l;
41-
}
66+
dep.name = name;
67+
dep.version = version;
68+
dep.depStr = newDep;
69+
}
70+
71+
const keys = Object.keys(deps);
72+
for (let dep in deps) {
73+
const configs = getDependantConfigs(keys, dep);
74+
if (!configs.length) continue;
75+
76+
deps[dep].configs = [];
77+
configs.forEach(config => {
78+
const configDep = deps[config];
79+
deps[dep].configs.push({
80+
name: configDep.name,
81+
version: configDep.version,
82+
depStr: configDep.depStr
83+
});
84+
85+
delete deps[config];
86+
});
4287
}
43-
return -1;
88+
89+
return deps;
4490
}
4591

46-
/* Function updating existing deps version and also add new deps with postfix
47-
* Example: If we have dependency with name
48-
* Mseng.MS.TF.DistributedTask.Tasks.AndroidSigningV2
49-
* It will add Mseng.MS.TF.DistributedTask.Tasks.AndroidSigningV2_Node16 */
50-
function updateUnifiedDeps(pathToUnifiedDeps, pathToNewUnifiedDeps, outputPath) {
51-
const currentDeps = fs.readFileSync(pathToUnifiedDeps, 'utf8');
52-
const newDeps = fs.readFileSync(pathToNewUnifiedDeps, 'utf8');
92+
/**
93+
* The function removes all generated configs such as Node16/Node20 from the list of dependencies
94+
* @param {Array} depsArray - array of parsed dependencies from UnifiedDependencies.xml
95+
* @param {Object} depsForUpdate - dictionary of dependencies from getDeps method
96+
* @param {Object} updatedDeps - structure to track added/removed dependencies
97+
* @returns {Array} - updated array of dependencies and updatedDeps object { added: [], removed: []
98+
*/
99+
function removeConfigsForTasks(depsArray, depsForUpdate, updatedDeps) {
100+
const newDepsArr = depsArray.slice();
101+
const updatedDepsObj = Object.assign({}, updatedDeps);
102+
const basicDepsForUpdate = Object.keys(depsForUpdate).map(dep => dep.toLowerCase());
103+
let index = 0;
104+
105+
while (index < newDepsArr.length) {
106+
const currentDep = newDepsArr[index];
107+
const depDetails = currentDep.split('"');
108+
const name = depDetails[1];
109+
if (!name) {
110+
index++;
111+
continue;
112+
}
53113

54-
const currentDepsArr = currentDeps.split('\n');
55-
const newDepsArr = newDeps.split('\n');
56-
const newDepsDict = formatDeps(newDepsArr);
114+
const basicName = name.toLowerCase();
115+
116+
if (isIncludeButNotEqual(basicDepsForUpdate, basicName)) {
117+
newDepsArr.splice(index, 1);
118+
updatedDepsObj.removed.push(name);
119+
continue;
120+
}
121+
122+
index++;
123+
}
57124

58-
const updatedDeps = [];
59-
// Tasks that was updated and should be presented in TfsServer.Servicing.core.xml
60-
const changedTasks = [];
125+
return [newDepsArr, updatedDepsObj];
126+
}
61127

62-
currentDepsArr.forEach(currentDep => {
128+
/**
129+
* The function updates task dependencies with configs such as Node16/Node20
130+
* @param {Array} depsArray - array of parsed dependencies from UnifiedDependencies.xml
131+
* @param {Object} depsForUpdate - dictionary of dependencies from getDeps method
132+
* @param {Object} updatedDeps - structure to track added/removed dependencies
133+
* @returns {Array} - updated array of dependencies and updatedDeps object { added: [], removed: []
134+
*/
135+
function updateConfigsForTasks(depsArray, depsForUpdate, updatedDeps) {
136+
const newDepsArr = depsArray.slice();
137+
const updatedDepsObj = Object.assign({}, updatedDeps);
138+
const basicDepsForUpdate = new Set(Object.keys(depsForUpdate));
139+
let index = 0;
140+
141+
while (index < newDepsArr.length) {
142+
const currentDep = newDepsArr[index];
63143
const depDetails = currentDep.split('"');
64144
const name = depDetails[1];
65145

66-
// find if there is a match in new (ignoring case)
67-
if (name) {
68-
const newDepsKey = Object.keys(newDepsDict).find(key => key.toLowerCase() === name.toLowerCase());
69-
if (newDepsKey && newDepsDict[newDepsKey]) {
70-
// update the version
71-
depDetails[3] = newDepsDict[newDepsKey];
72-
updatedDeps.push(depDetails.join('"'));
73-
74-
changedTasks.push(newDepsKey);
75-
delete newDepsDict[newDepsKey];
76-
} else {
77-
updatedDeps.push(currentDep);
78-
console.log(`"${currentDep}"`);
79-
}
80-
} else {
81-
updatedDeps.push(currentDep);
146+
if (!name || !basicDepsForUpdate.has(name)) {
147+
index++;
148+
continue;
82149
}
83-
});
84-
85-
// add the new deps from the start
86-
// working only for generated deps
87-
88-
if (Object.keys(newDepsDict).length > 0) {
89-
for (let packageName in newDepsDict) {
90-
// new deps should include old packages completely
91-
// Example:
92-
// Mseng.MS.TF.DistributedTask.Tasks.AndroidSigningV2-Node16(packageName) should include
93-
// Mseng.MS.TF.DistributedTask.Tasks.AndroidSigningV2(basePackageName)
94-
const depToBeInserted = newDepsArr.find(dep => dep.includes(packageName));
95-
const pushingIndex = findLastIndex(updatedDeps, basePackage => {
96-
if (!basePackage) return false;
97-
98-
const depDetails = basePackage.split('"');
99-
const name = depDetails[1];
100-
return name && name.startsWith(msPrefix) && packageName.includes(name.split("_")[0])
101-
});
102150

103-
if (pushingIndex !== -1) {
104-
// We need to insert new package after the old one
105-
updatedDeps.splice(pushingIndex + 1, 0, depToBeInserted);
106-
changedTasks.push(packageName);
107-
}
151+
newDepsArr.splice(index, 1, depsForUpdate[name].depStr);
152+
index++;
153+
154+
if (depsForUpdate[name].configs) {
155+
depsForUpdate[name].configs
156+
.sort((a, b) => a.name > b.name)
157+
.forEach(config => {
158+
updatedDepsObj.added.push(config.name);
159+
newDepsArr.splice(index, 0, config.depStr);
160+
index++;
161+
});
108162
}
109163
}
110-
// write it as a new file where currentDeps is
111-
fs.writeFileSync(outputPath, updatedDeps.join('\n'));
164+
165+
return [newDepsArr, updatedDepsObj];
166+
}
167+
168+
/**
169+
* The main function for unified dependencies update
170+
* The function parses unified dependencies file and updates it with new dependencies/remove unused
171+
* Since the generated tasks can only be used and build with default version, if unified_deps.xml doesn't contain
172+
* the default version, the specific config (e.g. Node16) will be removed from the list of dependencies
173+
* @param {String} pathToUnifiedDeps - path to UnifiedDependencies.xml
174+
* @param {String} pathToNewUnifiedDeps - path to unified_deps.xml which contains dependencies updated on current week
175+
*/
176+
function updateUnifiedDeps(pathToUnifiedDeps, pathToNewUnifiedDeps) {
177+
const currentDeps = fs.readFileSync(pathToUnifiedDeps, 'utf8');
178+
const newDeps = fs.readFileSync(pathToNewUnifiedDeps, 'utf8');
179+
180+
const newDepsArr = newDeps.split('\n');
181+
const depsForUpdate = getDeps(newDepsArr);
182+
183+
let depsArray = currentDeps.split('\n');
184+
let updatedDeps = { added: [], removed: [] };
185+
186+
[depsArray, updatedDeps] = removeConfigsForTasks(depsArray, depsForUpdate, updatedDeps);
187+
[depsArray, updatedDeps] = updateConfigsForTasks(depsArray, depsForUpdate, updatedDeps);
188+
189+
fs.writeFileSync(pathToUnifiedDeps, depsArray.join('\n'));
112190
console.log('Updating Unified Dependencies file done.');
113-
return changedTasks;
114-
};
191+
return updatedDeps;
192+
}
115193

116-
/* Function to insert new tasks into TfsServer.Servicing.core.xml
117-
* Only if the was modified/added into UnifiedDependencies.xml and not exists in the TfsServer.Servicing.core.xml file
194+
/**
195+
* The function update TfsServer.Servicing.core.xml with new dependencies
196+
* The function check the depsToUpdate which was created during updateUnifiedDeps
197+
* and add/remove dependencies from TfsServer.Servicing.core.xml
198+
* @param {String} pathToTfsCore - path to TfsServer.Servicing.core.xml
199+
* @param {Object} depsToUpdate - structure to track added/removed dependencies (formed in updateUnifiedDeps)
118200
*/
119-
function updateTfsServerDeps(pathToTfsCore, depsToUpdateArr, outputPath) {
201+
async function updateTfsServerDeps(pathToTfsCore, depsToUpdate) {
120202
const tfsCore = fs.readFileSync(pathToTfsCore, 'utf8');
121-
const tfsToUpdate = tfsCore.split('\n');
122-
const tfsCoreLowerCase = tfsCore.toLowerCase();
123-
124-
const insertedIndex = tfsToUpdate.findIndex(tfsString => directoryTag.test(tfsString));
125-
depsToUpdateArr.forEach(dependencyName => {
126-
const dependencyNameLower = dependencyName.toLowerCase();
127-
if (tfsCoreLowerCase.indexOf(dependencyNameLower) === -1) {
128-
const insertedString = formDirectoryString(dependencyName);
129-
tfsToUpdate.splice(insertedIndex, 0, insertedString);
130-
console.log(`${insertedString}`);
131-
}
203+
const tfxCoreJson = await xml2js.parseStringPromise(tfsCore);
204+
const depsToAdd = depsToUpdate.added.filter(dep => depsToUpdate.removed.indexOf(dep) === -1);
205+
const depsToRemove = depsToUpdate.removed.filter(dep => depsToUpdate.added.indexOf(dep) === -1);
206+
207+
// removing dependencies
208+
for (let idx = 0; idx < tfxCoreJson.Component.Directory.length; idx++) {
209+
const directory = tfxCoreJson.Component.Directory[idx];
210+
const files = directory.File;
211+
const needToRemove = files.filter(file => depsToRemove.findIndex(dep => file.$.Origin.includes(dep)) !== -1);
212+
if (needToRemove.length) {
213+
tfxCoreJson.Component.Directory.splice(idx, 1);
214+
idx--;
215+
}
216+
}
217+
218+
depsToAdd.forEach(dep => {
219+
const directory = formDirectoryTag(dep);
220+
tfxCoreJson.Component.Directory.unshift(directory);
221+
});
222+
223+
const builder = new xml2js.Builder({
224+
xmldec: { version: '1.0', encoding: 'utf-8' },
225+
renderOpts: { pretty: true, indent: ' ', newline: '\n', allowEmpty: false, spacebeforeslash: ' ' }
132226
});
227+
const xml = builder.buildObject(tfxCoreJson);
133228

134-
fs.writeFileSync(outputPath, tfsToUpdate.join('\n'));
229+
fs.writeFileSync(pathToTfsCore, xml);
135230
console.log('Inserting into Tfs Servicing Core file done.');
136231
}
137232

138-
const changedTasks = updateUnifiedDeps(unifiedDepsPath, newDeps, unifiedDepsPath);
139-
updateTfsServerDeps(tfsServerPath, changedTasks, tfsServerPath);
233+
const changedTasks = updateUnifiedDeps(unifiedDepsPath, newDeps);
234+
updateTfsServerDeps(tfsServerPath, changedTasks);

‎ci/courtesy-push/package-lock.json‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎ci/courtesy-push/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"azure-devops-node-api": "12.0.0"
12+
"azure-devops-node-api": "12.0.0",
13+
"xml2js": "^0.6.2"
1314
}
1415
}

0 commit comments

Comments
 (0)