Skip to content

Commit 0e887d8

Browse files
MathiasVPMathiasVP
authored
Merge pull request #235 from microsoft/update-typegen-2
PS: Restructure generated files
2 parents 2f0b064 + a17f10d commit 0e887d8

File tree

4,349 files changed

+152690
-128901
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

4,349 files changed

+152690
-128901
lines changed

‎powershell/misc/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ Run the following commands
1212

1313
```
1414
# Clone dotnet/dotnet-api-docs
15-
git clone https://github.com/dotnet/dotnet-api-docs
15+
git clone https://github.com/dotnet/dotnet-api-docs --depth 1
1616
# Clone MicrosoftDocs/powershell-docs-sdk-dotnet
17-
git clone [email protected]:MicrosoftDocs/powershell-docs-sdk-dotnet.git
17+
git clone [email protected]:MicrosoftDocs/powershell-docs-sdk-dotnet.git --depth 1
1818
# Generate data extensions
1919
python3 misc/typemodelgen.py dotnet-api-docs/xml/ powershell-docs-sdk-dotnet/dotnet/xml
2020
```

‎powershell/misc/typemodelgen.py

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,25 @@
22
from pathlib import Path
33
import sys
44
import os
5+
import re
56
from collections import defaultdict
67

78

89
def fixup(t):
910
"""Sometimes the docs specify a type that doesn't align with what
1011
PowerShell reports. This function fixes up those types so that it aligns with PowerShell.
1112
"""
12-
if t.startswith("System.ReadOnlySpan<"):
13-
return "System.String"
14-
return t
13+
if t.startswith("system.readonlyspan<"):
14+
return "system.string"
15+
# A regular expression that matches strings like a.b.c<T, U, W>
16+
# and replacee it with a.b.c
17+
return re.sub(r"<.*>", "", t)
18+
19+
def skipQualifier(name):
20+
"""Removes the qualifier from the name."""
21+
# A regular expression that matches strings like a.b.c and returns c
22+
# and replaces it with c
23+
return re.sub(r".*\.", "", name)
1524

1625

1726
def isStatic(member):
@@ -22,7 +31,7 @@ def isStatic(member):
2231
return False
2332

2433

25-
def isA(x):
34+
def isA(member, x):
2635
"""Returns True if member is an `x`."""
2736
for child in member:
2837
if child.tag == "MemberType" and child.text == x:
@@ -77,22 +86,29 @@ def generateTypeModels(arg):
7786
try:
7887
if not file_path.name.endswith(".xml"):
7988
continue
80-
89+
8190
if not file_path.is_file():
8291
continue
83-
92+
8493
tree = ET.parse(str(file_path))
8594
root = tree.getroot()
8695
if not root.tag == "Type":
8796
continue
8897

8998
thisType = root.attrib["FullName"]
9099

91-
if "`" in file_path.stem or "+" in file_path.stem:
92-
continue # Skip generics (and nested types?) for now
100+
parentName = file_path.parent.name
101+
# Remove `and + in parentName
102+
parentName = parentName.replace("`", "").replace("+", "")
93103

94-
folderName = file_path.parent.name.replace(".", "")
95-
filename = folderName + "/model.yml"
104+
# Remove ` in file_path.stem
105+
# and + in file_path.stem
106+
if thisType == "":
107+
print("Error: Empty type name")
108+
continue
109+
110+
folderName = "generated/" + parentName
111+
filename = folderName + ".typemodel.yml"
96112
s = set()
97113
for elem in root.findall(".//Members/Member"):
98114
name = elem.attrib["MemberName"]
@@ -106,10 +122,10 @@ def generateTypeModels(arg):
106122
startSelectorMarker = ""
107123
endSelectorMarker = ""
108124
if isField(elem):
109-
startSelectorMarker = "Field"
125+
startSelectorMarker = "Member"
110126
endSelectorMarker = ""
111127
if isProperty(elem):
112-
startSelectorMarker = "Property"
128+
startSelectorMarker = "Member"
113129
endSelectorMarker = ""
114130
if isMethod(elem):
115131
startSelectorMarker = "Method"
@@ -134,8 +150,9 @@ def generateTypeModels(arg):
134150
returnType = elem.find(".//ReturnValue/ReturnType").text
135151
if returnType == "System.Void":
136152
continue # Don't generate type summaries for void methods
153+
137154
s.add(
138-
f' - ["{fixup(returnType)}", "{thisType + staticMarker}", "{startSelectorMarker}[{name}]{endSelectorMarker}"]\n'
155+
f' - ["{fixup(returnType.lower())}", "{fixup(thisType.lower()) + staticMarker}", "{startSelectorMarker}[{skipQualifier(fixup(name.lower()))}]{endSelectorMarker}"]\n'
139156
)
140157

141158
summaries[filename].update(s)
@@ -152,7 +169,8 @@ def writeModels():
152169
if len(s) == 0:
153170
continue
154171
os.makedirs(os.path.dirname(filename), exist_ok=True)
155-
with open(filename, "x") as file:
172+
with open(filename, "w") as file:
173+
file.write("# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.\n")
156174
file.write("extensions:\n")
157175
file.write(" - addsTo:\n")
158176
file.write(" pack: microsoft/powershell-all\n")

‎powershell/ql/lib/qlpack.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,6 @@ dependencies:
1616
dataExtensions:
1717
- semmle/code/powershell/frameworks/**/model.yml
1818
- semmle/code/powershell/frameworks/**/*.model.yml
19+
- semmle/code/powershell/frameworks/**/typemodel.yml
20+
- semmle/code/powershell/frameworks/**/*.typemodel.yml
1921
warnOnImplicitThis: true

‎powershell/ql/lib/semmle/code/powershell/Frameworks.qll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
* Helper file that imports all framework modeling.
33
*/
44

5-
import semmle.code.powershell.frameworks.SystemManagementAutomationRunspaces.Runspaces
6-
import semmle.code.powershell.frameworks.SystemManagementAutomationPowerShell.PowerShell
7-
import semmle.code.powershell.frameworks.SystemManagementAutomationEngineIntrinsics.EngineIntrinsics
5+
import semmle.code.powershell.frameworks.Runspaces
6+
import semmle.code.powershell.frameworks.PowerShell
7+
import semmle.code.powershell.frameworks.EngineIntrinsics

‎powershell/ql/lib/semmle/code/powershell/frameworks/Accessibility/model.yml

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎powershell/ql/lib/semmle/code/powershell/frameworks/SystemManagementAutomationEngineIntrinsics/EngineIntrinsics.qll renamed to ‎powershell/ql/lib/semmle/code/powershell/frameworks/EngineIntrinsics.qll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ private import semmle.code.powershell.dataflow.internal.DataFlowPublic as DataFl
55
module EngineIntrinsics {
66
private class EngineIntrinsicsGlobalEntry extends ModelInput::TypeModel {
77
override DataFlow::Node getASource(string type) {
8-
type = "System.Management.Automation.EngineIntrinsics" and
9-
result.asExpr().getExpr().(VarReadAccess).getVariable().matchesName("ExecutionContext")
8+
type = "system.management.automation.engineintrinsics" and
9+
result.asExpr().getExpr().(VarReadAccess).getVariable().matchesName("executioncontext")
1010
}
1111
}
1212
}

‎powershell/ql/lib/semmle/code/powershell/frameworks/IEHostExecute/model.yml

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎powershell/ql/lib/semmle/code/powershell/frameworks/Microsoft.PowerShell.Utility.model.yml

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft/powershell-all
4+
extensible: sourceModel
5+
data:
6+
- ["microsoft.powershell.utility!", "Method[read-host].ReturnValue", "stdin"]
7+
- ["microsoft.powershell.utility!", "Method[select-xml].ReturnValue[path]", "file"]
8+
- ["microsoft.powershell.utility!", "Method[format-hex].ReturnValue[path]", "file"]
9+
10+
- addsTo:
11+
pack: microsoft/powershell-all
12+
extensible: summaryModel
13+
data:
14+
- ["microsoft.powershell.utility!", "Method[get-unique]", "Argument[-inputobject,pipeline].Element[?]", "ReturnValue.Element[?]", "value"]
15+
- ["microsoft.powershell.utility!", "Method[join-string]", "Argument[-inputobject,pipeline].Element[?]", "ReturnValue", "taint"]
16+
- ["microsoft.powershell.utility!", "Method[convertfrom-clixmlreference]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
17+
- ["microsoft.powershell.utility!", "Method[convertfrom-csv]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
18+
- ["microsoft.powershell.utility!", "Method[convertfrom-json]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
19+
- ["microsoft.powershell.utility!", "Method[convertfrom-markdown]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
20+
- ["microsoft.powershell.utility!", "Method[convertfrom-sddlstring]", "Argument[-sddl,0,pipeline]", "ReturnValue", "taint"]
21+
- ["microsoft.powershell.utility!", "Method[convertfrom-stringdata]", "Argument[-stringdata,0,pipeline]", "ReturnValue", "taint"]
22+
- ["microsoft.powershell.utility!", "Method[convertto-clixml]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
23+
- ["microsoft.powershell.utility!", "Method[convertto-csv]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
24+
- ["microsoft.powershell.utility!", "Method[convertto-html]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
25+
- ["microsoft.powershell.utility!", "Method[convertto-json]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
26+
- ["microsoft.powershell.utility!", "Method[convertto-xml]", "Argument[-inputobject,0,pipeline]", "ReturnValue", "taint"]
27+
- ["microsoft.powershell.utility!", "Method[out-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
28+
- ["microsoft.powershell.utility!", "Method[select-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
29+
- ["microsoft.powershell.utility!", "Method[select-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
30+
- ["microsoft.powershell.utility!", "Method[select-xml]", "Argument[-content,-path,-xml]", "ReturnValue", "taint"]
31+
- ["microsoft.powershell.utility!", "Method[sort-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
32+
- ["microsoft.powershell.utility!", "Method[tee-object]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
33+
- ["microsoft.powershell.utility!", "Method[write-output]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
34+
- ["microsoft.powershell.utility!", "Method[format-custom]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
35+
- ["microsoft.powershell.utility!", "Method[format-hex]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
36+
- ["microsoft.powershell.utility!", "Method[format-list]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
37+
- ["microsoft.powershell.utility!", "Method[format-table]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
38+
- ["microsoft.powershell.utility!", "Method[format-wide]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
39+
- ["microsoft.powershell.utility!", "Method[get-unique]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
40+
- ["microsoft.powershell.utility!", "Method[join-string]", "Argument[-inputobject,pipeline]", "ReturnValue", "taint"]
41+
42+
- addsTo:
43+
pack: microsoft/powershell-all
44+
extensible: typeModel
45+
data:
46+
- ["system.datetime", "microsoft.powershell.utility!", "Method[get-date].ReturnValue"]
47+
- ["system.object", "microsoft.powershell.utility!", "Method[convertfrom-clixmlreference].ReturnValue"]
48+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[convertFrom-json].ReturnValue"]
49+
- ["system.management.automation.hashtable", "microsoft.powershell.utility!", "Method[convertFrom-json].ReturnValue"]
50+
- ["microsoft.powershell.markdownrender.markdownInfo", "microsoft.powershell.utility!", "Method[convertfrom-markdown].ReturnValue"]
51+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[convertfrom-sddlstring].ReturnValue"]
52+
- ["system.collections.hashtable", "microsoft.powershell.utility!", "Method[convertfrom-stringdata].ReturnValue"]
53+
- ["system.string", "microsoft.powershell.utility!", "Method[convertto-clixml].ReturnValue"]
54+
- ["system.string", "microsoft.powershell.utility!", "Method[convertto-csv].ReturnValue"]
55+
- ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-csv].ReturnValue"]
56+
- ["system.string", "microsoft.powershell.utility!", "Method[convertto-html].ReturnValue"]
57+
- ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-html].ReturnValue"]
58+
- ["system.string", "microsoft.powershell.utility!", "Method[convertto-json].ReturnValue"]
59+
- ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-json].ReturnValue"]
60+
- ["system.string", "microsoft.powershell.utility!", "Method[convertto-xml].ReturnValue"]
61+
- ["system.string[]", "microsoft.powershell.utility!", "Method[convertto-xml].ReturnValue"]
62+
- ["system.string", "microsoft.powershell.utility!", "Method[out-string].ReturnValue"]
63+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[select-object].ReturnValue"]
64+
- ["microsoft.powerShell.commands.matchinfo", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
65+
- ["system.boolean", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
66+
- ["system.string", "microsoft.powershell.utility!", "Method[select-string].ReturnValue"]
67+
- ["microsoft.powerShell.commands.selectxmlinfo", "microsoft.powershell.utility!", "Method[select-xml].ReturnValue"]
68+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[sort-object].ReturnValue"]
69+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[tee-object].ReturnValue"]
70+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[write-output].ReturnValue"]
71+
- ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-custom].ReturnValue"]
72+
- ["microsoft.powershell.commands.bytecollection", "microsoft.powershell.utility!", "Method[format-hex].ReturnValue"]
73+
- ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-list].ReturnValue"]
74+
- ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-table].ReturnValue"]
75+
- ["microsoft.powershell.commands.internal.format", "microsoft.powershell.utility!", "Method[format-wide].ReturnValue"]
76+
- ["pscustomobject", "microsoft.powershell.utility!", "Method[get-unique].ReturnValue"]
77+
- ["system.string", "microsoft.powershell.utility!", "Method[join-string].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftWin32RegistryKey/model.yml renamed to ‎powershell/ql/lib/semmle/code/powershell/frameworks/Microsoft.Win32.RegistryKey.model.yml

File renamed without changes.

‎powershell/ql/lib/semmle/code/powershell/frameworks/Microsoft.Win32.model.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft/powershell-all
4+
extensible: sourceModel
5+
data:
6+
- ["microsoft.win32.registry!", "Method[getvalue].ReturnValue", "windows-registry"]
7+
- ["microsoft.win32.registrykey", "Method[getvalue].ReturnValue", "windows-registry"]
8+
- ["microsoft.win32.registrykey", "Method[getvaluenames].ReturnValue", "windows-registry"]
9+
- ["microsoft.win32.registrykey", "Method[getsubkeynames].ReturnValue", "windows-registry"]

0 commit comments

Comments
 (0)