Skip to content

Commit 61796da

Browse files
MathiasVPMathiasVP
authored
Merge pull request #171 from microsoft/ps-add-dotnet-type-models
PS: Add .NET and PowerShell SDK type models.
2 parents 5c54c81 + 3dbe7f4 commit 61796da

File tree

677 files changed

+123433
-4
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

677 files changed

+123433
-4
lines changed

‎powershell/misc/README.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
# Type model generation
3+
4+
Type information about .NET and PowerShell SDK methods are obtained by generating data extension files that populate the `typeModel` extensible predicate.
5+
6+
The type models are located here: https://github.com/microsoft/codeql/blob/main/powershell/ql/lib/semmle/code/powershell/frameworks/
7+
8+
Follow the steps below in order to generate new type models:
9+
1. Join the `MicrosoftDocs` organisation to ensure that you can access https://github.com/MicrosoftDocs/powershell-docs-sdk-dotnet/tree/main/dotnet/xml (if you haven't already).
10+
2.
11+
Run the following commands
12+
13+
```
14+
# Clone dotnet/dotnet-api-docs
15+
git clone https://github.com/dotnet/dotnet-api-docs
16+
# Clone MicrosoftDocs/powershell-docs-sdk-dotnet
17+
git clone [email protected]:MicrosoftDocs/powershell-docs-sdk-dotnet.git
18+
# Generate data extensions
19+
python3 misc/typemodelgen.py dotnet-api-docs/xml/ powershell-docs-sdk-dotnet/dotnet/xml
20+
```
21+
This will generate 600+ folders that need to be copied into https://github.com/microsoft/codeql/blob/main/powershell/ql/lib/semmle/code/powershell/frameworks/.
22+
23+
Note: Care must be taken to ensure that manually modified versions aren't overwritten.

‎powershell/misc/typemodelgen.py

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
import xml.etree.ElementTree as ET
2+
from pathlib import Path
3+
import sys
4+
import os
5+
from collections import defaultdict
6+
7+
8+
def fixup(t):
9+
"""Sometimes the docs specify a type that doesn't align with what
10+
PowerShell reports. This function fixes up those types so that it aligns with PowerShell.
11+
"""
12+
if t.startswith("System.ReadOnlySpan<"):
13+
return "System.String"
14+
return t
15+
16+
17+
def isStatic(member):
18+
"""Returns True if the member is static, False otherwise."""
19+
for child in member:
20+
if child.tag == "MemberSignature" and "static" in child.attrib["Value"]:
21+
return True
22+
return False
23+
24+
25+
def isA(x):
26+
"""Returns True if member is an `x`."""
27+
for child in member:
28+
if child.tag == "MemberType" and child.text == x:
29+
return True
30+
return False
31+
32+
33+
def isMethod(member):
34+
"""Returns True if the member is a method, False otherwise."""
35+
return isA(member, "Method")
36+
37+
38+
def isField(member):
39+
"""Returns True if the member is a field, False otherwise."""
40+
return isA(member, "Field")
41+
42+
43+
def isProperty(member):
44+
"""Returns True if the member is a property, False otherwise."""
45+
return isA(member, "Property")
46+
47+
48+
def isEvent(member):
49+
"""Returns True if the member is an event, False otherwise."""
50+
return isA(member, "Event")
51+
52+
53+
def isAttachedProperty(member):
54+
"""Returns True if the member is an attached property, False otherwise."""
55+
return isA(member, "AttachedProperty")
56+
57+
58+
def isAttachedEvent(member):
59+
"""Returns True if the member is an attached event, False otherwise."""
60+
return isA(member, "AttachedEvent")
61+
62+
63+
def isConstructor(member):
64+
"""Returns True if the member is a constructor, False otherwise."""
65+
return isA(member, "Constructor")
66+
67+
68+
# A map from filenames to a set of type models to be stored in the file
69+
summaries = defaultdict(set)
70+
71+
72+
def generateTypeModels(arg):
73+
"""Generates type models for the given XML file."""
74+
folder_path = Path(arg)
75+
76+
for file_path in folder_path.rglob("*"):
77+
try:
78+
if not file_path.name.endswith(".xml"):
79+
continue
80+
81+
if not file_path.is_file():
82+
continue
83+
84+
tree = ET.parse(str(file_path))
85+
root = tree.getroot()
86+
if not root.tag == "Type":
87+
continue
88+
89+
thisType = root.attrib["FullName"]
90+
91+
if "`" in file_path.stem or "+" in file_path.stem:
92+
continue # Skip generics (and nested types?) for now
93+
94+
folderName = file_path.parent.name.replace(".", "")
95+
filename = folderName + "/model.yml"
96+
s = set()
97+
for elem in root.findall(".//Members/Member"):
98+
name = elem.attrib["MemberName"]
99+
if name == ".ctor":
100+
continue
101+
102+
staticMarker = ""
103+
if isStatic(elem):
104+
staticMarker = "!"
105+
106+
startSelectorMarker = ""
107+
endSelectorMarker = ""
108+
if isField(elem):
109+
startSelectorMarker = "Field"
110+
endSelectorMarker = ""
111+
if isProperty(elem):
112+
startSelectorMarker = "Property"
113+
endSelectorMarker = ""
114+
if isMethod(elem):
115+
startSelectorMarker = "Method"
116+
endSelectorMarker = ".ReturnValue"
117+
118+
if isEvent(elem):
119+
continue # What are these?
120+
if isAttachedProperty(elem):
121+
continue # What are these?
122+
if isAttachedEvent(elem):
123+
continue # What are these?
124+
if isConstructor(elem):
125+
continue # No need to model the type information for constructors
126+
if startSelectorMarker == "":
127+
print(f"Error: Unknown type for {thisType}.{name}")
128+
continue
129+
130+
if elem.find(".//ReturnValue/ReturnType") is None:
131+
print(f"Error: {name} has no return type!")
132+
continue
133+
134+
returnType = elem.find(".//ReturnValue/ReturnType").text
135+
if returnType == "System.Void":
136+
continue # Don't generate type summaries for void methods
137+
s.add(
138+
f' - ["{fixup(returnType)}", "{thisType + staticMarker}", "{startSelectorMarker}[{name}]{endSelectorMarker}"]\n'
139+
)
140+
141+
summaries[filename].update(s)
142+
143+
except ET.ParseError as e:
144+
print(f"Error parsing XML: {e}")
145+
except Exception as e:
146+
print(f"An error occurred: {repr(e)}")
147+
148+
149+
def writeModels():
150+
"""Writes the type models to disk."""
151+
for filename, s in summaries.items():
152+
if len(s) == 0:
153+
continue
154+
os.makedirs(os.path.dirname(filename), exist_ok=True)
155+
with open(filename, "x") as file:
156+
file.write("extensions:\n")
157+
file.write(" - addsTo:\n")
158+
file.write(" pack: microsoft-sdl/powershell-all\n")
159+
file.write(" extensible: typeModel\n")
160+
file.write(" data:\n")
161+
for summary in s:
162+
for x in summary:
163+
file.write(x)
164+
165+
166+
if __name__ == "__main__":
167+
if len(sys.argv) < 2:
168+
print("Usage: python parse_xml.py <xml_file1> <xml_file2> ... <xml_fileN>")
169+
sys.exit(1)
170+
171+
for arg in sys.argv[1:]:
172+
print(f"Processing {arg}...")
173+
generateTypeModels(arg)
174+
175+
print("Writing models...")
176+
writeModels()

‎powershell/ql/lib/semmle/code/powershell/frameworks/Accessibility/model.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.String", "Accessibility.IAccessible", "Property[accHelp]"]
7+
- ["System.Object", "Accessibility.IAccessible", "Method[accHitTest].ReturnValue"]
8+
- ["System.Int32", "Accessibility._RemotableHandle", "Field[fContext]"]
9+
- ["System.Int32", "Accessibility.IAccessible", "Property[accChildCount]"]
10+
- ["System.Object", "Accessibility.IAccessible", "Property[accState]"]
11+
- ["Accessibility.AnnoScope", "Accessibility.AnnoScope!", "Field[ANNO_THIS]"]
12+
- ["System.Int32", "Accessibility.__MIDL_IWinTypes_0009", "Field[hRemote]"]
13+
- ["System.Object", "Accessibility.IAccessible", "Property[accParent]"]
14+
- ["System.Object", "Accessibility.IAccessible", "Property[accRole]"]
15+
- ["System.Object", "Accessibility.IAccessible", "Property[accChild]"]
16+
- ["System.String", "Accessibility.IAccessible", "Property[accKeyboardShortcut]"]
17+
- ["System.Object", "Accessibility.IAccessible", "Property[accSelection]"]
18+
- ["System.Int32", "Accessibility.IAccessible", "Property[accHelpTopic]"]
19+
- ["System.String", "Accessibility.IAccessible", "Property[accDescription]"]
20+
- ["System.String", "Accessibility.IAccessible", "Property[accDefaultAction]"]
21+
- ["System.Object", "Accessibility.IAccessible", "Property[accFocus]"]
22+
- ["Accessibility.__MIDL_IWinTypes_0009", "Accessibility._RemotableHandle", "Field[u]"]
23+
- ["System.String", "Accessibility.IAccessible", "Property[accValue]"]
24+
- ["System.Int32", "Accessibility.__MIDL_IWinTypes_0009", "Field[hInproc]"]
25+
- ["System.String", "Accessibility.IAccessible", "Property[accName]"]
26+
- ["Accessibility.AnnoScope", "Accessibility.AnnoScope!", "Field[ANNO_CONTAINER]"]
27+
- ["System.Object", "Accessibility.IAccessible", "Method[accNavigate].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/IEHostExecute/model.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Int32", "IEHost.Execute.IEExecuteRemote", "Method[ExecuteAsDll].ReturnValue"]
7+
- ["System.IO.Stream", "IEHost.Execute.IEExecuteRemote", "Property[Exception]"]
8+
- ["System.Object", "IEHost.Execute.IEExecuteRemote", "Method[InitializeLifetimeService].ReturnValue"]
9+
- ["System.Int32", "IEHost.Execute.IEExecuteRemote", "Method[ExecuteAsAssembly].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftActivitiesBuild/model.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Boolean", "Microsoft.Activities.Build.WorkflowBuildMessageTask", "Method[Execute].ReturnValue"]
7+
- ["System.String", "Microsoft.Activities.Build.WorkflowBuildMessageTask", "Property[ResourceName]"]
8+
- ["System.String", "Microsoft.Activities.Build.WorkflowBuildMessageTask", "Property[MessageType]"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftActivitiesBuildDebugger/model.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Boolean", "Microsoft.Activities.Build.Debugger.DebugBuildExtension", "Method[Execute].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftActivitiesBuildExpressions/model.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Boolean", "Microsoft.Activities.Build.Expressions.ExpressionsBuildExtension", "Method[Execute].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftActivitiesBuildValidation/model.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Boolean", "Microsoft.Activities.Build.Validation.ReportDeferredValidationErrorsTask", "Method[Execute].ReturnValue"]
7+
- ["System.String", "Microsoft.Activities.Build.Validation.DeferredValidationTask", "Property[DeferredValidationErrorsFilePath]"]
8+
- ["System.String", "Microsoft.Activities.Build.Validation.ReportDeferredValidationErrorsTask", "Property[DeferredValidationErrorsFilePath]"]
9+
- ["System.Boolean", "Microsoft.Activities.Build.Validation.DeferredValidationTask", "Method[Execute].ReturnValue"]

‎powershell/ql/lib/semmle/code/powershell/frameworks/MicrosoftAspnetSnapin/model.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
extensions:
2+
- addsTo:
3+
pack: microsoft-sdl/powershell-all
4+
extensible: typeModel
5+
data:
6+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IExtendPropertySheet2", "Method[CreatePropertyPages].ReturnValue"]
7+
- ["Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE", "Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE!", "Field[TOOLBAR]"]
8+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[mask]"]
9+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[EnumDAdvise].ReturnValue"]
10+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[cChildren]"]
11+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[lParam]"]
12+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IExtendPropertySheet", "Method[QueryPagesFor].ReturnValue"]
13+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[relativeID]"]
14+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[DAdvise].ReturnValue"]
15+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[EnumFormatEtc].ReturnValue"]
16+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IExtendPropertySheet2", "Method[GetWatermarks].ReturnValue"]
17+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IContextMenuCallback", "Method[AddItem].ReturnValue"]
18+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[nOpenImage]"]
19+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[QueryGetData].ReturnValue"]
20+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[ID]"]
21+
- ["System.IntPtr", "Microsoft.Aspnet.Snapin.AspNetManagementUtility!", "Method[GetActiveWindow].ReturnValue"]
22+
- ["Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE", "Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE!", "Field[COMBOBOXBAR]"]
23+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[nState]"]
24+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[SetData].ReturnValue"]
25+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[GetData].ReturnValue"]
26+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IExtendPropertySheet", "Method[CreatePropertyPages].ReturnValue"]
27+
- ["System.IntPtr", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[displayname]"]
28+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[DUnadvise].ReturnValue"]
29+
- ["System.Int32", "Microsoft.Aspnet.Snapin.SCOPEDATAITEM", "Field[nImage]"]
30+
- ["Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE", "Microsoft.Aspnet.Snapin.MMC_CONTROL_TYPE!", "Field[MENUBUTTON]"]
31+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[GetDataHere].ReturnValue"]
32+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IExtendPropertySheet2", "Method[QueryPagesFor].ReturnValue"]
33+
- ["System.Int32", "Microsoft.Aspnet.Snapin.IDataObject", "Method[GetCanonicalFormatEtc].ReturnValue"]
34+
- ["System.Int32", "Microsoft.Aspnet.Snapin.AspNetManagementUtility!", "Method[MessageBox].ReturnValue"]

0 commit comments

Comments
 (0)