Skip to content

Commit f9fc28e

Browse files
authored
Merge pull request #81 from trailofbits/polytracker-python
PolyTracker Python and Installation Refactor
2 parents bc3382e + 91b6a6e commit f9fc28e

File tree

25 files changed

+501
-73
lines changed

25 files changed

+501
-73
lines changed

.github/workflows/dockerimage.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ jobs:
3737
--exclude 'polytracker/src/dfsan_rt/interception/*'
3838
- name: Python lint/typecheck
3939
run: |
40-
black --check polybuild polyprocess tests --line-length=127
41-
mypy --ignore-missing-imports polybuild polyprocess tests
42-
- name: Build the base image
40+
black --check polytracker tests --exclude '/(polytracker/src|polytracker/scripts)/' --line-length=127
41+
mypy --ignore-missing-imports polytracker tests
42+
- name: Build the base image
4343
run: docker build . --file Dockerfile --tag trailofbits/polytracker --no-cache
4444
- name: Poly* tests
4545
run: |
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# This workflows will upload a Python Package using Twine when a release is created
2+
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
3+
4+
name: Upload Python Package
5+
6+
on:
7+
release:
8+
types: [published]
9+
10+
jobs:
11+
deploy:
12+
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: Set up Python
18+
uses: actions/setup-python@v1
19+
with:
20+
python-version: '3.x'
21+
- name: Install dependencies
22+
run: |
23+
python -m pip install --upgrade pip
24+
pip install setuptools wheel twine
25+
- name: Build and publish
26+
env:
27+
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
28+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
29+
run: |
30+
python setup.py sdist bdist_wheel
31+
twine upload dist/*

CMakeLists.txt

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,15 @@ project(TAPP)
55
# If there is no explicit -DCMAKE_INSTALL_PREFIX=DIR setting given,
66
# then install underneath the build directory
77
if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
8-
set (CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/bin CACHE PATH "default install path" FORCE)
8+
set (CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR} CACHE PATH "default install path" FORCE)
99
endif()
1010

11-
set(POLYTRACK_BIN_DIR "${CMAKE_INSTALL_PREFIX}/polytracker")
12-
set(POLYTRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/lib")
13-
set(POLYTRACK_TRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/track")
14-
set(POLYTRACK_RULE_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/abi_lists")
15-
set(POLYTRACK_TESTS_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/tests")
16-
set(POLYTRACK_PASS_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/pass")
17-
set(POLYTRACK_CXX_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/")
11+
set(POLYTRACK_BIN_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/bin")
12+
set(POLYTRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/lib")
13+
set(POLYTRACK_TRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/track")
14+
set(POLYTRACK_RULE_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/abi_lists")
15+
set(POLYTRACK_TESTS_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/tests")
16+
set(POLYTRACK_PASS_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/pass")
17+
set(POLYTRACK_CXX_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker")
1818

1919
add_subdirectory(polytracker)
20-
install(DIRECTORY "./polybuild" DESTINATION ${POLYTRACK_CXX_DIR})
21-
22-

Dockerfile

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y update \
2626
libgraphviz-dev \
2727
graphviz
2828

29-
RUN python3.7 -m pip install pip
29+
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10
30+
RUN python3 -m pip install pip
3031

3132
RUN go get github.com/SRI-CSL/gllvm/cmd/...
3233

@@ -36,12 +37,7 @@ COPY . /polytracker
3637

3738
WORKDIR /polytracker
3839

39-
RUN python3.7 -m pip install pytest
40-
41-
RUN python3.7 -m pip install .
42-
43-
RUN rm /usr/bin/python3
44-
RUN cp /usr/bin/python3.7 /usr/bin/python3
40+
RUN pip3 install pytest .
4541

4642
RUN rm -rf build && mkdir -p build
4743

@@ -50,13 +46,13 @@ WORKDIR /polytracker/build
5046
ENV PATH="/usr/lib/llvm-7/bin:${PATH}"
5147

5248
RUN cmake -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_VERBOSE_MAKEFILE=TRUE .. && ninja install
53-
ENV CC=/polytracker/build/bin/polytracker/polybuild/polybuild.py
54-
ENV CXX=/polytracker/build/bin/polytracker/polybuild/polybuild++.py
49+
ENV PATH="/polytracker/build/bin/:${PATH}"
50+
ENV CC=polybuild
51+
ENV CXX=polybuild++
5552
ENV LLVM_COMPILER=clang
56-
RUN chmod +x ${CC}
5753
RUN mkdir -p "/build_artifacts"
5854

5955
# Set the BC store path to the <install_path>/cxx_libs/bitcode/bitcode_store}
60-
ENV WLLVM_BC_STORE="/polytracker/build/bin/polytracker/cxx_libs/bitcode/bitcode_store"
56+
ENV WLLVM_BC_STORE="/polytracker/build/share/polytracker/cxx_libs/bitcode/bitcode_store"
6157
ENV WLLVM_ARTIFACT_STORE="/build_artifacts"
6258
WORKDIR /polytracker

format.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ clang-format -i polytracker/include/polyclang/*.h
1010
clang-format -i polytracker/include/dfsan/*.h
1111

1212
# Black to auto format code, mypy for type checking
13-
black polybuild polyprocess tests --line-length=127
14-
mypy --ignore-missing-imports polybuild polyprocess tests
13+
black polytracker tests --exclude '/(polytracker/src|polytracker/scripts)/' --line-length=127
14+
mypy --ignore-missing-imports polytracker tests

polybuild/polybuild++.py

Lines changed: 0 additions & 1 deletion
This file was deleted.

polyprocess/__init__.py

Lines changed: 0 additions & 1 deletion
This file was deleted.

polytracker/CMakeLists.txt

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,27 @@ project(polytracker LANGUAGES C CXX ASM)
22

33
include_directories(include)
44

5-
set(POLYTRACK_BIN_DIR "${CMAKE_INSTALL_PREFIX}/polytracker")
6-
set(POLYTRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/lib")
7-
set(POLYTRACK_TRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/track")
8-
set(POLYTRACK_RULE_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/abi_lists")
9-
set(POLYTRACK_TESTS_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/tests")
10-
set(POLYTRACK_PASS_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/pass")
11-
set(POLYTRACK_CXX_DIR "${CMAKE_INSTALL_PREFIX}/polytracker/")
5+
set(POLYTRACK_BIN_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/bin")
6+
set(POLYTRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/lib")
7+
set(POLYTRACK_TRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/track")
8+
set(POLYTRACK_RULE_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/abi_lists")
9+
set(POLYTRACK_TESTS_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/tests")
10+
set(POLYTRACK_PASS_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/pass")
11+
set(POLYTRACK_CXX_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/")
1212

1313
add_subdirectory(src)
1414
#add_subdirectory(custom_abi)
1515
install(DIRECTORY "./cxx_libs" DESTINATION ${POLYTRACK_CXX_DIR})
1616
install(DIRECTORY "./abi_lists" DESTINATION ${POLYTRACK_CXX_DIR})
17+
18+
install(PROGRAMS "scripts/polybuild.py" DESTINATION ${POLYTRACK_BIN_DIR} RENAME "polybuild")
19+
install(PROGRAMS "scripts/polybuild.py" DESTINATION ${POLYTRACK_BIN_DIR} RENAME "polybuild++")
20+
21+
macro(install_symlink filepath sympath)
22+
install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${filepath} ${sympath})")
23+
install(CODE "message(\"-- Created symlink: ${sympath} -> ${filepath}\")")
24+
endmacro(install_symlink)
25+
26+
install(DIRECTORY DESTINATION "${CMAKE_INSTALL_PREFIX}/bin")
27+
install_symlink(../share/polytracker/bin/polybuild ${CMAKE_INSTALL_PREFIX}/bin/polybuild)
28+
install_symlink(../share/polytracker/bin/polybuild ${CMAKE_INSTALL_PREFIX}/bin/polybuild++)

polytracker/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .polytracker import *

polytracker/cfg.py

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
import math
2+
3+
from typing import (
4+
Callable,
5+
Collection,
6+
Dict,
7+
FrozenSet,
8+
Generic,
9+
ItemsView,
10+
Iterable,
11+
KeysView,
12+
List,
13+
Optional,
14+
Set,
15+
TypeVar,
16+
Union,
17+
)
18+
19+
import graphviz
20+
import networkx as nx
21+
22+
N = TypeVar("N")
23+
24+
25+
class DiGraph(nx.DiGraph, Generic[N]):
26+
def __init__(self, *args, **kwargs):
27+
super().__init__(*args, **kwargs)
28+
self._dominator_forest: Optional[DiGraph[N]] = None
29+
self._roots: Optional[Collection[N]] = None
30+
self._path_lengths: Optional[Dict[N, Dict[N, int]]] = None
31+
32+
def path_length(self, from_node: N, to_node: N) -> Union[int, float]:
33+
if self._path_lengths is None:
34+
self._path_lengths = dict(nx.all_pairs_shortest_path_length(self, cutoff=None))
35+
if from_node not in self._path_lengths or to_node not in self._path_lengths[from_node]:
36+
return math.inf
37+
else:
38+
return self._path_lengths[from_node][to_node]
39+
40+
def set_roots(self, roots: Collection[N]):
41+
self._roots = roots
42+
43+
def _find_roots(self) -> Iterable[N]:
44+
return (n for n, d in self.in_degree() if d == 0)
45+
46+
@property
47+
def roots(self) -> Collection[N]:
48+
if self._roots is None:
49+
self._roots = tuple(self._find_roots())
50+
return self._roots
51+
52+
def depth(self, node: N) -> Union[int, float]:
53+
return min(self.path_length(root, node) for root in self.roots)
54+
55+
def ancestors(self, node: N) -> Set[N]:
56+
return nx.ancestors(self, node)
57+
58+
def descendants(self, node: N) -> FrozenSet[N]:
59+
return frozenset(nx.dfs_successors(self, node).keys())
60+
61+
@property
62+
def dominator_forest(self) -> "DAG[N]":
63+
if self._dominator_forest is not None:
64+
return self._dominator_forest
65+
self._dominator_forest = DAG()
66+
for root in self.roots:
67+
for node, dominated_by in nx.immediate_dominators(self, root).items():
68+
if node != dominated_by:
69+
self._dominator_forest.add_edge(dominated_by, node)
70+
return self._dominator_forest
71+
72+
def to_dot(
73+
self, comment: Optional[str] = None, labeler: Optional[Callable[[N], str]] = None, node_filter=None
74+
) -> graphviz.Digraph:
75+
if comment is not None:
76+
dot = graphviz.Digraph(comment=comment)
77+
else:
78+
dot = graphviz.Digraph()
79+
if labeler is None:
80+
labeler = str
81+
node_ids = {node: i for i, node in enumerate(self.nodes)}
82+
for node in self.nodes:
83+
if node_filter is None or node_filter(node):
84+
dot.node(f"func{node_ids[node]}", label=labeler(node))
85+
for caller, callee in self.edges:
86+
if node_filter is None or (node_filter(caller) and node_filter(callee)):
87+
dot.edge(f"func{node_ids[caller]}", f"func{node_ids[callee]}")
88+
return dot
89+
90+
91+
class DAG(DiGraph[N], Generic[N]):
92+
def vertex_induced_subgraph(self, vertices: Iterable[N]) -> "DAG[N]":
93+
vertices = frozenset(vertices)
94+
subgraph = self.copy()
95+
to_remove = set(self.nodes) - vertices
96+
for v in vertices:
97+
node = v
98+
parent = None
99+
while True:
100+
parents = tuple(subgraph.predecessors(node))
101+
if not parents:
102+
if parent is not None:
103+
subgraph.remove_edge(parent, v)
104+
subgraph.add_edge(node, v)
105+
break
106+
assert len(parents) == 1
107+
ancestor = parents[0]
108+
if parent is None:
109+
parent = ancestor
110+
if ancestor in vertices:
111+
to_remove.add(v)
112+
break
113+
node = ancestor
114+
subgraph.remove_nodes_from(to_remove)
115+
return subgraph
116+
117+
118+
class FunctionInfo:
119+
def __init__(
120+
self,
121+
name: str,
122+
cmp_bytes: Dict[str, List[int]],
123+
input_bytes: Dict[str, List[int]] = None,
124+
called_from: Iterable[str] = (),
125+
):
126+
self.name: str = name
127+
self.called_from: FrozenSet[str] = frozenset(called_from)
128+
self.cmp_bytes: Dict[str, List[int]] = cmp_bytes
129+
if input_bytes is None:
130+
self.input_bytes: Dict[str, List[int]] = cmp_bytes
131+
else:
132+
self.input_bytes = input_bytes
133+
134+
@property
135+
def taint_sources(self) -> KeysView[str]:
136+
return self.input_bytes.keys()
137+
138+
def __getitem__(self, input_source_name: str) -> List[int]:
139+
return self.input_bytes[input_source_name]
140+
141+
def __iter__(self) -> Iterable[str]:
142+
return self.taint_sources
143+
144+
def items(self) -> ItemsView[str, List[int]]:
145+
return self.input_bytes.items()
146+
147+
def __hash__(self):
148+
return hash(self.name)
149+
150+
def __str__(self):
151+
return self.name
152+
153+
def __repr__(self):
154+
return f"{self.__class__.__name__}(name={self.name!r}, cmp_bytes={self.cmp_bytes!r}, input_bytes={self.input_bytes!r}, called_from={self.called_from!r})"
155+
156+
157+
class CFG(DiGraph[FunctionInfo]):
158+
def __init__(self):
159+
super().__init__()
160+
161+
def to_dot(
162+
self,
163+
comment: Optional[str] = "PolyTracker Program Trace",
164+
labeler: Optional[Callable[[FunctionInfo], str]] = None,
165+
node_filter=None,
166+
) -> graphviz.Digraph:
167+
function_labels: Dict[str, str] = {}
168+
169+
def func_labeler(f):
170+
if labeler is not None:
171+
return labeler(f)
172+
elif f.name in function_labels:
173+
return f"{f.name} ({function_labels[f.name]})"
174+
else:
175+
return f.name
176+
177+
return super().to_dot(comment, labeler=func_labeler, node_filter=node_filter)

0 commit comments

Comments
 (0)