Skip to content

Commit b354326

Browse files
j3ssiej3ssie
committed
Added TypeScript analysis, Optimize the build image, and also added a new helper script
1 parent 1983eff commit b354326

File tree

7 files changed

+310
-91
lines changed

7 files changed

+310
-91
lines changed

‎Dockerfile‎

Lines changed: 3 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,8 @@
1-
FROM ubuntu:20.04 AS codeql_base
2-
LABEL maintainer="Github codeql team"
3-
4-
# tzdata install needs to be non-interactive
5-
ENV DEBIAN_FRONTEND=noninteractive
6-
7-
# install/update basics and python
8-
RUN apt-get update && \
9-
apt-get upgrade -y && \
10-
apt-get install -y --no-install-recommends \
11-
software-properties-common \
12-
vim \
13-
curl \
14-
wget \
15-
git \
16-
jq \
17-
build-essential \
18-
unzip \
19-
apt-transport-https \
20-
python3.8 \
21-
python3-venv \
22-
python3-pip \
23-
python3-setuptools \
24-
python3-dev \
25-
gnupg \
26-
g++ \
27-
make \
28-
gcc \
29-
apt-utils \
30-
rsync \
31-
file \
32-
dos2unix \
33-
default-jdk \
34-
gettext && \
35-
apt-get clean && \
36-
ln -sf /usr/bin/python3.8 /usr/bin/python && \
37-
ln -sf /usr/bin/pip3 /usr/bin/pip
38-
39-
# Install Golang
40-
RUN wget -q -O - https://raw.githubusercontent.com/canha/golang-tools-install-script/master/goinstall.sh | bash
41-
42-
# Install latest codeQL
1+
FROM j3ssie/codeql-base:latest
432
ENV CODEQL_HOME /root/codeql-home
44-
45-
# Get CodeQL verion
46-
RUN curl --silent "https://api.github.com/repos/github/codeql-cli-binaries/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' > /tmp/codeql_version
47-
48-
# record the latest version of the codeql-cli
49-
RUN mkdir -p ${CODEQL_HOME} \
50-
${CODEQL_HOME}/codeql-repo \
51-
${CODEQL_HOME}/codeql-go-repo \
52-
/opt/codeql
53-
54-
# get the latest codeql queries and record the HEAD
55-
RUN git clone --depth=1 https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
56-
git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
57-
RUN git clone --depth=1 https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \
58-
git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit
59-
60-
RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
61-
wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
62-
unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME} && \
63-
rm /tmp/codeql_linux.zip
64-
653
ENV PATH="$PATH:${CODEQL_HOME}/codeql:/root/go/bin:/root/.go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
4+
COPY containers /root/containers
665
COPY scripts /root/scripts
676

68-
# Pre-compile our queries to save time later
69-
RUN /root/scripts/compile-qs.sh
70-
717
WORKDIR /root/
72-
ENTRYPOINT ["/root/scripts/analyze.sh"]
8+
ENTRYPOINT ["/root/scripts/analyze.py"]

‎README.md‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ cat results/issues.sarif-latest| jq '.runs[].results'
3131
# with custom format and output
3232
./scripts/run.py -l javascript -s /tmp/cc/code-scanning-javascript-demo --format=csv -o sample
3333
# your output will be store at sample/issues.csv
34-
3534
```
3635

3736
### Run with docker command
@@ -41,12 +40,15 @@ With `/tmp/src` is your source code and `/tmp/results` is where result store.
4140
> NOTE: make sure /tmp/results folder exist otherwise it won't work
4241
4342
```shell
44-
# simple usage
43+
# simple usage which will run the QL Packs of that language
4544
docker run --rm --name codeql-docker -v "/tmp/src:/opt/src" -v "/tmp/results:/opt/results" -e "LANGUAGE=go" j3ssie/codeql-docker:latest
4645

47-
# more options
46+
# Run with specific Queries Suite and different output format
4847
docker run --rm --name codeql-docker -v "/tmp/src:/opt/src" -v "/tmp/results:/opt/results" -e "LANGUAGE=javascript" -e "FORMAT=csv" -e "QS=javascript-security-and-quality.qls" j3ssie/codeql-docker:latest
4948

49+
# Override the source code DB tree
50+
docker run --rm --name codeql-docker -v "/tmp/src:/opt/src" -v "/tmp/results:/opt/results" -e "LANGUAGE=javascript" -e "FORMAT=csv" -e "QS=javascript-security-and-quality.qls" -e "OVERRIDE=True" j3ssie/codeql-docker:latest
51+
5052
```
5153

5254
### Manual analyze
@@ -58,12 +60,16 @@ docker run -it --entrypoint=/bin/bash -t j3ssie/codeql-docker:latest
5860
# Copy your code to container
5961
docker cp <your-source-cde> <docker-ID>:/opt/src
6062

61-
# create DB in this folder /opt/src/db
62-
# This might take a while depend on your code
63+
# You use the helper scripts to run CodeQL
64+
python3 analyze.py -d /opt/src/db -s /opt/src/ -l javascript --override=True
65+
66+
# Or using raw command from codeQL
67+
## create DB in this folder /opt/src/db
68+
## This might take a while depend on your code
6369
codeql database create --language=<language> /opt/src/db -s /opt/src
6470

65-
# run analyze
66-
# normally query-suites will will be: <language>-security-and-quality.qls
71+
## run analyze
72+
## normally query-suites will will be: <language>-security-and-quality.qls
6773
codeql database analyze --format=sarif-latest --output=/opt/issues.sarif /opt/src/db <query-suites>
6874

6975
# copy the result back to host machine
@@ -73,12 +79,17 @@ docker cp <docker-ID>:/opt/issues.sarif .
7379
### Other commands
7480

7581
```shell
82+
# List all query packs
83+
codeql resolve qlpacks --format=json | jq -r 'keys[]'
84+
7685
# List all query suites
7786
codeql resolve queries
7887

7988
# Upgrade DB
8089
codeql database upgrade <database>
8190

91+
# Building the base image
92+
docker build -f base-image-Dockerfile -t j3ssie/codeql-base:latest .
8293
```
8394

8495
## Donation

‎base-image-Dockerfile‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
FROM ubuntu:20.04 AS codeql_base
2+
LABEL maintainer="Github codeql team"
3+
4+
# tzdata install needs to be non-interactive
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# install/update basics and python
8+
RUN apt-get update && \
9+
apt-get upgrade -y && \
10+
apt-get install -y --no-install-recommends \
11+
software-properties-common \
12+
vim \
13+
curl \
14+
wget \
15+
git \
16+
jq \
17+
build-essential \
18+
unzip \
19+
apt-transport-https \
20+
python3.8 \
21+
python3-venv \
22+
python3-pip \
23+
python3-setuptools \
24+
python3-dev \
25+
gnupg \
26+
g++ \
27+
make \
28+
gcc \
29+
apt-utils \
30+
rsync \
31+
file \
32+
dos2unix \
33+
default-jdk \
34+
gettext
35+
# support typescript
36+
RUN apt-get install nodejs -y -qq
37+
RUN apt-get clean && \
38+
ln -sf /usr/bin/python3.8 /usr/bin/python && \
39+
ln -sf /usr/bin/pip3 /usr/bin/pip
40+
41+
# Install Golang
42+
RUN wget -q -O - https://raw.githubusercontent.com/canha/golang-tools-install-script/master/goinstall.sh | bash
43+
44+
# Install latest codeQL
45+
ENV CODEQL_HOME /root/codeql-home
46+
47+
# Get CodeQL verion
48+
RUN curl --silent "https://api.github.com/repos/github/codeql-cli-binaries/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' > /tmp/codeql_version
49+
50+
# record the latest version of the codeql-cli
51+
RUN mkdir -p ${CODEQL_HOME} \
52+
${CODEQL_HOME}/codeql-repo \
53+
${CODEQL_HOME}/codeql-go-repo \
54+
/opt/codeql
55+
56+
# get the latest codeql queries and record the HEAD
57+
RUN git clone --depth=1 https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
58+
git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
59+
60+
RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
61+
wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
62+
unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME} && \
63+
rm /tmp/codeql_linux.zip
64+
65+
ENV PATH="$PATH:${CODEQL_HOME}/codeql:/root/go/bin:/root/.go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
66+
COPY containers /root/containers
67+
# Pre-compile our queries to save time later
68+
RUN /root/containers/compile-qs.sh
69+

scripts/analyze.sh renamed to containers/analyze.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
RED="\033[31m"
44
YELLOW="\033[33m"
5+
CYAN="\033[96m"
56
GREEN="\033[32m"
67
RESET="\033[0m"
78

@@ -35,8 +36,8 @@ print_green " [+] Output: $OUTPUT"
3536
print_green " [+] Format: $FORMAT"
3637
echo "----------------"
3738

38-
echo -e "Creating DB: codeql database create --language=$LANGUAGE $DB -s $SRC"
39+
echo -e "$CYAN ==> Creating DB:$GREEN codeql database create --language=$LANGUAGE $DB -s $SRC"
3940
codeql database create --language=$LANGUAGE $DB -s $SRC
4041

41-
echo -e "Start Scanning: codeql database analyze --format=$FORMAT --output=$OUTPUT/issues.$FORMAT $DB $QS"
42-
codeql database analyze --format=$FORMAT --output=$OUTPUT/issues.$FORMAT $DB $QS
42+
echo -e "$CYAN ==> Start Scanning:$GREEN codeql database analyze --format=$FORMAT --output=$OUTPUT/issues.$FORMAT $DB $QS"
43+
codeql database analyze --overwrite --format=$FORMAT --output=$OUTPUT/issues.$FORMAT $DB $QS

scripts/compile-qs.sh renamed to containers/compile-qs.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@ print_green() {
1212
print_green "[+] Start Compiling query suites"
1313

1414
# get all query suites except Java
15-
ls /root/codeql-home/codeql-go-repo/ql/src/codeql-suites/*.qls > /tmp/list-querysuites
15+
ls /root/codeql-home/codeql-repo/go/ql/src/codeql-suites/*.qls > /tmp/list-querysuites
1616
ls /root/codeql-home/codeql-repo/*/ql/src/codeql-suites/*.qls | grep -v 'java-' | grep -v 'csharp' >> /tmp/list-querysuites
1717

1818
while IFS="" read -r qs || [ -n "$qs" ]
1919
do
2020
print_green "[+] Compiling query suites: $qs"
21-
codeql query compile --threads=0 $qs
21+
codeql query compile --threads=20 $qs
2222
done < /tmp/list-querysuites
2323

2424
print_green "[+] Done Compiling query suites ..."

0 commit comments

Comments
 (0)