Merge pull request #629 from qilingframework/dev

1.2.1 ready

Author: xwings

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,41 @@
-- [ ] Have you checked that there aren't other open [pull requests](https://github.com/qilingframework/qiling/pulls) for the same update/change?
-- [ ] Is this pull request for [dev](https://github.com/qilingframework/qiling/tree/dev) branch? (Do not pull request for **master** branch).
+<!-- 
+We highly appreciate your interest and contribution to our project. 
+Before submiting your PR, please finish the checklist below. 
+-->
+
+## Checklist
+
+### Which kind of PR do you create?
+
+- [ ] This PR only contains minor fixes.
+- [ ] This PR contains major feature update.
+- [ ] This PR introduces a new function/api for Qiling Framework.
+
+### Coding convention?
+
+- [ ] The new code conforms to Qiling Framework naming convention.
+- [ ] The imports are arranged properly.
+- [ ] Essential comments are added.
+- [ ] The reference of the new code is pointed out.
+
+### Extra tests?
+
+- [ ] No extra tests are needed for this PR.
+- [ ] I have added enough tests for this PR.
+- [ ] Tests will be added after some discussion and review.
+
+### Changelog?
+
+- [ ] This PR doesn't need to update Changelog.
+- [ ] Changelog will be updated after some proper review.
+- [ ] Changelog has been updated in my PR.
+
+### Target branch?
+
+- [ ] The target branch is dev branch.
+
+### One last thing
+
+- [ ] I have read the [contribution guide](https://docs.qiling.io/en/latest/contribution/)
 
 -----

.github/workflows/build-ci.yml

@@ -0,0 +1,79 @@
+name: Run tests
+on: [push, pull_request]
+env:
+  CI: true
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04]
+        python-version: [3.6.8, 3.7.6]
+        exclude:
+          - os: windows-2019
+            python-version: 3.7.6
+          - os: macos-10.15
+            python-version: 3.7.6
+          - os: ubuntu-20.04
+            python-version: 3.7.6
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: win setup MSVC
+      if: contains(matrix.os, 'windows')
+      uses: microsoft/setup-msbuild@v1
+
+    - name: win run tests
+      if: contains(matrix.os, 'windows')
+      shell: bash
+      run: |
+         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
+         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
+         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
+         powershell Add-MpPreference -ExclusionPath $GITHUB_WORKSPACE
+         pip3 install setuptools wheel
+         pip3 install .
+         cmd.exe //C 'examples\scripts\dllscollector.bat'
+         cd $GITHUB_WORKSPACE/examples/rootfs/x86_windows/bin
+         unzip -Pinfected wannacry.bin.zip
+         unzip -Pinfected UselessDisk.bin.zip
+         unzip -Pinfected GandCrab502.bin.zip
+         unzip -Pinfected al-khaser.bin.zip
+         unzip -Pinfected sality.dll.zip
+         cd $GITHUB_WORKSPACE/tests
+         cmd.exe //C '.\test_pe.bat'
+    - name: mac run tests
+      if: contains(matrix.os, 'macos')
+      continue-on-error: true
+      shell: bash
+      run: |
+         pip3 install setuptools wheel
+         pip3 install .
+         ./examples/scripts/dylibcollector.sh
+         cd $GITHUB_WORKSPACE/examples/rootfs/x8664_macos/kext
+         unzip -Pinfected SuperRootkit.kext.zip
+         cd $GITHUB_WORKSPACE/tests
+         ./test_macho.sh
+    - name: linux run tests
+      if: contains(matrix.os, 'ubuntu')
+      shell: 'script -q -e -c "bash {0}"'
+      run: |
+        if [ ${{ matrix.os }} == 'ubuntu-18.04' ] && [ ${{ matrix.python-version }} == '3.7.6' ]; then
+              pip3 install setuptools wheel flake8
+              flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+              pip3 install .
+              cd tests && ./test_elf.sh
+        elif [ ${{ matrix.os }} == 'ubuntu-20.04' ]; then
+              docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd tests && ./test_elf.sh"
+        else
+              pip3 install setuptools wheel
+              pip3 install .
+              cd tests && ./test_elf.sh
+        fi

.gitignore

@@ -32,3 +32,5 @@ test.file
 *.o
 core
 *.perf
+examples/rootfs/x86_windows/Windows/registry
+examples/rootfs/x8664_windows/Windows/registry

.travis.yml

@@ -2,34 +2,6 @@ language: shell
 matrix:
   fast_finish: true
   include:
-    - name: "Python 3.6 on Linux"
-      os: linux
-      dist: bionic
-      language: python
-      python: "3.6"
-      cache: pip
-      install:
-        - pip3 install .
-      before_script:
-        - cd tests
-      script:
-        - ./test_elf.sh
-
-    - name: "Python 3.7 on Linux"
-      os: linux
-      dist: bionic
-      language: python
-      python: "3.7"
-      cache: pip
-      install:
-        - pip3 install flake8
-        - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        - pip3 install .
-      before_script:
-        - cd tests
-      script:
-        - ./test_elf.sh
-
     - name: "Python 3.7 on OSX"
       os: osx
       osx_image: xcode10.1
@@ -61,101 +33,3 @@ matrix:
       script:
         - ./test_macho.sh
 
-    - name: "Python 3.6.8 on Windows"
-      os: windows
-      language: shell
-      env:
-        - PATH="/c/Python36:/c/Python36/Scripts:$PATH"
-      cache:
-        directories:
-#          - $HOME/AppData/Local/Temp/chocolatey
-          - $HOME/AppData/Local/pip/Cache
-          - /c/Python36
-#          - $HOME/AppData/Local/NuGet/Cache
-      before_install:
-        - | 
-          if [[ ! -f /c/Python36/python ]]; then 
-               choco install python --version=3.6.8
-          fi
-        # Prevent worker settings failure 
-        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
-        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
-        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
-        #- choco install kb2999226
-        #- python -m pip install --upgrade pip
-      install:
-#        - pip3 install wheel 'capstone>=4.0.1' 'pefile>=2019.4.18' 'python-registry>=1.3.1' 'unicorn>=1.0.2rc3'
-#        - |
-#          if [ ! -f $HOME/dist/keystone*.zip ]; then
-#               git clone https://github.com/keystone-engine/keystone && \
-#               cd keystone && \
-#               mkdir build && \
-#               cd build && \
-#               cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&' cmd.exe //C '..\nmake-dll.bat' '&&' nmake instalL && \
-#               cd ../bindings/python && \
-#               python setup.py install && \
-#               python setup.py bdist --formats=zip && \
-#               cp dist/*.zip $HOME/dist/ && \
-#               cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll /c/Python36/Lib/site-packages/keystone/ && \
-#               cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll $HOME/dist/;
-#          else
-#              unzip $HOME/dist/*.zip -d /c && \
-#              cp $HOME/dist/keystone.dll /c/Python36/Lib/site-packages/keystone/;
-#          fi
-        - cd $TRAVIS_BUILD_DIR
-        - pip3 install .
-#        - cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll /c/Python36/Lib/site-packages/keystone/
-      before_script:
-        - cmd.exe //C 'examples\scripts\dllscollector.bat'
-        - cd $TRAVIS_BUILD_DIR/examples/rootfs/x86_windows/bin
-        - unzip -Pinfected wannacry.bin.zip
-        - unzip -Pinfected UselessDisk.bin.zip
-        - unzip -Pinfected GandCrab502.bin.zip
-        - unzip -Pinfected al-khaser.bin.zip
-        - unzip -Pinfected sality.dll.zip
-        - cd $TRAVIS_BUILD_DIR/tests
-      script:
-        - cmd.exe //C '.\test_pe.bat' 
-
-    - name: "Python 3.6 on Docker"
-      services:
-        - docker
-#      cache:
-#        bundler: true
-#        directories:
-#          - $HOME/docker
-      before_script:
-#        - docker build -t qiling:1.1 .
-#        - docker load -i $HOME/docker/images.tar || true
-#        - |
-#          if [[ ! $(docker images -q qiling:1.1) ]]; then
-#              docker build -t qiling:1.1 . && docker save -o $HOME/docker/images.tar qiling:1.1;
-#          fi
-#        - docker run -dt --name qiling -v ${TRAVIS_BUILD_DIR}:/qiling qiling:1.1
-#        - docker exec qiling pip3 install -r requirements.txt
-#        - docker exec qiling python3 setup.py install
-      script:
-        - docker run -it --rm -v ${TRAVIS_BUILD_DIR}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd tests && ./test_elf.sh"
-#        - docker exec qiling bash -c "cd tests && ./test_elf.sh"
-
-#    - name: "Python 3.6 on WSL1 Ubuntu"
-#      os: windows
-#      language: shell
-#      before_install:
-#        - choco install wsl-ubuntu-1804
-#        - wsl apt update
-#        - wsl DEBIAN_FRONTEND=noninteractive apt dist-upgrade -y
-#        - wsl DEBIAN_FRONTEND=noninteractive apt install -y python3-pip cmake
-#        - wsl pip3 install --upgrade pip
-#      install:
-#        - wsl pip3 install -r requirements.txt
-#        - wsl python3 setup.py install
-#      before_script:
-#        - MSYS_NO_PATHCONV=1 wsl cp $(wsl python3 -c "import site; print(site.getsitepackages()[0])")$(wsl python3 -c "import site; print(site.getsitepackages()[1])")/keystone/libkeystone.so $(wsl python3 -c "import site; print(site.getsitepackages()[0])")/keystone/
-#        - cd tests
-#      script:
-#        - wsl python3 test_all.py
-
-#  allow_failures:
-#    - os: osx
-#    - services: docker

CREDITS.TXT

@@ -49,6 +49,8 @@ iamyeh
 alfink
 bambu
 bkerler (viperbjk)
+Mark Jansen (learn-more)
+cq674350529
 
 Alpha testers (in no particular order, named by github id)
 ==========================================================

ChangeLog

@@ -1,8 +1,24 @@
 This file details the changelog of Qiling Framework.
 
+------------------------------------
+BREAK CHANGE
+- ql.multithread can be only set during Qiling.__init__ now.
+- ql.nprint and ql.dpring is depreciated. Please use logging directly instead.
+- ql.filename is renamed to ql.argv.
+- ql.output and ql.verbose now has slightly different meanings and can be adjusted runtime. See their docstring for details.
+- ql.filter now accepts a regular expression.
+
 ------------------------------------
 [Version 1.2.1]: December [SOMETHING], 2020
-- 
+
+- Added support for custom envs variables inside IDA plugin
+- Demigod: Fixed lkm mapping and added support for MIPS32EL
+- Demigod: Added support for Linux x86 32bit
+- Added support for binaries that return from their entrypoint (PE / ELF).
+- Configure Qiling with 'stop_on_stackpointer' or 'stop_on_exit_trap'.
+- Add basic Windows driver tests / example
+- UEFI refactor
+
 
 ------------------------------------
 [Version 1.2]: November 16th, 2020
@@ -20,7 +36,7 @@ This file details the changelog of Qiling Framework.
 
 
 ------------------------------------
-[Version 1.1.3]: September 30, 2020
+[Version 1.1.3]: September 30th, 2020
 
 - Added Doogie example and implement more interrupts
 - Added ollvm de-flattern support for IDA plugin

Dockerfile

@@ -19,7 +19,6 @@ COPY --from=builder /qiling /qiling
 WORKDIR /qiling
 
 RUN apt-get update \
-  && apt-get install -y libmagic-dev \ 
   && rm -rf /var/lib/apt/lists/* \
   && pip3 install wheels/*.whl \
   && rm -rf wheels

examples/doogie_8086_crack.py

@@ -3,7 +3,7 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 # Built on top of Unicorn emulator (www.unicorn-engine.org)
 
-import sys, curses, math, struct, string, time
+import sys, curses, math, struct, string, time, logging
 sys.path.append("..")
 from qiling import *
 from qiling.const import *
@@ -129,11 +129,11 @@ def show_once(ql: Qiling, key):
 # In this stage, we show every key.
 def third_stage(keys):
     # To setup terminal again, we have to restart the whole program.
-    ql = Qiling(["rootfs/8086/doogie/doogie.bin"], 
+    ql = Qiling(["rootfs/8086/doogie/doogie.DOS_MBR"], 
                  "rootfs/8086",
                  console=False,
                  log_dir=".")
-    ql.add_fs_mapper(0x80, QlDisk("rootfs/8086/doogie/doogie.bin", 0x80))
+    ql.add_fs_mapper(0x80, QlDisk("rootfs/8086/doogie/doogie.DOS_MBR", 0x80))
     ql.set_api((0x1a, 4), set_required_datetime, QL_INTERCEPT.EXIT)
     hk = ql.hook_code(stop, begin=0x8018, end=0x8018)
     ql.run()
@@ -172,7 +172,7 @@ def read_until_zero(ql: Qiling, addr):
     return buf
 
 def set_required_datetime(ql: Qiling):
-    ql.nprint("Setting Feburary 06, 1990")
+    logging.info("Setting Feburary 06, 1990")
     ql.reg.ch = BIN2BCD(19)
     ql.reg.cl = BIN2BCD(1990%100)
     ql.reg.dh = BIN2BCD(2)
@@ -183,11 +183,11 @@ def stop(ql, addr, data):
 
 # In this stage, we get the encrypted data which xored with the specific date.
 def first_stage():
-    ql = Qiling(["rootfs/8086/doogie/doogie.bin"], 
+    ql = Qiling(["rootfs/8086/doogie/doogie.DOS_MBR"], 
                  "rootfs/8086",
                  console=False,
                  log_dir=".")
-    ql.add_fs_mapper(0x80, QlDisk("rootfs/8086/doogie/doogie.bin", 0x80))
+    ql.add_fs_mapper(0x80, QlDisk("rootfs/8086/doogie/doogie.DOS_MBR", 0x80))
     # Doogie suggests that the datetime should be 1990-02-06.
     ql.set_api((0x1a, 4), set_required_datetime, QL_INTERCEPT.EXIT)
     # A workaround to stop the program.

examples/hello_arm_linux_custom_syscall.py

@@ -14,16 +14,16 @@ def my_syscall_write(ql, write_fd, write_buf, write_count, *args, **kw):
 
     try:
         buf = ql.mem.read(write_buf, write_count)
-        ql.nprint("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn))
+        logging.info("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn))
         ql.os.fd[write_fd].write(buf)
         regreturn = write_count
     except:
         regreturn = -1
-        ql.nprint("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn))
+        logging.info("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn))
         if ql.output in (QL_OUTPUT.DEBUG, QL_OUTPUT.DUMP):
             raise
 
-    ql.os.definesyscall_return(regreturn)
+    return regreturn
 
 
 if __name__ == "__main__":

examples/hello_arm_linux_debug.py

@@ -12,6 +12,7 @@ def run_sandbox(path, rootfs, output):
     ql.multithread = False
     ql.debugger = "qdb:rr" # switch on record and replay with rr
     # ql.debugger = "qdb:" # enable qdb without options
+    # ql.debugger = "qdb:0x1030c" # enable qdb and setup breakpoin at 0x1030c
     ql.run()