Merge branch 'dev' into qnx

Author: madprogrammer

examples/hello_x8664_linux_part_debug.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# 
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import sys
+sys.path.append("..")
+
+from qiling import Qiling
+from qiling.const import QL_VERBOSE
+
+def dump(ql, *args, **kw):
+    ql.save(reg=False, cpu_context=True, snapshot="/tmp/snapshot.bin")
+    ql.emu_stop()
+
+if __name__ == "__main__":
+    ql = Qiling(["rootfs/x8664_linux/bin/sleep_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DEFAULT)
+    # load base address from profile file
+    X64BASE = int(ql.profile.get("OS64", "load_address"), 16)
+    # take a snapshot
+    ql.hook_address(dump, X64BASE + 0x1094)
+    ql.run()
+
+    ql = Qiling(["rootfs/x8664_linux/bin/sleep_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG)
+    X64BASE = int(ql.profile.get("OS64", "load_address"), 16)
+    ql.restore(snapshot="/tmp/snapshot.bin")
+    # enable gdbserver to listen at localhost address, port 9999
+    ql.debugger = "gdb:0.0.0.0:9999"
+    begin_point = X64BASE + 0x109e
+    end_point = X64BASE + 0x10bc
+    ql.run(begin = begin_point, end = end_point)
+
+'''
+Partial Execution: https://docs.qiling.io/en/latest/snapshot/
+
+This example shows how to partially debug an elf file. First let the program run, hook at the main address and take a snapshot. Then resume the snapshot to construct a reasonable call_state (registers, memory mapping, dynamic library loading, etc) for our target piece of code, and directly assign the pc pointer to the beginning of the part you want to simulate.
+
+Run it with:
+    $ python3 hello_x8664_linux_part_debug.py 
+
+Then in a new terminal start gdb remote debug:
+    $ gdb -q
+    (gdb) target remote localhost:9999
+        Remote debugging using localhost:9999
+        Reading /home/qiling/examples/rootfs/x8664_linux/bin/sleep_hello from remote target...
+        warning: File transfers from remote targets can be slow. Use "set sysroot" to access files locally instead.
+        Reading /home/qiling/examples/rootfs/x8664_linux/bin/sleep_hello from remote target...
+        Reading symbols from target:/home/qiling/examples/rootfs/x8664_linux/bin/sleep_hello...(no debugging symbols found)...done.
+        warning: unable to open /proc file '/proc/42000/task/42000/maps'
+        Reading /lib64/ld-linux-x86-64.so.2 from remote target...
+        Reading /lib64/ld-linux-x86-64.so.2 from remote target...
+        Reading symbols from target:/lib64/ld-linux-x86-64.so.2...Reading /lib64/ld-2.27.so from remote target...
+        Reading /lib64/.debug/ld-2.27.so from remote target...
+        (no debugging symbols found)...done.
+        0x000055555555509e in ?? ()
+    (gdb) x/8i $pc
+        => 0x55555555509e:      lea    0xf83(%rip),%rdi        # 0x555555556028
+        0x5555555550a5:      callq  0x555555555060
+        0x5555555550aa:      lea    0xf53(%rip),%rdi        # 0x555555556004
+        0x5555555550b1:      callq  0x555555555060
+        0x5555555550b6:      xor    %eax,%eax
+        0x5555555550b8:      add    $0x8,%rsp
+        0x5555555550bc:      retq   
+        0x5555555550bd:      nopl   (%rax)
+
+The source code of sleep_hello can be found at qiling/examples/src/linux/sleep_hello.c. As the above gdb output shows, we skipped the sleep function to directly debug the code afterwards.
+'''

examples/rootfs

@@ -0,0 +1,23 @@
+#include "stdio.h"
+#include <sys/types.h>
+#include <dirent.h>
+void main()
+{
+    //int aa = O_RDONLY|O_NDELAY|O_DIRECTORY|O_LARGEFILE|O_CLOEXEC;
+    DIR *dirptr = NULL;
+    struct dirent *ptr;
+    dirptr = opendir("/");
+
+    while ((ptr = readdir(dirptr)) != NULL)
+    {
+        printf("%s\n", ptr->d_name);
+    }
+
+    if (dirptr == NULL)
+    {
+        printf("Open dir error !\n");
+    }
+    else
+        printf("Open Success!\n");
+    close(dirptr);
+}

examples/src/linux/getdents.c

@@ -7,100 +7,87 @@
 This module is intended for general purpose functions that are only used in qiling.arch
 """
 
-from unicorn import UcError, UC_ERR_READ_UNMAPPED, UC_ERR_FETCH_UNMAPPED
-from keystone import *
-from capstone import *
+from capstone import Cs, CS_ARCH_ARM, CS_ARCH_ARM64, CS_ARCH_X86, CS_ARCH_MIPS, CS_MODE_16, CS_MODE_32, CS_MODE_64, CS_MODE_ARM, CS_MODE_THUMB, CS_MODE_MIPS32 ,CS_MODE_BIG_ENDIAN, CS_MODE_LITTLE_ENDIAN
+from keystone import Ks, KS_ARCH_ARM, KS_ARCH_ARM64, KS_ARCH_X86, KS_ARCH_MIPS, KS_MODE_16, KS_MODE_32, KS_MODE_64, KS_MODE_ARM, KS_MODE_THUMB, KS_MODE_MIPS32 ,KS_MODE_BIG_ENDIAN, KS_MODE_LITTLE_ENDIAN
 
-from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS, QL_DEBUGGER, QL_ARCH_32BIT, QL_ARCH_64BIT, QL_ARCH_16BIT
-from qiling.exception import *
+from qiling.const import QL_ARCH, QL_ENDIAN
+from qiling.exception import QlErrorArch
 
+__cs_endian = {
+    QL_ENDIAN.EL : CS_MODE_LITTLE_ENDIAN,
+    QL_ENDIAN.EB : CS_MODE_BIG_ENDIAN
+}
 
-def ql_create_disassembler(archtype, archendian, reg_cpsr=None):
-    if archtype == QL_ARCH.ARM:  # QL_ARM
-        mode = CS_MODE_ARM
-        if archendian == QL_ENDIAN.EB:
-            # TODO: Test for big endian.
-            reg_cpsr_v = 0b100000
-            # reg_cpsr_v = 0b000000
-        else:
-            reg_cpsr_v = 0b100000
+__ks_endian = {
+    QL_ENDIAN.EL : KS_MODE_LITTLE_ENDIAN,
+    QL_ENDIAN.EB : KS_MODE_BIG_ENDIAN
+}
 
-        if reg_cpsr & reg_cpsr_v != 0:
-            mode = CS_MODE_THUMB
+__reg_cpsr_v = {
+    QL_ENDIAN.EL : 0b100000,
+    QL_ENDIAN.EB : 0b100000   # FIXME: should be: 0b000000
+}
 
-        if archendian == QL_ENDIAN.EB:
-            md = Cs(CS_ARCH_ARM, mode)
-            # md = Cs(CS_ARCH_ARM, mode + CS_MODE_BIG_ENDIAN)
-        else:
-            md = Cs(CS_ARCH_ARM, mode)
-
-    elif archtype == QL_ARCH.ARM_THUMB:
-        md = Cs(CS_ARCH_ARM, CS_MODE_THUMB)
-
-    elif archtype == QL_ARCH.X86:  # QL_X86
+def ql_create_disassembler(archtype: QL_ARCH, archendian: QL_ENDIAN, reg_cpsr=None) -> Cs:
+    if archtype == QL_ARCH.X86:
         md = Cs(CS_ARCH_X86, CS_MODE_32)
 
-    elif archtype == QL_ARCH.X8664:  # QL_X86_64
+    elif archtype == QL_ARCH.X8664:
         md = Cs(CS_ARCH_X86, CS_MODE_64)
 
-    elif archtype == QL_ARCH.ARM64:  # QL_ARM64
+    elif archtype == QL_ARCH.ARM:
+        mode = CS_MODE_THUMB if reg_cpsr & __reg_cpsr_v[archendian] else CS_MODE_ARM
+
+        md = Cs(CS_ARCH_ARM, mode) # FIXME: should be: mode + __cs_endian[archendian]
+
+    elif archtype == QL_ARCH.ARM_THUMB:
+        md = Cs(CS_ARCH_ARM, CS_MODE_THUMB)
+
+    elif archtype == QL_ARCH.ARM64:
         md = Cs(CS_ARCH_ARM64, CS_MODE_ARM)
 
-    elif archtype == QL_ARCH.A8086:  # QL_A8086
+    elif archtype == QL_ARCH.MIPS:
+        md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS32 + __cs_endian[archendian])
+
+    elif archtype == QL_ARCH.A8086:
         md = Cs(CS_ARCH_X86, CS_MODE_16)
 
-    elif archtype == QL_ARCH.MIPS:  # QL_MIPS32
-        if archendian == QL_ENDIAN.EB:
-            md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS32 + CS_MODE_BIG_ENDIAN)
-        else:
-            md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS32 + CS_MODE_LITTLE_ENDIAN)
+    elif archtype == QL_ARCH.EVM:
+        raise NotImplementedError('evm')
 
     else:
-        raise QlErrorArch("Unknown arch defined in utils.py (debug output mode)")
+        raise QlErrorArch(f'{archtype:d}')
 
     return md
 
-def ql_create_assembler(archtype, archendian, reg_cpsr=None):
-    if archtype == QL_ARCH.ARM:  # QL_ARM
-        mode = KS_MODE_ARM
-        if archendian == QL_ENDIAN.EB:
-            # TODO: Test for big endian.
-            reg_cpsr_v = 0b100000
-            # reg_cpsr_v = 0b000000
-        else:
-            reg_cpsr_v = 0b100000
-
-        if reg_cpsr & reg_cpsr_v != 0:
-            mode = KS_MODE_THUMB
-
-        if archendian == QL_ENDIAN.EB:
-            ks = Ks(KS_ARCH_ARM, mode)
-            # md = Cs(CS_ARCH_ARM, mode + CS_MODE_BIG_ENDIAN)
-        else:
-            ks = Ks(KS_ARCH_ARM, mode)
-
-    elif archtype == QL_ARCH.ARM_THUMB:
-        ks = Ks(KS_ARCH_ARM, KS_MODE_THUMB)
-
-    elif archtype == QL_ARCH.X86:  # QL_X86
+def ql_create_assembler(archtype: QL_ARCH, archendian: QL_ENDIAN, reg_cpsr=None) -> Ks:
+    if archtype == QL_ARCH.X86:
         ks = Ks(KS_ARCH_X86, KS_MODE_32)
 
-    elif archtype == QL_ARCH.X8664:  # QL_X86_64
+    elif archtype == QL_ARCH.X8664:
         ks = Ks(KS_ARCH_X86, KS_MODE_64)
 
-    elif archtype == QL_ARCH.ARM64:  # QL_ARM64
+    elif archtype == QL_ARCH.ARM:
+        mode = KS_MODE_THUMB if reg_cpsr & __reg_cpsr_v[archendian] else KS_MODE_ARM
+
+        ks = Ks(KS_ARCH_ARM, mode) # FIXME: should be: mode + __ks_endian[archendian]
+
+    elif archtype == QL_ARCH.ARM_THUMB:
+        ks = Ks(KS_ARCH_ARM, KS_MODE_THUMB)
+
+    elif archtype == QL_ARCH.ARM64:
         ks = Ks(KS_ARCH_ARM64, KS_MODE_LITTLE_ENDIAN)
 
-    elif archtype == QL_ARCH.A8086:  # QL_A8086
+    elif archtype == QL_ARCH.MIPS:
+        ks = Ks(KS_ARCH_MIPS, KS_MODE_MIPS32 + __ks_endian[archendian])
+
+    elif archtype == QL_ARCH.A8086:
         ks = Ks(KS_ARCH_X86, KS_MODE_16)
 
-    elif archtype == QL_ARCH.MIPS:  # QL_MIPS32
-        if archendian == QL_ENDIAN.EB:
-            ks = Ks(KS_ARCH_MIPS, KS_MODE_MIPS32 + KS_MODE_BIG_ENDIAN)
-        else:
-            ks = Ks(KS_ARCH_MIPS, KS_MODE_MIPS32 + KS_MODE_LITTLE_ENDIAN)
+    elif archtype == QL_ARCH.EVM:
+        raise NotImplementedError('evm')
 
     else:
-        raise QlErrorArch("Unknown arch defined in utils.py (debug output mode)")
+        raise QlErrorArch(f'{archtype:d}')
 
     return ks

qiling/arch/utils.py

@@ -3,13 +3,13 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
-from enum import IntEnum
+from enum import EnumMeta, IntEnum
+from typing import Mapping
 
 class QL_ENDIAN(IntEnum):
     EL = 1
     EB = 2
 
-
 class QL_ARCH(IntEnum):
     X86 = 101
     X8664 = 102
@@ -42,7 +42,6 @@ class QL_DEBUGGER(IntEnum):
     IDAPRO = 2
     QDB = 3
 
-
 class QL_INTERCEPT(IntEnum):
     CALL = 1
     ENTER = 2
@@ -65,38 +64,21 @@ class QL_INTERCEPT(IntEnum):
 QL_HOOK_BLOCK = 0b0001
 QL_CALL_BLOCK = 0b0010
 
-debugger_map = {
-    "gdb" : QL_DEBUGGER.GDB,
-    "ida" : QL_DEBUGGER.IDAPRO,
-    "qdb" : QL_DEBUGGER.QDB
-}
+def __reverse_enum(e: EnumMeta) -> Mapping[str, int]:
+    '''Create a reverse mapping for an enum.
+    '''
 
-arch_map = {
-    "x86"       : QL_ARCH.X86,
-    "x8664"     : QL_ARCH.X8664,
-    "mips"      : QL_ARCH.MIPS,
-    "arm"       : QL_ARCH.ARM,
-    "arm_thumb" : QL_ARCH.ARM_THUMB,
-    "arm64"     : QL_ARCH.ARM64,
-    "a8086"     : QL_ARCH.A8086,
-    "evm"       : QL_ARCH.EVM,
-}
+    return dict((k.lower(), v.value) for k, v in e.__members__.items())
 
-os_map = {
-    "linux"     : QL_OS.LINUX,
-    "macos"     : QL_OS.MACOS,
-    "freebsd"   : QL_OS.FREEBSD,
-    "qnx"       : QL_OS.QNX,
-    "windows"   : QL_OS.WINDOWS,
-    "uefi"      : QL_OS.UEFI,
-    "dos"       : QL_OS.DOS,
-    "evm"       : QL_OS.EVM,
-}
+debugger_map = __reverse_enum(QL_DEBUGGER)
+arch_map     = __reverse_enum(QL_ARCH)
+os_map       = __reverse_enum(QL_OS)
+verbose_map  = __reverse_enum(QL_VERBOSE)
 
 loader_map = {
     QL_OS.LINUX   : "ELF",
     QL_OS.FREEBSD : "ELF",
-    QL_OS.QNX     : "QNX",
+    QL_OS.QNX     : "ELF",
     QL_OS.MACOS   : "MACHO",
     QL_OS.WINDOWS : "PE",
     QL_OS.UEFI    : "PE_UEFI",

qiling/const.py

@@ -16,7 +16,7 @@
     from .os.memory import QlMemoryManager
     from .loader.loader import QlLoader
 
-from .const import QL_ARCH_ENDIAN, QL_ENDIAN, QL_OS, QL_CUSTOM_ENGINE
+from .const import QL_ARCH_ENDIAN, QL_ENDIAN, QL_OS, QL_VERBOSE, QL_CUSTOM_ENGINE
 from .exception import QlErrorFileNotFound, QlErrorArch, QlErrorOsType, QlErrorOutput
 from .utils import *
 from .core_struct import QlCoreStructs
@@ -35,7 +35,7 @@ def __init__(
             ostype=None,
             archtype=None,
             bigendian=False,
-            verbose=1,
+            verbose=QL_VERBOSE.DEFAULT,
             profile=None,
             console=True,
             log_file=None,
@@ -563,7 +563,8 @@ def verbose(self):
     def verbose(self, v):
         self._verbose = v
         self.log.setLevel(ql_resolve_logger_level(self._verbose))
-        self.os.utils.setup_output()
+        if not self._custom_engine:
+            self.os.utils.setup_output()
 
     @property
     def patch_bin(self) -> list:

qiling/core.py

@@ -74,12 +74,20 @@ def _hook_insn_cb(self, uc, *args):
         ql, hook_type = args[-1]
 
         if hook_type in self._insn_hook.keys():
+            retval = None
+
             for h in self._insn_hook[hook_type]:
                 if h.bound_check(ql.reg.arch_pc):
                     ret = h.call(ql, *args[ : -1])
-                    if isinstance(ret, int) == True and ret & QL_HOOK_BLOCK  != 0:
+
+                    if type(ret) is tuple:
+                        ret, retval = ret
+
+                    if type(ret) is int and ret & QL_HOOK_BLOCK:
                         break
 
+            # use the last return value received
+            return retval
 
     def _callback_type4(self, uc, addr, size, pack_data):
         ql, user_data, callback = pack_data

qiling/core_hooks.py

@@ -65,6 +65,11 @@ def __init__(self, ql, ip, port):
         else:
             self.entry_point = self.ql.os.entry_point
 
+        # Only part of the binary file will be debugged.
+        if self.ql.entry_point is not None and self.ql.exit_point is not None:
+            self.entry_point = self.ql.entry_point
+            exit_point = self.ql.exit_point
+
         self.gdb.initialize(self.ql, self.entry_point, exit_point=exit_point, mappings=[(hex(load_address))])
 
         #Setup register tables, order of tables is important

qiling/debugger/gdb/gdb.py

@@ -149,7 +149,7 @@ def context_reg(ql, saved_states=None, *args, **kwargs):
 
         for idx in range(8):
             _addr = ql.reg.arch_sp + idx * 4
-            _val = ql.mem.read(_addr, ql.archbit // 8)
+            _val = ql.mem.read(_addr, ql.pointersize)
             print(f"$sp+0x{idx*4:02x}|[0x{_addr:08x}]=> 0x{ql.unpack(_val):08x}", end="")
 
             try: # try to deference wether its a pointer