Replace setup.py with everything using pip now

Author: AndrewQuijano

.gitignore

@@ -147,6 +147,7 @@ trace-ust-all.h
 trace-ust-all.c
 
 .vscode/
+.idea/
 
 # Pypanda Examples
 pwc_log

Dockerfile

@@ -56,6 +56,7 @@ RUN git -C /panda submodule update --init dtc && \
     git -C /panda rev-parse HEAD > /usr/local/panda_commit_hash && \
     mkdir  /panda/build && cd /panda/build && \
     python3 -m pip install setuptools_scm && \
+    python3 -m pip install build && \
     python3 -m setuptools_scm -r .. --strip-dev 2>/dev/null >/tmp/savedversion && \
     /panda/configure \
         --target-list="${TARGET_LIST}" \
@@ -70,7 +71,8 @@ RUN PRETEND_VERSION=$(cat /tmp/savedversion) make -C /panda/build -j "$(nproc)"
 #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3
 FROM builder as developer
 RUN cd /panda/panda/python/core && \
-    PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py develop && \
+    python3 create_panda_datatypes.py && \
+    PRETEND_VERSION=$(cat /tmp/savedversion) pip install -e . && \
     ldconfig && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \
     cd /panda && \
@@ -90,11 +92,13 @@ RUN  make -C /panda/build install && \
 
 # Install pypanda
 RUN cd /panda/panda/python/core && \
-    PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py install
+    python3 create_panda_datatypes.py --install && \
+    PRETEND_VERSION=$(cat /tmp/savedversion) pip install .
 RUN python3 -m pip install --ignore-install pycparser && python3 -m pip install --force-reinstall --no-binary :all: cffi
 # Build a whl too
 RUN cd /panda/panda/python/core && \
-    PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py bdist_wheel
+    python3 create_panda_datatypes.py --install && \
+    PRETEND_VERSION=$(cat /tmp/savedversion) python3 -m build --wheel .
 
 # BUG: PANDA sometimes fails to generate all the necessary files for PyPANDA. This is a temporary fix to detect and fail when this occurs
 RUN ls -alt $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/

panda/python/core/MANIFEST.in

@@ -1,2 +1,3 @@
 graft pandare
-global-exclude *.py[cod] __pycache__ *.sw*
+prune pandare/include
+global-exclude *.py[cod] __pycache__ *.sw* .git*

panda/python/core/create_panda_datatypes.py

@@ -3,6 +3,7 @@
 import os
 import sys
 import shutil
+import argparse
 if sys.version_info[0] < 3:
     raise RuntimeError('Requires python3')
 
@@ -14,15 +15,16 @@
 # those include files.  See panda/include/panda/README.pypanda for
 # so proscriptions wrt those headers we use here. They need to be kept
 # fairly clean if we are to be able to make sense of them with this script
-# which isn't terriby clever.
+# which isn't terribly clever.
 
-# Also copy all of the generated plog_pb2.py's into pandare/plog_pb/
+# Also copy all the generated plog_pb2.py's into pandare/plog_pb/
 # XXX: WIP if we do this this file should be renamed
 root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."]) # panda-git/ root dir
 build_root = os.path.join(root_dir, "build")
 lib_dir = os.path.join("pandare", "data")
 
-for arch in ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64']:
+arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64', 'mips64el']
+for arch in arches:
     softmmu = arch+"-softmmu"
     plog = os.path.join(*[build_root, softmmu, "plog_pb2.py"])
     if os.path.isfile(plog):
@@ -187,17 +189,12 @@ def include_this(pdth, fn):
             pdth.write(line)
     pn += 1
 
-def compile(arch, bits, pypanda_headers, install, static_inc):
-    #from ..ffi_importer import ffi
+def compile(arch, bits, pypanda_headers, include_dir):
     from cffi import FFI
     ffi = FFI()
 
     ffi.set_source(f"panda_{arch}_{bits}", None)
-    if install:
-        import os
-        include_dir = os.path.abspath(os.path.join(*[os.path.dirname(__file__),  "pandare", "include"]))
-    else:
-        include_dir = static_inc
+
 
     def define_clean_header(ffi, fname):
         '''Convenience function to pull in headers from file in C'''
@@ -325,7 +322,7 @@ def expand_ppp_def(line):
     ffi.compile(verbose=True,debug=True,tmpdir='./pandare/autogen')
 
 
-def main(install=False,recompile=True):
+def main(recompile=True):
     '''
     Copy and reformat panda header files into the autogen directory
 
@@ -338,8 +335,6 @@ def main(install=False,recompile=True):
     # out pypanda bits to go in INCLUDE_DIR files
     plugin_dirs = os.listdir(PLUGINS_DIR)
 
-    INCLUDE_DIR_PYP_INSTALL = 'os.path.abspath(os.path.join(*[os.path.dirname(__file__), "..", "..", "pandare", "data", "pypanda", "include"]))'  # ... /python3.6/site-packages/panda/data/pypanda/include/
-
     # We want the various cpu_loop_... functions
     create_pypanda_header("%s/exec/exec-all.h" % INCLUDE_DIR_CORE)
 
@@ -631,7 +626,7 @@ def exception(self):
 
         for arch in arches:
             print("Compiling headers for:", arch)
-            compile(arch[0], arch[1], pypanda_headers, install, INCLUDE_DIR_PYP)
+            compile(arch[0], arch[1], pypanda_headers, INCLUDE_DIR_PYP)
 
         #ps = (
         #    Process(target=compile, args=(arch[0], arch[1], pypanda_headers, install, INCLUDE_DIR_PYP))
@@ -641,5 +636,68 @@ def exception(self):
         #[p.join() for p in ps]
 
 
+# Check for PANDA binaries in /usr/local/bin/ or in our build directory
+# panda_binaries = ['/usr/local/bin/panda-system-{arch}' for arch in arches]
+# Do we actually care at this point?
+def copy_objs():
+    '''
+    Run to copy objects into a (local and temporary) python module before installing to the system.
+    Shouldn't be run if you're just installing in develop mode
+    '''
+    print("Copying objects into pandare/data/pypanda/include")
+    if os.path.isdir(lib_dir):
+        shutil.rmtree(lib_dir)
+    os.mkdir(lib_dir)
+
+    # Copy pypanda's include directory (different than core panda's) into a datadir
+    pypanda_inc = os.path.join(*[root_dir, "panda", "python", "core", "pandare", "include"])
+    if not os.path.isdir(pypanda_inc):
+        raise RuntimeError(f"Could not find pypanda include directory at {pypanda_inc}")
+    pypanda_inc_dest = os.path.join(*["pandare", "data", "pypanda", "include"])
+    if os.path.isdir(pypanda_inc_dest):
+        shutil.rmtree(pypanda_inc_dest)
+    shutil.copytree(pypanda_inc, pypanda_inc_dest)
+
+    # For each arch, copy llvm-helpers
+    # XXX Should these be in standard panda deb?
+    # What actually uses these? Taint? Disabling for now
+    '''
+    # Check if we have llvm-support
+    with open(os.path.join(*[build_root, 'config-host.mak']), 'r') as cfg:
+        llvm_enabled = True if 'CONFIG_LLVM=y' in cfg.read() else False
+
+    for arch in arches:
+        libname = "libpanda-"+arch+".so"
+        softmmu = arch+"-softmmu"
+        path      = os.path.join(*[build_root, softmmu, libname])
+        llvm1     = os.path.join(*[build_root, softmmu, "llvm-helpers.bc1"])
+        llvm2     = os.path.join(*[build_root, softmmu, f"llvm-helpers-{arch}.bc"])
+
+        if os.path.isfile(path) is False:
+            print(("Missing file {} - did you run build.sh from panda/build directory?\n"
+                   "Skipping building pypanda for {}").format(path, arch))
+            continue
+
+        os.mkdir(os.path.join(lib_dir, softmmu))
+        if llvm_enabled:
+            shutil.copy(    llvm1,      os.path.join(lib_dir, softmmu))
+            shutil.copy(    llvm2,      os.path.join(lib_dir, softmmu))
+    '''
+
+
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser(prog='A python script to populate panda/autogen folder '
+                                          'and prep for PyPanda wheel installation')
+    parser.add_argument('--install', '-i', dest='install', action='store_true',
+                        help='If set, this means update pandare folder for installation')
+    parser.add_argument('--recompile', '-r', dest='recompile', action='store_true',
+                        help='If set, recompile the headers with cffi')
+    args = parser.parse_args()
+    """
+    Install as a local module (not to system) by
+        1) Creating datatype files for local-use
+        2) Running regular setup tools logic
+    """
     main()
+    if args.install:
+        copy_objs()

panda/python/core/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=64", "wheel", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = [
+    "pandare",
+    "pandare.*"
+]
+
+[tool.setuptools.package-data]
+pandare = [
+    "data/*-softmmu/llvm-helpers*.bc*",  # LLVM Helpers
+    "data/pypanda/include/*.h",  # Includes files
+    "qcows.json"  # Generic Images
+]
+
+# This is how to setup cmd replacement, we can figure this out later
+# [tool.setuptools.cmdclass]
+# install = "setup.custom_install"
+# develop = "setup.custom_develop"
+
+[project]
+name = "pandare"
+dynamic = ["version"]
+description = "Python Interface to PANDA"
+readme = "README.md"
+authors = [
+    { name = "Luke Craig", email = "luke.craig@mit.edu" },
+    { name = "Andrew Fasano" },
+    { name = "Tim Leek" }
+]
+requires-python = ">=3.6"
+dependencies = [
+    "cffi>=1.14.3",
+    "protobuf>=4.25.1",
+    "colorama"
+]
+
+[tool.setuptools_scm]
+root = "../../.."
+fallback_version = "0.0.0.1"
+version_scheme = "guess-next-dev"
+local_scheme = "no-local-version"