diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ba58a14b..c8c348b4ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,12 +59,20 @@ jobs: cd build ../configure --tooldir=$TOOLDIR ci/toolchain_install.sh --all + ci/sst_install.sh - name: Setup Third Party if: steps.cache-thirdparty.outputs.cache-hit != 'true' run: | make -C third_party > /dev/null + - name: Export SST paths + run: | + echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH + echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH + echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV + echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV + build: needs: setup strategy: @@ -99,6 +107,13 @@ jobs: restore-keys: | ${{ matrix.os }}-thirdparty- + - name: Export SST paths + run: | + echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH + echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH + echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV + echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV + - name: Run Build run: | TOOLDIR=$PWD/tools @@ -157,6 +172,13 @@ jobs: name: build-${{ matrix.os }}-${{ matrix.xlen }} path: build${{ matrix.xlen }} + - name: Export SST paths + run: | + echo "$PWD/tools/sst-install/sst-core/bin" >> $GITHUB_PATH + echo "$PWD/tools/sst-install/sst-elements/bin" >> $GITHUB_PATH + echo "SST_CORE_HOME=$PWD/tools/sst-install/sst-core" >> $GITHUB_ENV + echo "SST_ELEMENTS_HOME=$PWD/tools/sst-install/sst-elements" >> $GITHUB_ENV + - name: Run tests run: | cd build${{ matrix.xlen }} @@ -167,6 +189,7 @@ jobs: ./ci/regression.sh --isa ./ci/regression.sh --kernel ./ci/regression.sh --regression + ./ci/regression.sh --sst_tests else ./ci/regression.sh --${{ matrix.name }} fi diff --git a/ci/regression.sh.in b/ci/regression.sh.in index 90a46b83d1..00366435ed 100755 --- a/ci/regression.sh.in +++ b/ci/regression.sh.in @@ -78,6 +78,23 @@ kernel() echo "kernel tests done!" } +sst_tests() +{ + echo "begin sst tests..." + + make -C sim/simx USE_SST=1 + make -C tests/kernel + + cp sim/simx/libvortex.so $SST_ELEMENTS_HOME/lib/sst-elements-library/ # alternatively - $ sst --add-lib-path `pwd` myConfig.py + + sst ci/sst_test_vortex_hello.py + sst ci/sst_test_vortex_fibonacci.py + sst ci/sst_test_vortex_vecadd.py + sst ci/sst_test_vortex_conform.py + + echo "sst tests done!" +} + regression() { echo "begin regression tests..." @@ -505,7 +522,7 @@ cupbop() { show_usage() { echo "Vortex Regression Test" - echo "Usage: $0 [--clean] [--unittest] [--isa] [--kernel] [--regression] [--opencl] [--cache] [--config1] [--config2] [--debug] [--scope] [--stress] [--synthesis] [--vector] [--tensor] [--cupbop] [--all] [--h|--help]" + echo "Usage: $0 [--clean] [--unittest] [--isa] [--kernel] [--regression] [--opencl] [--cache] [--config1] [--config2] [--debug] [--scope] [--stress] [--synthesis] [--vector] [--tensor] [--cupbop] [--sst_tests] [--all] [--h|--help]" } declare -a tests=() @@ -564,6 +581,9 @@ while [ "$1" != "" ]; do --cupbop ) tests+=("cupbop") ;; + --sst_tests ) + tests+=("sst_tests") + ;; --all ) tests=() tests+=("unittest") diff --git a/ci/sst_install.sh.in b/ci/sst_install.sh.in new file mode 100755 index 0000000000..8c001a94bd --- /dev/null +++ b/ci/sst_install.sh.in @@ -0,0 +1,97 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# exit when any command fails +set -e + +OPENMPI_416=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz # version dependency +SST_CORE_1510=https://github.com/sstsimulator/sst-core/releases/download/v15.1.0_Final/sstcore-15.1.0.tar.gz +SST_ELEMENTS_1510=https://github.com/sstsimulator/sst-elements/releases/download/v15.1.0_Final/sstelements-15.1.0.tar.gz +TOOLDIR=${TOOLDIR:=@TOOLDIR@} + +DEBIAN_FRONTEND=noninteractive sudo apt install openmpi-bin openmpi-common libtool libtool-bin autoconf python3 python3-dev automake build-essential git +wget $OPENMPI_416 +tar -xvzf openmpi-4.1.6.tar.gz +mkdir -p $TOOLDIR && rm -rf $TOOLDIR/openmpi-4.1.6 && mv openmpi-4.1.6 $TOOLDIR +rm -rf openmpi-4.1.6.tar.gz + +mkdir -p $TOOLDIR/openmpi_install +cd $TOOLDIR/openmpi-4.1.6 + +export MPIHOME=$TOOLDIR/openmpi_install +./configure --prefix=$MPIHOME +make all install + +export PATH=$MPIHOME/bin:$PATH +export MPICC=mpicc +export MPICXX=mpicxx + +echo 'export PATH='"$MPIHOME"'/bin:$PATH' >> ~/.bashrc +echo 'export MPICC=mpicc' >> ~/.bashrc +echo 'export MPICXX=mpicxx' >> ~/.bashrc + +rm -r $TOOLDIR/openmpi-4.1.6 + +cd $TOOLDIR +wget $SST_CORE_1510 +tar -xvzf sstcore-15.1.0.tar.gz +rm sstcore-15.1.0.tar.gz +cd sst-core + +mkdir -p $TOOLDIR/sst-install/sst-core +export SST_CORE_HOME=$TOOLDIR/sst-install/sst-core +export SST_CORE_ROOT=$TOOLDIR/sst-core +echo 'export SST_CORE_HOME='"$SST_CORE_HOME" >> ~/.bashrc +#echo 'export SST_CORE_ROOT='"$SST_CORE_ROOT" >> ~/.bashrc + +autoreconf -fi +./configure --prefix=$SST_CORE_HOME +make -j$(nproc) all +make install + +rm -r $SST_CORE_ROOT + +export PATH=$SST_CORE_HOME/bin:$PATH +echo 'export PATH='"$SST_CORE_HOME"'/bin:$PATH' >> ~/.bashrc + +cd $TOOLDIR +wget $SST_ELEMENTS_1510 +tar -xvzf sstelements-15.1.0.tar.gz +rm sstelements-15.1.0.tar.gz +cd sst-elements + +mkdir -p $TOOLDIR/sst-install/sst-elements +export SST_ELEMENTS_HOME=$TOOLDIR/sst-install/sst-elements +export SST_ELEMENTS_ROOT=$TOOLDIR/sst-elements +echo 'export SST_ELEMENTS_HOME='"$SST_ELEMENTS_HOME" >> ~/.bashrc +#echo 'export SST_ELEMENTS_ROOT='"$SST_ELEMENTS_ROOT" >> ~/.bashrc + +./configure --prefix=$SST_ELEMENTS_HOME --with-sst-core=$SST_CORE_HOME +make -j2 all +make install + +rm -r $SST_ELEMENTS_ROOT + +export PATH=$SST_ELEMENTS_HOME/bin:$PATH +echo 'export PATH='"$SST_ELEMENTS_HOME"'/bin:$PATH' >> ~/.bashrc + +if [ -n "$GITHUB_PATH" ]; then + echo "$SST_CORE_HOME/bin" >> "$GITHUB_PATH" + echo "$SST_ELEMENTS_HOME/bin" >> "$GITHUB_PATH" +fi + + + + diff --git a/ci/sst_test_vortex_conform.py b/ci/sst_test_vortex_conform.py new file mode 100644 index 0000000000..2f2a86535f --- /dev/null +++ b/ci/sst_test_vortex_conform.py @@ -0,0 +1,7 @@ +import sst + +gpu = sst.Component("gpu0", "vortex.VortexGPGPU") +gpu.addParams({ + "clock": "1GHz", + "program": "tests/kernel/conform/conform.bin" +}) diff --git a/ci/sst_test_vortex_fibonacci.py b/ci/sst_test_vortex_fibonacci.py new file mode 100644 index 0000000000..a066765c9e --- /dev/null +++ b/ci/sst_test_vortex_fibonacci.py @@ -0,0 +1,7 @@ +import sst + +gpu = sst.Component("gpu0", "vortex.VortexGPGPU") +gpu.addParams({ + "clock": "1GHz", + "program": "tests/kernel/fibonacci/fibonacci.bin" +}) diff --git a/ci/sst_test_vortex_hello.py b/ci/sst_test_vortex_hello.py new file mode 100644 index 0000000000..21c956639e --- /dev/null +++ b/ci/sst_test_vortex_hello.py @@ -0,0 +1,7 @@ +import sst + +gpu = sst.Component("gpu0", "vortex.VortexGPGPU") +gpu.addParams({ + "clock": "1GHz", + "program": "tests/kernel/hello/hello.bin" +}) diff --git a/ci/sst_test_vortex_vecadd.py b/ci/sst_test_vortex_vecadd.py new file mode 100644 index 0000000000..1a50958056 --- /dev/null +++ b/ci/sst_test_vortex_vecadd.py @@ -0,0 +1,7 @@ +import sst + +gpu = sst.Component("gpu0", "vortex.VortexGPGPU") +gpu.addParams({ + "clock": "1GHz", + "program": "tests/kernel/vecadd/vecadd.bin" +}) diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 1eca622701..6ed58b9aa0 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -1,6 +1,8 @@ include ../common.mk DESTDIR ?= $(CURDIR) +USE_SST ?= 0 +#SST_PKG ?= SST-14.1 # default SST package name OBJ_DIR = $(DESTDIR)/obj CONFIG_FILE = $(DESTDIR)/simx_config.stamp @@ -27,6 +29,14 @@ SRCS += $(SRC_DIR)/execute.cpp $(SRC_DIR)/func_unit.cpp SRCS += $(SRC_DIR)/cache_sim.cpp $(SRC_DIR)/mem_sim.cpp $(SRC_DIR)/local_mem.cpp $(SRC_DIR)/mem_coalescer.cpp SRCS += $(SRC_DIR)/dcrs.cpp $(SRC_DIR)/types.cpp +ifeq ($(USE_SST), 1) +VORTEX_SST_SRCS := $(SRC_DIR)/vortex_simulator.cpp +VORTEX_SST_SRCS += $(SRC_DIR)/VortexGPGPU.cpp +SST_CFLAGS := $(shell sst-config --ELEMENT_CXXFLAGS) +SST_LFLAGS := $(shell sst-config --ELEMENT_LDFLAGS) +CXXFLAGS += $(SST_CFLAGS) -DUSE_SST +endif + # Add V extension sources ifneq ($(findstring -DEXT_V_ENABLE, $(CONFIGS)),) SRCS += $(SRC_DIR)/voperands.cpp @@ -65,14 +75,25 @@ DEPS := $(OBJS:.o=.d) $(MAIN_OBJ:.o=.d) # generate .d files alongside .o files CXXFLAGS += -MMD -MP -MF $(@:.o=.d) +ifeq ($(USE_SST), 1) +VORTEX_SST_OBJS := $(patsubst $(SRC_DIR)/%.cpp,$(OBJ_DIR)/%.o,$(VORTEX_SST_SRCS)) +DEPS += $(VORTEX_SST_OBJS:.o=.d) +endif + + # optional: pipe through ccache if you have it CXX := $(if $(shell which ccache),ccache $(CXX),$(CXX)) PROJECT := simx +VORTEX_LIB := libvortex.so -.PHONY: all force clean clean-lib clean-exe clean-obj +.PHONY: all force clean clean-lib clean-exe clean-obj libvortex clean-libvortex +ifeq ($(USE_SST), 1) +all: $(DESTDIR)/$(PROJECT) $(DESTDIR)/$(VORTEX_LIB) +else all: $(DESTDIR)/$(PROJECT) +endif # build common object files $(OBJ_DIR)/common/%.o: $(SW_COMMON_DIR)/%.cpp $(CONFIG_FILE) @@ -84,6 +105,11 @@ $(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp $(CONFIG_FILE) @mkdir -p $(@D) $(CXX) $(CXXFLAGS) -c $< -o $@ +# build SST-specific source object files +$(VORTEX_SST_OBJS): $(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp $(CONFIG_FILE) + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(SST_CFLAGS) -c $< -o $@ + # build main object file $(MAIN_OBJ): $(SRC_DIR)/main.cpp $(CONFIG_FILE) @mkdir -p $(@D) @@ -97,6 +123,16 @@ $(DESTDIR)/$(PROJECT): $(OBJS) $(MAIN_OBJ) $(DESTDIR)/lib$(PROJECT).so: $(OBJS) $(CXX) $(CXXFLAGS) $^ -shared $(LDFLAGS) -o $@ +# Vortex SST simulator component shared library +libvortex: $(DESTDIR)/$(VORTEX_LIB) + +$(DESTDIR)/$(VORTEX_LIB): $(OBJS) $(VORTEX_SST_OBJS) + $(CXX) $(CXXFLAGS) $(SST_CFLAGS) \ + -I./sim/simx \ + $(OBJS) $(VORTEX_SST_SRCS) \ + -shared -o $@ \ + $(LDFLAGS) $(SST_LFLAGS) + # updates the timestamp when flags changed. $(CONFIG_FILE): force @mkdir -p $(@D) @@ -113,10 +149,13 @@ $(CONFIG_FILE): force clean-lib: rm -f $(DESTDIR)/lib$(PROJECT).so +clean-libvortex: + rm -f $(DESTDIR)/libvortex.so + clean-exe: rm -f $(DESTDIR)/$(PROJECT) clean-obj: rm -rf $(OBJ_DIR) -clean: clean-lib clean-exe clean-obj \ No newline at end of file +clean: clean-lib clean-exe clean-obj clean-libvortex \ No newline at end of file diff --git a/sim/simx/VortexGPGPU.cpp b/sim/simx/VortexGPGPU.cpp new file mode 100644 index 0000000000..eff4af33d8 --- /dev/null +++ b/sim/simx/VortexGPGPU.cpp @@ -0,0 +1,54 @@ +#include +#include "VortexGPGPU.h" +#include +#include +#include +#include + +using namespace SST; +using namespace SST::Vortex; + +VortexGPGPU::VortexGPGPU(ComponentId_t id, Params ¶ms) + : Component(id), + sim_(std::make_unique()) { + + std::cout << "VortexGPGPU Component: Initializing Vortex GPGPU simulator\n"; + + // Parameter: clock frequency (default 1GHz) + std::string clockfreq = params.find("clock", "1GHz"); + + // Parameter: program path + std::string kernel = params.find("program", ""); + + // Register our clock handler with SST + registerClock(clockfreq, + new SST::Clock::Handler(this, &VortexGPGPU::clockTick)); + + // Load the kernel image + if (!sim_->init(kernel)) { + SST::Output out; + out.fatal(CALL_INFO, -1, "VortexSimulator init failed\n"); + } + else{ + std::cout << "VortexGPGPU Component: loaded kernel: " << kernel << std::endl; + } + + registerAsPrimaryComponent(); + primaryComponentDoNotEndSim(); +} + +VortexGPGPU::~VortexGPGPU() = default; + +void VortexGPGPU::setup() {} +void VortexGPGPU::finish() {} + +// Advance the GPU execution one cycle based on SST clock handler callback +bool VortexGPGPU::clockTick(SST::Cycle_t cycle) { + bool running = sim_->cycle(); + if (!running) { + primaryComponentOKToEndSim(); + std::cout << "VortexGPGPU Component: simulation finished\n"; + return true; + } + return false; +} diff --git a/sim/simx/VortexGPGPU.h b/sim/simx/VortexGPGPU.h new file mode 100644 index 0000000000..1421bd0c10 --- /dev/null +++ b/sim/simx/VortexGPGPU.h @@ -0,0 +1,48 @@ +// VortexGPGPU.h +#pragma once +#include +#include +#include +#include "vortex_simulator.h" // wrapper around SimX +#include + +namespace SST { +namespace Vortex { + +class VortexGPGPU : public SST::Component { +public: + VortexGPGPU(SST::ComponentId_t id, SST::Params& params); + ~VortexGPGPU() override; + + void setup() override; + void finish() override; + + // Register with SST + SST_ELI_REGISTER_COMPONENT( + VortexGPGPU, + "vortex", // Element library name + "VortexGPGPU", // Component name + SST_ELI_ELEMENT_VERSION(1,0,0), + "Vortex GPGPU Simulator", + COMPONENT_CATEGORY_PROCESSOR + ) + + SST_ELI_DOCUMENT_PARAMS( + {"clock", "Clock frequency", "1GHz"}, + {"program", "Path to the kernel or ELF to load", ""}, + ) + + // for future usage with SST memory + SST_ELI_DOCUMENT_SUBCOMPONENT_SLOTS( + {"memIface", "StandardMem interface to memory hierarchy", "SST::Interfaces::StandardMem"} + ) + +private: + + bool clockTick(SST::Cycle_t cycle); + + std::unique_ptr sim_; +}; + +} // namespace Vortex +} // namespace SST diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index d3b730dc8b..114b275d03 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -159,6 +159,27 @@ ProcessorImpl::PerfStats ProcessorImpl::perf_stats() const { return perf; } +// Advance the simulation by one cycle for SST - code adapted from run() method +bool ProcessorImpl::cycle() { + if (!is_cycle_initialized_) { + std::cout << "ProcessorImpl: Initializing cycle()\n"; + SimPlatform::instance().reset(); + this->reset(); + is_cycle_initialized_ = true; + } + + SimPlatform::instance().tick(); + bool anyRunning = false; + for (auto cluster : clusters_) { + if (cluster->running()) { + anyRunning = true; + break; + } + } + perf_mem_latency_ += perf_mem_pending_reads_; + return anyRunning; +} + /////////////////////////////////////////////////////////////////////////////// Processor::Processor(const Arch& arch) @@ -196,6 +217,15 @@ void Processor::dcr_write(uint32_t addr, uint32_t value) { return impl_->dcr_write(addr, value); } +// advance the simulation by one cycle for SST +bool Processor::cycle() { + try { + return impl_->cycle(); + } catch (...) { + return false; + } +} + #ifdef VM_ENABLE int16_t Processor::set_satp_by_addr(uint64_t base_addr) { uint16_t asid = 0; diff --git a/sim/simx/processor.h b/sim/simx/processor.h index 741b04f57d..4bb3f23fc6 100644 --- a/sim/simx/processor.h +++ b/sim/simx/processor.h @@ -35,6 +35,8 @@ class Processor { int run(); + bool cycle(); + void dcr_write(uint32_t addr, uint32_t value); #ifdef VM_ENABLE bool is_satp_unset(); diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index 952b28222f..031ce14afa 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -38,6 +38,8 @@ class ProcessorImpl { int run(); + bool cycle(); + void dcr_write(uint32_t addr, uint32_t value); #ifdef VM_ENABLE @@ -50,6 +52,7 @@ class ProcessorImpl { void reset(); + bool is_cycle_initialized_ = false; const Arch& arch_; std::vector> clusters_; DCRS dcrs_; diff --git a/sim/simx/vortex_simulator.cpp b/sim/simx/vortex_simulator.cpp new file mode 100644 index 0000000000..f339f434d8 --- /dev/null +++ b/sim/simx/vortex_simulator.cpp @@ -0,0 +1,65 @@ +#include "vortex_simulator.h" +#include +#include +#include +#include +#include +#include +#include +#include "simobject.h" +#include "dcrs.h" +#include +#include +#include "util.h" + +namespace vortex { + +VortexSimulator::VortexSimulator() +: arch_(NUM_THREADS, NUM_WARPS, NUM_CORES) +, ram_(0, MEM_PAGE_SIZE) +, proc_(std::make_unique(arch_)) +, halted_(true) {} + +bool VortexSimulator::init(const std::string& kernelPath) { + proc_->attach_ram(&ram_); + + // setup base DCRs + const uint64_t startup_addr(STARTUP_ADDR); + proc_->dcr_write(VX_DCR_BASE_STARTUP_ADDR0, startup_addr & 0xffffffff); + #if (XLEN == 64) + proc_->dcr_write(VX_DCR_BASE_STARTUP_ADDR1, startup_addr >> 32); + #endif + proc_->dcr_write(VX_DCR_BASE_MPM_CLASS, 0); + + // load program/kernel + { + std::string program_ext(fileExtension(kernelPath.c_str())); + if (program_ext == "bin") { + std::cout << "vortex_simulator: Loading binary image: " << kernelPath << " with startup address: 0x" << std::hex << startup_addr << std::dec << std::endl; + ram_.loadBinImage(kernelPath.c_str(), startup_addr); + } else if (program_ext == "hex") { + std::cout << "vortex_simulator: Loading hex image: " << kernelPath << std::endl; + ram_.loadHexImage(kernelPath.c_str()); + } else { + std::cerr << "Error: only *.bin or *.hex images supported." << std::endl; + return -1; + } + } + + halted_ = false; + return true; +} + +bool VortexSimulator::cycle() { +if (halted_) return false; +// Advance one cycle through the processor interface +bool running = proc_->cycle(); +halted_ = !running; +return running; +} + +bool VortexSimulator::isHalted() const { + return halted_; +} + +} // namespace vortex diff --git a/sim/simx/vortex_simulator.h b/sim/simx/vortex_simulator.h new file mode 100644 index 0000000000..028b2cc795 --- /dev/null +++ b/sim/simx/vortex_simulator.h @@ -0,0 +1,47 @@ +// vortex_simulator.h +#pragma once + +#include "processor.h" // for Processor, RAM +#include "arch.h" // for Arch +#include "constants.h" +#include +#include +#include +#include + +namespace vortex { + +/** + * A wrapper class used by the SST integration to drive the Vortex GPU + * one cycle at a time. It encapsulates the architecture definition, + * memory subsystem, and processor instance. + */ +class VortexSimulator { +public: + VortexSimulator(); + + /** + * Initializes the simulator. If @p kernelPath is non-empty, the + * kernel image at the given path will be loaded into memory. + * Returns false if the image format is not supported. + */ + bool init(const std::string& kernelPath); + + /** + * Advances the simulation by one cycle. Returns false once the + * simulation has completed (i.e. all clusters are halted). + */ + bool cycle(); + + /** Returns true if the simulation has finished. */ + bool isHalted() const; + +private: + + Arch arch_; + RAM ram_; + std::unique_ptr proc_; + bool halted_; +}; + +} // namespace vortex