Merge a80cf13 into f244f72

Author: ananos

.github/workflows/e2e.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   prepare:
-    runs-on: [self-hosted]
+    runs-on: [cloud]
     outputs:
       vaccelrt_sha: ${{ steps.vars.outputs.vaccelrt_sha }}
       firecracker_sha: ${{ steps.vars.outputs.firecracker_sha }}
@@ -34,7 +34,7 @@ jobs:
 
   test:
     needs: prepare
-    runs-on: [self-hosted, gpu, "${{ matrix.arch }}" ]
+    runs-on: [self-hosted, cloud, "${{ matrix.arch }}" ]
     env:
       NBFC_S3_ACCESS: ${{ secrets.AWS_ACCESS_KEY }}
       NBFC_S3_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -43,7 +43,7 @@ jobs:
       FIRECRACKER_TYPE: ${{ needs.prepare.outputs.firecracker_build_type }}
       VIRTIO_SHA: ${{ needs.prepare.outputs.virtio_accel_sha }}
       JOB_TYPE: ${{ matrix.build_type }}
-      ARCH: ${{ matrix.arch }}
+      ARCHITECTURE: ${{ matrix.arch }}
 
     strategy:
       matrix:
@@ -52,12 +52,17 @@ jobs:
       fail-fast: false
 
     steps:
+    - name: Cleanup previous runs
+      run: |
+        sudo rm -rf ${{ github.workspace }}/*
+        sudo rm -rf ${{ github.workspace }}/.??*
+
     - uses: actions/checkout@v2
 
     - name: Install packages
       run: |
         sudo apt update && sudo apt install -y \
-          subversion coreutils rsync iproute2 libfdt-dev
+          subversion coreutils rsync iproute2 libfdt-dev qemu-system-x86-64 parted debootstrap bc udev libssl-dev libelf-dev
 
     - name: Setup vars
       id: vars
@@ -73,19 +78,90 @@ jobs:
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
         local-path: /github/workspace/artifacts/opt/
-        remote-path: nbfc-assets/github/vaccelrt/${{env.VACCELRT_SHA}}/${{env.ARCH}}/${{ env.JOB_TYPE }}/opt/
+        remote-path: nbfc-assets/github/vaccelrt/${{env.VACCELRT_SHA}}/${{env.ARCHITECTURE}}/${{ env.JOB_TYPE }}/opt/
       env:
         ACTION_UID: ${{ steps.vars.outputs.uid }}
         ACTION_GID: ${{ steps.vars.outputs.gid }}
 
+    - name: Download go & kata containers
+      run: |
+        if [[ $ARCHITECTURE == "x86_64" ]]
+             then
+                export ARCH=amd64
+             else
+                export ARCH=arm64
+             fi
+        #wget https://golang.org/dl/go1.15.11.linux-$ARCH.tar.gz
+        #sudo tar -zxf go1.15.11.linux-$ARCH.tar.gz -C /usr/local/
+        git submodule update --init kata-containers
+        git submodule update --init virtio-accel
+
+    - name: Checkout private tools
+      uses: actions/checkout@v2
+      with:
+        repository: nubificus/vaccel-go-runtime
+        ref: main
+        token: ${{ secrets.NBFC_BUILDER_TOKEN }} # `GitHub_PAT` is a secret that contains your PAT
+        path: kata-containers/src/runtime/vendor/github.com/nubificus/vaccel-go-runtime
+
+    - name: Build and install kata-containers
+      env:
+        GOROOT: /usr/local/go
+        GOPATH: /home/runner/go
+      run: |
+        export PATH=$GOROOT/bin:$PATH
+        export PATH=$GOPATH/bin:$PATH
+        if [[ $ARCHITECTURE == "x86_64" ]]
+             then
+                export ARCH=amd64
+                export ARCH_KERNEL=x86_64
+             else
+                export ARCH_KERNEL=arm64
+             fi
+        cd kata-containers
+        make -C src/runtime
+        sudo cp src/runtime/containerd-shim-kata-v2 ${{ github.workspace }}/artifacts/opt/bin/
+        sudo cp src/runtime/cli/config/configuration-fc.toml ${{ github.workspace }}/artifacts/opt/share/
+        cd tools/packaging/kernel
+        echo "CONFIG_MODULES=y" > configs/fragments/$ARCH_KERNEL/vaccel.conf
+        echo "CONFIG_MODULE_UNLOAD=y" >> configs/fragments/$ARCH_KERNEL/vaccel.conf
+        echo "CONFIG_MODULE_SIG=y" >> configs/fragments/$ARCH_KERNEL/vaccel.conf
+        VERSION=`./build-kernel.sh setup 2>&1 |grep Kernel\ version\: | awk '{print $4}'`
+        ./build-kernel.sh build
+        if [[ $ARCHITECTURE == "x86_64" ]]
+        then
+                sudo cp kata-linux*/vmlinux ${{ github.workspace }}/artifacts/opt/share/vmlinux-kata-fc
+        else
+                sudo cp kata-linux*/arch/arm64/boot/Image ${{ github.workspace }}/artifacts/opt/share/vmlinux-kata-fc
+        fi
+        cd ${{ github.workspace }}/virtio-accel
+        git checkout fix_kzfree
+        KDIR=${{ github.workspace }}/kata-containers/tools/packaging/kernel/kata-linux*/ make ZC=0 ARCH=$ARCH_KERNEL
+        sudo cp virtio_accel.ko ${{ github.workspace }}/artifacts/opt/share/virtio_accel-kata.ko
+        if [[ $ARCHITECTURE == "x86_64" ]]
+        then
+                cd ${{ github.workspace }}/kata-containers/tools/osbuilder/rootfs-builder/
+                export ROOTFS_DIR=$PWD/rootfs
+                rm -rf $ROOTFS_DIR
+                script -fec 'sudo su root -c ". /opt/cargo/env && GOPATH=$GOPATH GOROOT=$GOROOT PATH=$GOROOT/bin:$PATH RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/opt/cargo/bin:$PATH ./rootfs.sh -r $ROOTFS_DIR ubuntu"'
+                MODULES_DIR=${ROOTFS_DIR}/lib/modules/$VERSION
+                sudo mkdir -p ${MODULES_DIR}
+                sudo cp ${{ github.workspace }}/virtio-accel/virtio_accel.ko $MODULES_DIR
+                sudo touch ${MODULES_DIR}/modules.builtin
+                sudo touch ${MODULES_DIR}/modules.order
+                sudo chroot ${ROOTFS_DIR} /sbin/depmod $VERSION
+                cd ../image-builder
+                script -fec 'sudo bash -x ./image_builder.sh $ROOTFS_DIR'
+                sudo cp kata-containers.img ${{ github.workspace }}/artifacts/opt/share/
+        fi
 
     - name: Fetch Firecracker
       uses: cloudkernels/minio-download@master
       with:
         url: https://s3.nubificus.co.uk
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
-        remote-path: nbfc-assets/github/firecracker/${{ env.FIRECRACKER_SHA }}/${{env.ARCH}}/${{ steps.vars.outputs.firecracker_build_type }}/firecracker
+        remote-path: nbfc-assets/github/firecracker/${{ env.FIRECRACKER_SHA }}/${{env.ARCHITECTURE}}/${{ steps.vars.outputs.firecracker_build_type }}/firecracker
         local-path: /github/workspace/artifacts/opt/bin/
       env:
         ACTION_UID: ${{ steps.vars.outputs.uid }}
@@ -97,7 +173,7 @@ jobs:
         url: https://s3.nubificus.co.uk
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
-        remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCH}}/linux/vmlinux
+        remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCHITECTURE}}/linux/vmlinux
         local-path: /github/workspace/artifacts/opt/share/
       env:
         ACTION_UID: ${{ steps.vars.outputs.uid }}
@@ -109,7 +185,7 @@ jobs:
         url: https://s3.nubificus.co.uk
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
-        remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCH}}/linux/virtio_accel.ko
+        remote-path: nbfc-assets/github/virtio-accel/${{ env.VIRTIO_SHA }}/${{env.ARCHITECTURE}}/linux/virtio_accel.ko
         local-path: /github/workspace/artifacts/opt/share/
       env:
         ACTION_UID: ${{ steps.vars.outputs.uid }}
@@ -145,7 +221,7 @@ jobs:
         sudo mkdir -p /usr/local/share/imagenet-models
         sudo chmod a+rw /usr/local/share/imagenet-models
         ln -s ${{ github.workspace }}/artifacts/opt/share/networks /usr/local/share/imagenet-models/networks
-        sudo VACCEL_BACKENDS=${{ github.workspace }}/artifacts/opt/lib/libvaccel-jetson.so \
+        sudo VACCEL_BACKENDS=${{ github.workspace }}/artifacts/opt/lib/libvaccel-noop.so \
           LD_LIBRARY_PATH=${{ github.workspace }}/artifacts/opt/lib:/usr/local/lib:/usr/lib \
           VACCEL_DEBUG_LEVEL=4 \
           ./bin/firecracker \
@@ -192,32 +268,42 @@ jobs:
         cargo install --path .
 
     - name: Test vsock plugin
-      run: ./scripts/test_vsock.sh -i ${{ github.workspace }}/artifacts/opt/share/fc_test -p ${{ github.workspace }}/artifacts/opt/lib/libvaccel-jetson.so
+      run: ./scripts/test_vsock.sh -i ${{ github.workspace }}/artifacts/opt/share/fc_test -p ${{ github.workspace }}/artifacts/opt/lib/libvaccel-noop.so
 
     - name: Pack latest release
-      if: ${{ github.event_name == 'push' }}
+      if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       working-directory: ${{ github.workspace }}/artifacts/opt
       run: |
         cp ${{github.workspace}}/conf/{config_virtio_accel.json,config_vsock.json} share/
         cp /opt/cargo/bin/vaccelrt-agent bin/
         zip -r ${{github.workspace}}/vaccel_${{matrix.arch}}_${{matrix.build_type}}.zip bin/ include/ lib/ \
           share/config_virtio_accel.json share/config_vsock.json \
-          share/fc_test share/fc_test.pub \
           share/rootfs.img share/virtio_accel.ko share/vmlinux \
+          share/kata-containers.img share/virtio_accel-kata.ko share/vmlinux-kata-fc \
+          share/configuration-fc.toml \
           share/vaccel.pc
 
+    - name: Extract branch name
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${{ github.head_ref }})"
+      id: extract_branch
+
+
     - name: Upload latest master release to s3
-      if: ${{ github.event_name == 'push' }}
+      if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       uses: cloudkernels/minio-upload@master
       with:
         url: https://s3.nubificus.co.uk
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
-        remote-path: nbfc-assets/github/vaccel/master/${{env.ARCH}}/${{env.JOB_TYPE}}/
+        remote-path: nbfc-assets/github/vaccel/${{ steps.extract_branch.outputs.branch }}/${{env.ARCHITECTURE}}/${{env.JOB_TYPE}}/
         local-path: /github/workspace/vaccel_${{matrix.arch}}_${{matrix.build_type}}.zip
 
     - name: Cleanup run
       if: ${{ always() }}
+      #if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         sudo rm -rf ${{ github.workspace }}/*
         sudo rm -rf ${{ github.workspace }}/.??*
@@ -249,27 +335,35 @@ jobs:
         echo "::set-output name=uid::$(id -u)"
         echo "::set-output name=gid::$(id -g)"
 
+    - name: Extract branch name
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${{ github.head_ref }})"
+      id: extract_branch
+
+
     - name: Download artifacts
-      if: ${{ github.event_name == 'push' }}
+      if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       uses: cloudkernels/minio-download@master
       with:
         url: https://s3.nubificus.co.uk
         access-key: ${{ env.NBFC_S3_ACCESS }}
         secret-key: ${{ env.NBFC_S3_SECRET }}
-        remote-path: nbfc-assets/github/vaccel/master/
+        remote-path: nbfc-assets/github/vaccel/${{ steps.extract_branch.outputs.branch }}/
         local-path: /github/workspace/master/
       env:
         ACTION_UID: ${{ steps.vars.outputs.uid }}
         ACTION_GID: ${{ steps.vars.outputs.gid }}
-
+        
     - name: Update the release
-      if: ${{ github.event_name == 'push' }}
+      if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       uses: marvinpinto/action-automatic-releases@latest
       with:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
-        automatic_release_tag: "latest"
+        automatic_release_tag: ${{ steps.extract_branch.outputs.branch }}
         prerelease: true
-        title: "Latest master build"
+        title: "Latest ${{ steps.extract_branch.outputs.branch }} build"
         files: |
           master/vaccel_x86_64_Debug.zip
           master/vaccel_x86_64_Release.zip

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "firecracker"]
 	path = firecracker
 	url = git@github.com:cloudkernels/firecracker.git
-[submodule "virtio-accel"]
-	path = virtio-accel
-	url = git@github.com:cloudkernels/virtio-accel.git
 [submodule "vaccel-grpc"]
 	path = vaccel-grpc
 	url = git@github.com:cloudkernels/vaccel-grpc.git
@@ -16,3 +13,10 @@
 [submodule "agent"]
 	path = agent
 	url = git@github.com:cloudkernels/vaccelrt-agent
+[submodule "kata-containers"]
+	path = kata-containers
+	url = https://github.com/nubificus/kata-containers
+	branch = vaccel-release
+[submodule "virtio-accel"]
+	path = virtio-accel
+	url = https://github.com/cloudkernels/virtio-accel

dockerfiles/ubuntu/latest/Dockerfile

@@ -31,12 +31,14 @@ RUN echo "export VACCEL_BACKENDS=/opt/vaccel/lib/libvaccel-virtio.so" >> /root/.
 
 # Enable ssh server
 RUN systemctl enable ssh
-COPY fc_test.pub id_rsa.pub
-RUN mkdir /root/.ssh/ && \
-    cat id_rsa.pub >> /root/.ssh/authorized_keys && \
-    chmod 0700 /root/.ssh && \
-    chmod 0600 /root/.ssh/authorized_keys && \
-    chown -R root:root /root/.ssh
+#COPY fc_test.pub id_rsa.pub
+RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+RUN echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config
+#RUN mkdir /root/.ssh/ && \
+#    cat id_rsa.pub >> /root/.ssh/authorized_keys && \
+#    chmod 0700 /root/.ssh && \
+#    chmod 0600 /root/.ssh/authorized_keys && \
+#    chown -R root:root /root/.ssh
 
 # Disable root password
 RUN sed s/root\:x\:/root\:\:/ -i /etc/passwd

kata-containers

@@ -77,10 +77,10 @@ build() {
 	cd ${BUILD_DIR}/rootfs
 
 	# Create RSA key to rootfs
-	ssh-keygen -t rsa -f fc_test -N ""
+	#ssh-keygen -t rsa -f fc_test -N ""
 
 	# Create root filesystem
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build --no-cache \
 		--network=host \
 		-t vaccel-rootfs \
 		--build-arg "KERNEL_VERSION=4.20.0" \
@@ -96,7 +96,7 @@ build() {
 	sudo mount rootfs.img $mnt
 	ok_or_die "Could not mount rootfs"
 
-	sudo rsync -aogxvPH rootfs/* $mnt
+	sudo rsync -aogxPH rootfs/* $mnt
 	sudo chown -R root:root $mnt/root
 	ok_or_die "Could not populate rootfs"
 
@@ -110,7 +110,7 @@ build() {
 	sudo rmdir $mnt
 
 	cp rootfs.img ${INSTALL_PREFIX}/share/
-	cp fc_test* ${INSTALL_PREFIX}/share/
+	#cp fc_test* ${INSTALL_PREFIX}/share/
 	cp -r imagenet/{networks,images} ${INSTALL_PREFIX}/share/
 }
 

scripts/build_rootfs.sh

@@ -13,7 +13,7 @@ SSH_TIMEOUT=300
 FC_IP="172.42.0.2"
 
 # Path to ssh private key
-SSH_KEY=$(pwd)/opt/share/fc_test
+#SSH_KEY=$(pwd)/opt/share/fc_test
 
 # script name for logging
 LOG_NAME="$(basename $0)"
@@ -28,7 +28,7 @@ print_help() {
 	echo "    -v|--vaccel     Directory of vAccel installation (default: '/opt/vaccel')"
 	echo "    -t|--timeout    Timeout in seconds to wait response from Firecracker (default: 300)"
 	echo "    -a|--ip-address Address of Firecracker VM"
-	echo "    -i|--ssh-key    RSA key to use for SSHing inside the VM"
+	echo "    -i|--ssh-key    RSA key to use for SSHing inside the VM (not currently used)"
 	echo ""
 }
 
@@ -37,7 +37,8 @@ run_test() {
 	in_fc_cmd="$in_fc_cmd VACCEL_BACKENDS=$VACCEL_PATH/lib/libvaccel-virtio.so"
 	in_fc_cmd="$in_fc_cmd $VACCEL_PATH/bin/classify /root/images/dog_0.jpg 1"
 
-	ssh -o StrictHostKeyChecking=no -i $SSH_KEY root@$FC_IP $in_fc_cmd
+	ssh -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null \
+		-o UserKnownHostsFile=/dev/null root@$FC_IP $in_fc_cmd
 }
 
 main() {

scripts/test_virtio.sh

@@ -13,7 +13,7 @@ SSH_TIMEOUT=300
 FC_IP="172.42.0.2"
 
 # Path to ssh private key
-SSH_KEY=$(pwd)/opt/share/fc_test
+#SSH_KEY=$(pwd)/opt/share/fc_test
 
 # vsock socket to use inside the VM
 VACCEL_VSOCK="vsock://2:2048"
@@ -37,7 +37,7 @@ print_help() {
 	echo "    -v|--vaccel     Directory of vAccel installation (default: '/opt/vaccel')"
 	echo "    -t|--timeout    Timeout in seconds to wait response from Firecracker (default: 300)"
 	echo "    -a|--ip-address Address of Firecracker VM"
-	echo "    -i|--ssh-key    RSA key to use for SSHing inside the VM"
+	echo "    -i|--ssh-key    RSA key to use for SSHing inside the VM (not currently used)"
 	echo "    -p|--plugin     Plugin to use for agent"
 	echo "    --vsock         Vsock socket to use inside the VM"
 	echo "    --unix          Unix socket to use on host"
@@ -62,7 +62,8 @@ run_test() {
 	launch_agent
 	ok_or_die "Could not launch agent"
 
-	ssh -o StrictHostKeyChecking=no -i $SSH_KEY root@$FC_IP $in_fc_cmd
+	ssh -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null \
+		-o UserKnownHostsFile=/dev/null root@$FC_IP $in_fc_cmd
 }
 
 main() {