Feature: Add support for GPU with KVM hosts (apache#11143)

This PR allows attaching of GPU devices via PCI, mdev or VF to an Instance for KVM.

It allows the operator to discover the GPU devices on the KVM host and create a Compute Offering with GPU support based on the available GPU devices on the host. Once the operator has created the Compute offering, it can be used by users to launch Instances with GPU devices.

Author: vishesh92

.github/workflows/ci.yml

@@ -140,6 +140,7 @@ jobs:
                   smoke/test_vm_deployment_planner
                   smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
+                  smoke/test_deploy_vgpu_enabled_vm
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -221,6 +221,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.script", "libvirt-vm-xml-transformer.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM XML transformer shell script.<br>
+     * The shell script is used to execute the Libvirt VM XML transformer script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-xml-transformer.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.shell_script", "libvirt-vm-xml-transformer.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_xml_transformer.script properties to define the Libvirt VM XML transformer method.<br>
      * Libvirt XML transformer hook does XML-to-XML transformation.<br>
@@ -241,6 +250,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on start shell script.<br>
+     * The shell script is used to execute the Libvirt VM on start script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_start.script properties to define the Libvirt VM on start method.<br>
      * The hook is called right after Libvirt successfully launched the VM.<br>
@@ -260,6 +278,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on stop shell script.<br>
+     * The shell script is used to execute the Libvirt VM on stop script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_stop.script properties to define the Libvirt VM on stop method.<br>
      * The hook is called right after libvirt successfully stopped the VM.<br>

api/src/main/java/com/cloud/agent/api/VgpuTypesInfo.java

@@ -15,17 +15,32 @@
 // specific language governing permissions and limitations
 // under the License.
 package com.cloud.agent.api;
+
+import org.apache.cloudstack.gpu.GpuDevice;
+
 public class VgpuTypesInfo {
 
+    private boolean passthroughEnabled = true;
+    private GpuDevice.DeviceType deviceType;
+    private String parentBusAddress;
+    private String busAddress;
+    private String numaNode;
+    private String pciRoot;
+    private String deviceId;
+    private String deviceName;
+    private String vendorId;
+    private String vendorName;
     private String modelName;
     private String groupName;
+    private String vmName;
     private Long maxHeads;
     private Long videoRam;
     private Long maxResolutionX;
     private Long maxResolutionY;
     private Long maxVgpuPerGpu;
     private Long remainingCapacity;
     private Long maxCapacity;
+    private boolean display = false;
 
     public String getModelName() {
         return modelName;
@@ -39,22 +54,42 @@ public Long getVideoRam() {
         return videoRam;
     }
 
+    public void setVideoRam(Long videoRam) {
+        this.videoRam = videoRam;
+    }
+
     public Long getMaxHeads() {
         return maxHeads;
     }
 
+    public void setMaxHeads(Long maxHeads) {
+        this.maxHeads = maxHeads;
+    }
+
     public Long getMaxResolutionX() {
         return maxResolutionX;
     }
 
+    public void setMaxResolutionX(Long maxResolutionX) {
+        this.maxResolutionX = maxResolutionX;
+    }
+
     public Long getMaxResolutionY() {
         return maxResolutionY;
     }
 
+    public void setMaxResolutionY(Long maxResolutionY) {
+        this.maxResolutionY = maxResolutionY;
+    }
+
     public Long getMaxVpuPerGpu() {
         return maxVgpuPerGpu;
     }
 
+    public void setMaxVgpuPerGpu(Long maxVgpuPerGpu) {
+        this.maxVgpuPerGpu = maxVgpuPerGpu;
+    }
+
     public Long getRemainingCapacity() {
         return remainingCapacity;
     }
@@ -71,8 +106,133 @@ public void setMaxVmCapacity(Long maxCapacity) {
         this.maxCapacity = maxCapacity;
     }
 
-    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX, Long maxResolutionY, Long maxVgpuPerGpu,
-            Long remainingCapacity, Long maxCapacity) {
+    public boolean isPassthroughEnabled() {
+        return passthroughEnabled;
+    }
+
+    public void setPassthroughEnabled(boolean passthroughEnabled) {
+        this.passthroughEnabled = passthroughEnabled;
+    }
+
+    public GpuDevice.DeviceType getDeviceType() {
+        return deviceType;
+    }
+
+    public void setDeviceType(GpuDevice.DeviceType deviceType) {
+        this.deviceType = deviceType;
+    }
+
+    public String getParentBusAddress() {
+        return parentBusAddress;
+    }
+
+    public void setParentBusAddress(String parentBusAddress) {
+        this.parentBusAddress = parentBusAddress;
+    }
+
+    public String getBusAddress() {
+        return busAddress;
+    }
+
+    public void setBusAddress(String busAddress) {
+        this.busAddress = busAddress;
+    }
+
+    public String getNumaNode() {
+        return numaNode;
+    }
+
+    public void setNumaNode(String numaNode) {
+        this.numaNode = numaNode;
+    }
+
+    public String getPciRoot() {
+        return pciRoot;
+    }
+
+    public void setPciRoot(String pciRoot) {
+        this.pciRoot = pciRoot;
+    }
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public void setDeviceId(String deviceId) {
+        this.deviceId = deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public void setDeviceName(String deviceName) {
+        this.deviceName = deviceName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public void setVendorId(String vendorId) {
+        this.vendorId = vendorId;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public void setVendorName(String vendorName) {
+        this.vendorName = vendorName;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    public void setVmName(String vmName) {
+        this.vmName = vmName;
+    }
+
+    public boolean isDisplay() {
+        return display;
+    }
+
+    public void setDisplay(boolean display) {
+        this.display = display;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+            String vendorId, String vendorName, String deviceId, String deviceName, String numaNode, String pciRoot
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+        this.numaNode = numaNode;
+        this.pciRoot = pciRoot;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+                         String vendorId, String vendorName, String deviceId, String deviceName
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+    }
+
+    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX,
+            Long maxResolutionY, Long maxVgpuPerGpu, Long remainingCapacity, Long maxCapacity
+    ) {
         this.groupName = groupName;
         this.modelName = modelName;
         this.videoRam = videoRam;

api/src/main/java/com/cloud/agent/api/to/GPUDeviceTO.java

@@ -16,17 +16,33 @@
 // under the License.
 package com.cloud.agent.api.to;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import com.cloud.agent.api.VgpuTypesInfo;
 
 public class GPUDeviceTO {
 
     private String gpuGroup;
     private String vgpuType;
+    private int gpuCount;
     private HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = new HashMap<String, HashMap<String, VgpuTypesInfo>>();
+    private List<VgpuTypesInfo> gpuDevices = new ArrayList<>();
 
-    public GPUDeviceTO( String gpuGroup, String vgpuType, HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
+    public GPUDeviceTO(String gpuGroup, String vgpuType, int gpuCount,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails,
+                       List<VgpuTypesInfo> gpuDevices) {
+        this.gpuGroup = gpuGroup;
+        this.vgpuType = vgpuType;
+        this.groupDetails = groupDetails;
+        this.gpuCount = gpuCount;
+        this.gpuDevices = gpuDevices;
+
+    }
+
+    public GPUDeviceTO(String gpuGroup, String vgpuType,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
         this.gpuGroup = gpuGroup;
         this.vgpuType = vgpuType;
         this.groupDetails = groupDetails;
@@ -48,6 +64,14 @@ public void setVgpuType(String vgpuType) {
         this.vgpuType = vgpuType;
     }
 
+    public int getGpuCount() {
+        return gpuCount;
+    }
+
+    public void setGpuCount(int gpuCount) {
+        this.gpuCount = gpuCount;
+    }
+
     public HashMap<String, HashMap<String, VgpuTypesInfo>> getGroupDetails() {
         return groupDetails;
     }
@@ -56,4 +80,11 @@ public void setGroupDetails(HashMap<String, HashMap<String, VgpuTypesInfo>> grou
         this.groupDetails = groupDetails;
     }
 
+    public List<VgpuTypesInfo> getGpuDevices() {
+        return gpuDevices;
+    }
+
+    public void setGpuDevices(List<VgpuTypesInfo> gpuDevices) {
+        this.gpuDevices = gpuDevices;
+    }
 }

api/src/main/java/com/cloud/configuration/Resource.java

@@ -37,7 +37,8 @@ enum ResourceType { // All storage type resources are allocated_storage and not
         backup("backup", 12),
         backup_storage("backup_storage", 13),
         bucket("bucket", 14),
-        object_storage("object_storage", 15);
+        object_storage("object_storage", 15),
+        gpu("gpu", 16);
 
         private String name;
         private int ordinal;

api/src/main/java/com/cloud/event/EventTypes.java

@@ -31,6 +31,9 @@
 import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet;
 import org.apache.cloudstack.extension.Extension;
 import org.apache.cloudstack.extension.ExtensionCustomAction;
+import org.apache.cloudstack.gpu.GpuCard;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.cloudstack.gpu.VgpuProfile;
 import org.apache.cloudstack.ha.HAConfig;
 import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap;
@@ -383,6 +386,21 @@ public class EventTypes {
     public static final String EVENT_DISK_OFFERING_EDIT = "DISK.OFFERING.EDIT";
     public static final String EVENT_DISK_OFFERING_DELETE = "DISK.OFFERING.DELETE";
 
+    // GPU Cards
+    public static final String EVENT_GPU_CARD_CREATE = "GPU.CARD.CREATE";
+    public static final String EVENT_GPU_CARD_EDIT = "GPU.CARD.EDIT";
+    public static final String EVENT_GPU_CARD_DELETE = "GPU.CARD.DELETE";
+
+    // vGPU Profile
+    public static final String EVENT_VGPU_PROFILE_CREATE = "VGPU.PROFILE.CREATE";
+    public static final String EVENT_VGPU_PROFILE_EDIT = "VGPU.PROFILE.EDIT";
+    public static final String EVENT_VGPU_PROFILE_DELETE = "VGPU.PROFILE.DELETE";
+
+    // GPU Devices
+    public static final String EVENT_GPU_DEVICE_CREATE = "GPU.DEVICE.CREATE";
+    public static final String EVENT_GPU_DEVICE_EDIT = "GPU.DEVICE.EDIT";
+    public static final String EVENT_GPU_DEVICE_DELETE = "GPU.DEVICE.DELETE";
+
     // Network offerings
     public static final String EVENT_NETWORK_OFFERING_CREATE = "NETWORK.OFFERING.CREATE";
     public static final String EVENT_NETWORK_OFFERING_ASSIGN = "NETWORK.OFFERING.ASSIGN";
@@ -1040,6 +1058,21 @@ public class EventTypes {
         entityEventDetails.put(EVENT_DISK_OFFERING_EDIT, DiskOffering.class);
         entityEventDetails.put(EVENT_DISK_OFFERING_DELETE, DiskOffering.class);
 
+        // GPU Cards
+        entityEventDetails.put(EVENT_GPU_CARD_CREATE, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_EDIT, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_DELETE, GpuCard.class);
+
+        // vGPU Profiles
+        entityEventDetails.put(EVENT_VGPU_PROFILE_CREATE, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_EDIT, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_DELETE, VgpuProfile.class);
+
+        // GPU Devices
+        entityEventDetails.put(EVENT_GPU_DEVICE_CREATE, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_EDIT, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_DELETE, GpuDevice.class);
+
         // Network offerings
         entityEventDetails.put(EVENT_NETWORK_OFFERING_CREATE, NetworkOffering.class);
         entityEventDetails.put(EVENT_NETWORK_OFFERING_ASSIGN, NetworkOffering.class);