Merge release branch 4.18 to main

DaanHoogland · DaanHoogland · commit 7b64236469c0 · 2023-08-10T14:50:46.000+02:00
* 4.18: server: remove registered userdata when cleanup an account (#7777) server: Use max secondary storage defined on the account during upload (#7441) test: upgrade kubernetes versions to 1.25.0/1.26.0 (#7685) kvm: Added VNI Devices as normal bridge slave devs (#7836) noVNC: fix JP keyboard on vmware7+ which uses websocket URL (#7694)
diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java
@@ -117,6 +117,15 @@ public interface ResourceLimitService {
      */
     public long findDefaultResourceLimitForDomain(ResourceType resourceType);
 
+    /**
+     * Finds the resource limit for a specified account, domain and type.
+     *
+     * @param domain
+     * @param type
+     * @return resource limit
+     */
+    public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type);
+
     /**
      * Increments the resource count
      *
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/UserDataDao.java b/engine/schema/src/main/java/com/cloud/user/dao/UserDataDao.java
@@ -25,4 +25,6 @@ public interface UserDataDao extends GenericDao<UserDataVO, Long> {
 
     public UserDataVO findByName(long accountId, long domainId, String name);
 
+    int removeByAccountId(long accountId);
+
 }
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/UserDataDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/dao/UserDataDaoImpl.java
@@ -63,4 +63,11 @@ public UserDataVO findByName(long accountId, long domainId, String name) {
 
         return findOneBy(sc);
     }
+
+    @Override
+    public int removeByAccountId(long accountId) {
+        SearchCriteria<UserDataVO> sc = userdataSearch.create();
+        sc.setParameters("accountId", accountId);
+        return remove(sc);
+    }
 }
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -1713,7 +1713,8 @@ private String matchPifFileInDirectory(final String bridgeName) {
             "^enx",
             "^dummy",
             "^lo",
-            "^p\\d+p\\d+"
+            "^p\\d+p\\d+",
+            "^vni"
     };
 
     /**
diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java
@@ -547,6 +547,18 @@ public long findDefaultResourceLimitForDomain(ResourceType resourceType) {
         return resourceLimit;
     }
 
+    @Override
+    public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type) {
+        long maxSecondaryStorageForAccount = findCorrectResourceLimitForAccount(account, type);
+        long maxSecondaryStorageForDomain = findCorrectResourceLimitForDomain(domain, type);
+
+        if (maxSecondaryStorageForDomain == Resource.RESOURCE_UNLIMITED || maxSecondaryStorageForAccount == Resource.RESOURCE_UNLIMITED) {
+            return Math.max(maxSecondaryStorageForDomain, maxSecondaryStorageForAccount);
+        }
+
+        return Math.min(maxSecondaryStorageForDomain, maxSecondaryStorageForAccount);
+    }
+
     @Override
     @DB
     public void checkResourceLimit(final Account account, final ResourceType type, long... count) throws ResourceAllocationException {
diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
@@ -34,6 +34,7 @@
 
 import javax.inject.Inject;
 
+import com.cloud.domain.dao.DomainDao;
 import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.InternalIdentity;
@@ -331,6 +332,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     @Inject
     protected SnapshotHelper snapshotHelper;
 
+    @Inject
+    protected DomainDao domainDao;
+
     @Inject
     protected ProjectManager projectManager;
     @Inject
@@ -486,13 +490,13 @@ public GetUploadParamsResponse doInTransaction(TransactionStatus status) throws
                 //using the existing max upload size configuration
                 command.setProcessTimeout(NumbersUtil.parseLong(_configDao.getValue("vmware.package.ova.timeout"), 3600));
                 command.setMaxUploadSize(_configDao.getValue(Config.MaxUploadVolumeSize.key()));
-                command.setAccountId(vol.getAccountId());
-                Account account = _accountDao.findById(vol.getAccountId());
-                if (account.getType().equals(Account.Type.PROJECT)) {
-                    command.setDefaultMaxSecondaryStorageInGB(ResourceLimitService.MaxProjectSecondaryStorage.value());
-                } else {
-                    command.setDefaultMaxSecondaryStorageInGB(ResourceLimitService.MaxAccountSecondaryStorage.value());
-                }
+
+                long accountId = vol.getAccountId();
+                Account account = _accountDao.findById(accountId);
+                Domain domain = domainDao.findById(account.getDomainId());
+
+                command.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage));
+                command.setAccountId(accountId);
                 Gson gson = new GsonBuilder().create();
                 String metadata = EncryptionUtil.encodeData(gson.toJson(command), key);
                 response.setMetadata(metadata);
diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -28,6 +28,7 @@
 
 import javax.inject.Inject;
 
+import com.cloud.domain.Domain;
 import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
 import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
 import org.apache.cloudstack.annotation.AnnotationService;
@@ -93,7 +94,6 @@
 import com.cloud.storage.download.DownloadMonitor;
 import com.cloud.template.VirtualMachineTemplate.State;
 import com.cloud.user.Account;
-import com.cloud.user.ResourceLimitService;
 import com.cloud.utils.Pair;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.db.DB;
@@ -402,13 +402,13 @@ public List<TemplateOrVolumePostUploadCommand> doInTransaction(TransactionStatus
                             templateOnStore.getDataStore().getRole().toString());
                     //using the existing max template size configuration
                     payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key()));
-                    payload.setAccountId(template.getAccountId());
-                    Account account = _accountDao.findById(template.getAccountId());
-                    if (account.getType().equals(Account.Type.PROJECT)) {
-                        payload.setDefaultMaxSecondaryStorageInGB(ResourceLimitService.MaxProjectSecondaryStorage.value());
-                    } else {
-                        payload.setDefaultMaxSecondaryStorageInGB(ResourceLimitService.MaxAccountSecondaryStorage.value());
-                    }
+
+                    Long accountId = template.getAccountId();
+                    Account account = _accountDao.findById(accountId);
+                    Domain domain = _domainDao.findById(account.getDomainId());
+
+                    payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage));
+                    payload.setAccountId(accountId);
                     payload.setRemoteEndPoint(ep.getPublicAddr());
                     payload.setRequiresHvm(template.requiresHvm());
                     payload.setDescription(template.getDisplayText());
diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
@@ -162,6 +162,7 @@
 import com.cloud.user.dao.SSHKeyPairDao;
 import com.cloud.user.dao.UserAccountDao;
 import com.cloud.user.dao.UserDao;
+import com.cloud.user.dao.UserDataDao;
 import com.cloud.utils.ConstantTimeComparator;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
@@ -292,6 +293,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
     private GlobalLoadBalancerRuleDao _gslbRuleDao;
     @Inject
     private SSHKeyPairDao _sshKeyPairDao;
+    @Inject
+    private UserDataDao userDataDao;
 
     private List<QuerySelector> _querySelectors;
 
@@ -1089,6 +1092,10 @@ public int compare(NetworkVO network1, NetworkVO network2) {
             for (SSHKeyPairVO keypair : sshkeypairs) {
                 _sshKeyPairDao.remove(keypair.getId());
             }
+
+            // Delete registered UserData
+            userDataDao.removeByAccountId(accountId);
+
             return true;
         } catch (Exception ex) {
             s_logger.warn("Failed to cleanup account " + account + " due to ", ex);
diff --git a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java
@@ -385,7 +385,7 @@ private ConsoleEndpoint composeConsoleAccessEndpoint(String rootUrl, VirtualMach
         String token = encryptor.encryptObject(ConsoleProxyClientParam.class, param);
         int vncPort = consoleProxyManager.getVncPort();
 
-        String url = generateConsoleAccessUrl(rootUrl, param, token, vncPort, vm);
+        String url = generateConsoleAccessUrl(rootUrl, param, token, vncPort, vm, hostVo, details);
 
         s_logger.debug("Adding allowed session: " + sessionUuid);
         persistConsoleSession(sessionUuid, vm.getId(), hostVo.getId());
@@ -413,7 +413,7 @@ protected void persistConsoleSession(String sessionUuid, long instanceId, long h
     }
 
     private String generateConsoleAccessUrl(String rootUrl, ConsoleProxyClientParam param, String token, int vncPort,
-                                            VirtualMachine vm) {
+                                            VirtualMachine vm, HostVO hostVo, UserVmDetailVO details) {
         StringBuilder sb = new StringBuilder(rootUrl);
         if (param.getHypervHost() != null || !ConsoleProxyManager.NoVncConsoleDefault.value()) {
             sb.append("/ajax?token=" + token);
@@ -422,6 +422,9 @@ private String generateConsoleAccessUrl(String rootUrl, ConsoleProxyClientParam
                     .append("?autoconnect=true")
                     .append("&port=" + vncPort)
                     .append("&token=" + token);
+            if (requiresVncOverWebSocketConnection(vm, hostVo) && details != null && details.getValue() != null) {
+                sb.append("&language=" + details.getValue());
+            }
         }
 
         if (StringUtils.isNotBlank(param.getExtraSecurityToken())) {
diff --git a/server/src/test/java/com/cloud/user/AccountManagerImplTest.java b/server/src/test/java/com/cloud/user/AccountManagerImplTest.java
@@ -164,6 +164,7 @@ public void deleteUserAccount() {
         sshkeyList.add(sshkey);
         Mockito.when(_sshKeyPairDao.listKeyPairs(Mockito.anyLong(), Mockito.anyLong())).thenReturn(sshkeyList);
         Mockito.when(_sshKeyPairDao.remove(Mockito.anyLong())).thenReturn(true);
+        Mockito.when(userDataDao.removeByAccountId(Mockito.anyLong())).thenReturn(222);
 
         Assert.assertTrue(accountManagerImpl.deleteUserAccount(42l));
         // assert that this was a clean delete
diff --git a/server/src/test/java/com/cloud/user/AccountManagetImplTestBase.java b/server/src/test/java/com/cloud/user/AccountManagetImplTestBase.java
@@ -73,6 +73,7 @@
 import com.cloud.user.dao.SSHKeyPairDao;
 import com.cloud.user.dao.UserAccountDao;
 import com.cloud.user.dao.UserDao;
+import com.cloud.user.dao.UserDataDao;
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.InstanceGroupDao;
@@ -193,6 +194,8 @@ public class AccountManagetImplTestBase {
     OrchestrationService _orchSrvc;
     @Mock
     SSHKeyPairDao _sshKeyPairDao;
+    @Mock
+    UserDataDao userDataDao;
 
     @Spy
     @InjectMocks
diff --git a/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java
@@ -93,6 +93,11 @@ public long findDefaultResourceLimitForDomain(ResourceType resourceType) {
         return 0;
     }
 
+    @Override
+    public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type) {
+        return 0;
+    }
+
     /* (non-Javadoc)
      * @see com.cloud.user.ResourceLimitService#incrementResourceCount(long, com.cloud.configuration.Resource.ResourceType, java.lang.Long[])
      */
diff --git a/systemvm/agent/noVNC/app/ui.js b/systemvm/agent/noVNC/app/ui.js
@@ -1077,6 +1077,7 @@ const UI = {
         UI.rfb = new RFB(document.getElementById('noVNC_container'), url,
                          { shared: UI.getSetting('shared'),
                            repeaterID: UI.getSetting('repeaterID'),
+                           language: WebUtil.getConfigVar('language'),
                            credentials: { password: password } });
         UI.rfb.addEventListener("connect", UI.connectFinished);
         UI.rfb.addEventListener("disconnect", UI.disconnectFinished);
diff --git a/systemvm/agent/noVNC/core/rfb.js b/systemvm/agent/noVNC/core/rfb.js
@@ -37,6 +37,7 @@ import TightDecoder from "./decoders/tight.js";
 import TightPNGDecoder from "./decoders/tightpng.js";
 import ZRLEDecoder from "./decoders/zrle.js";
 import JPEGDecoder from "./decoders/jpeg.js";
+import SCANCODES_JP from "../keymaps/keymap-ja-atset1.js"
 
 // How many seconds to wait for a disconnect to finish
 const DISCONNECT_TIMEOUT = 3;
@@ -118,8 +119,15 @@ export default class RFB extends EventTargetMixin {
         this._rfbCredentials = options.credentials || {};
         this._shared = 'shared' in options ? !!options.shared : true;
         this._repeaterID = options.repeaterID || '';
+        this._language = options.language || '';
         this._wsProtocols = ['binary'];
 
+        // Keymaps
+        this._scancodes = {};
+        if (this._language === "jp") {
+            this._scancodes = SCANCODES_JP;
+        }
+
         // Internal state
         this._rfbConnectionState = '';
         this._rfbInitState = '';
@@ -181,6 +189,10 @@ export default class RFB extends EventTargetMixin {
             encoding: null,
         };
 
+        // Keys
+        this._shiftPressed = false;
+        this._shiftKey = KeyTable.XK_Shift_L;
+
         // Mouse state
         this._mousePos = {};
         this._mouseButtonMask = 0;
@@ -506,13 +518,43 @@ export default class RFB extends EventTargetMixin {
 
         const scancode = XtScancode[code];
 
+        if (keysym === KeyTable.XK_Shift_L || keysym === KeyTable.XK_Shift_R) {
+            this._shiftPressed = down;
+            this._shiftKey = down ? keysym : KeyTable.XK_Shift_L;
+        }
+
         if (this._qemuExtKeyEventSupported && scancode) {
             // 0 is NoSymbol
             keysym = keysym || 0;
 
             Log.Info("Sending key (" + (down ? "down" : "up") + "): keysym " + keysym + ", scancode " + scancode);
 
             RFB.messages.QEMUExtendedKeyEvent(this._sock, keysym, down, scancode);
+        } else if (Object.keys(this._scancodes).length > 0) {
+            let vscancode = this._scancodes[keysym]
+            if (vscancode) {
+                let shifted = vscancode.includes("shift");
+                let vscancode_int = parseInt(vscancode);
+                let isLetter = (keysym >= 65 && keysym <=90) || (keysym >=97 && keysym <=122);
+                if (shifted && ! this._shiftPressed && ! isLetter) {
+                    RFB.messages.keyEvent(this._sock, this._shiftKey, 1);
+                }
+                if (! shifted && this._shiftPressed && ! isLetter) {
+                    RFB.messages.keyEvent(this._sock, this._shiftKey, 0);
+                }
+                RFB.messages.VMwareExtendedKeyEvent(this._sock, keysym, down, vscancode_int);
+                if (shifted && ! this._shiftPressed && ! isLetter) {
+                    RFB.messages.keyEvent(this._sock, this._shiftKey, 0);
+                }
+                if (! shifted && this._shiftPressed && ! isLetter) {
+                    RFB.messages.keyEvent(this._sock, this._shiftKey, 1);
+                }
+            } else {
+                if (this._language === "jp" && keysym === 65328) {
+                    keysym = 65509; // Caps lock
+                }
+                RFB.messages.keyEvent(this._sock, keysym, down ? 1 : 0);
+            }
         } else {
             if (!keysym) {
                 return;
@@ -3031,6 +3073,26 @@ RFB.messages = {
         sock.flush();
     },
 
+    VMwareExtendedKeyEvent(sock, keysym, down, keycode) {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        buff[offset] = 127;     // msgVMWClientMessage
+        buff[offset + 1] = 0;   // msgVMWKeyEvent
+
+        buff[offset + 2] = 0;
+        buff[offset + 3] = 8;
+
+        buff[offset + 4] = (keycode >> 8);
+        buff[offset + 5] = keycode;
+
+        buff[offset + 6] = down;
+        buff[offset + 7] = 0;
+
+        sock._sQlen += 8;
+        sock.flush();
+    },
+
     pointerEvent(sock, x, y, mask) {
         const buff = sock._sQ;
         const offset = sock._sQlen;
diff --git a/systemvm/agent/noVNC/keymaps/generate-language-keymaps.py b/systemvm/agent/noVNC/keymaps/generate-language-keymaps.py
diff --git a/systemvm/agent/noVNC/keymaps/keymap-ja b/systemvm/agent/noVNC/keymaps/keymap-ja
diff --git a/systemvm/agent/noVNC/keymaps/keymap-ja-atset1.js b/systemvm/agent/noVNC/keymaps/keymap-ja-atset1.js
diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py
diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py
diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py

Original file line number	Diff line number	Diff line change
`@@ -25,4 +25,6 @@ public interface UserDataDao extends GenericDao<UserDataVO, Long> {`
`25`	`25`
`26`	`26`	`public UserDataVO findByName(long accountId, long domainId, String name);`
`27`	`27`
	`28`	`+ int removeByAccountId(long accountId);`
	`29`	`+`
`28`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -63,4 +63,11 @@ public UserDataVO findByName(long accountId, long domainId, String name) {`
`63`	`63`
`64`	`64`	`return findOneBy(sc);`
`65`	`65`	`}`
	`66`	`+`
	`67`	`+ @Override`
	`68`	`+ public int removeByAccountId(long accountId) {`
	`69`	`+ SearchCriteria<UserDataVO> sc = userdataSearch.create();`
	`70`	`+ sc.setParameters("accountId", accountId);`
	`71`	`+ return remove(sc);`
	`72`	`+ }`
`66`	`73`	`}`