Please log out and log in again.`); if (this.__message === null) { - this.__message = osparc.FlashMessenger.getInstance().logAs(text, "WARNING", timeoutSec*1000); + this.__message = osparc.FlashMessenger.logAs(text, "WARNING", timeoutSec*1000); this.__message.getChildControl("closebutton").exclude(); } else { this.__message.setMessage(text); @@ -112,7 +112,7 @@ qx.Class.define("osparc.CookieExpirationTracker", { // /FLASH MESSAGE // __logoutUser: function() { - const reason = qx.locale.Manager.tr("Session expired"); + const reason = qx.locale.Manager.tr("Your session has expired"); qx.core.Init.getApplication().logout(reason); } } diff --git a/services/static-webserver/client/source/class/osparc/DownloadLinkTracker.js b/services/static-webserver/client/source/class/osparc/DownloadLinkTracker.js index 27128e42ad0..af259e97a8c 100644 --- a/services/static-webserver/client/source/class/osparc/DownloadLinkTracker.js +++ b/services/static-webserver/client/source/class/osparc/DownloadLinkTracker.js @@ -33,9 +33,10 @@ qx.Class.define("osparc.DownloadLinkTracker", { downloadAnchorNode.setAttribute("href", url); downloadAnchorNode.setAttribute("download", fileName); downloadAnchorNode.setAttribute("osparc", "downloadFile"); + document.body.appendChild(downloadAnchorNode); this.setDownloading(true); downloadAnchorNode.click(); - downloadAnchorNode.remove(); + document.body.removeChild(downloadAnchorNode); // This is needed to make it work in Firefox setTimeout(() => this.setDownloading(false), 100); } diff --git a/services/static-webserver/client/source/class/osparc/FlashMessenger.js b/services/static-webserver/client/source/class/osparc/FlashMessenger.js index eb0dfe7cab3..d9c886fd506 100644 --- a/services/static-webserver/client/source/class/osparc/FlashMessenger.js +++ b/services/static-webserver/client/source/class/osparc/FlashMessenger.js @@ -28,7 +28,7 @@ * Here is a little example of how to use the class. * *
- * osparc.FlashMessenger.getInstance().logAs(log); + * osparc.FlashMessenger.logAs(log); **/ @@ -55,9 +55,67 @@ qx.Class.define("osparc.FlashMessenger", { statics: { MAX_DISPLAYED: 3, + + extractMessage: function(input, defaultMessage = "") { + const isValidString = val => { + return ( + typeof val === "string" || + (osparc.utils.Utils.isObject(val) && ("basename" in val) && (val.basename === "LocalizedString")) + ); + } + if (input) { + if (isValidString(input)) { + return input; + } else if (osparc.utils.Utils.isObject(input) && "message" in input) { + if (isValidString(input["message"])) { + return input["message"]; + } else if (osparc.utils.Utils.isObject(input["message"]) && "message" in input["message"] && isValidString(input["message"]["message"])) { + return input["message"]["message"]; + } + } + } + return defaultMessage; + }, + logAs: function(message, level, duration) { return this.getInstance().logAs(message, level, duration); - } + }, + + logError: function(error, defaultMessage = qx.locale.Manager.tr("Oops... something went wrong"), duration = null) { + if (error) { + console.error(error); + } + const msg = this.extractMessage(error, defaultMessage); + const flashMessage = this.getInstance().logAs(msg, "ERROR", duration); + if (error && error["supportId"]) { + flashMessage.addWidget(this.__createCopyOECWidget(msg, error["supportId"])); + flashMessage.setDuration(flashMessage.getDuration()*2); + } + return flashMessage; + }, + + __createCopyOECWidget: function(message, supportId) { + const errorLabel = new qx.ui.basic.Atom().set({ + label: supportId, + icon: "@FontAwesome5Solid/copy/10", + iconPosition: "right", + gap: 8, + cursor: "pointer", + alignX: "center", + allowGrowX: false, + }); + errorLabel.addListener("tap", () => { + const dataToClipboard = { + message, + supportId, + timestamp: new Date().toString(), + url: window.location.href, + studyId: osparc.store.Store.getInstance().getCurrentStudy() || "", + } + osparc.utils.Utils.copyTextToClipboard(JSON.stringify(dataToClipboard)); + }); + return errorLabel; + }, }, members: { @@ -68,7 +126,7 @@ qx.Class.define("osparc.FlashMessenger", { /** * Public function to log a FlashMessage to the user. * - * @param {String} message Message that the message will show. + * @param {String || Object} message Message (or Object containing the message) that the message will show. * @param {String="INFO","DEBUG","WARNING","ERROR"} level Level of the warning. The color of the badge will change accordingly. * @param {Number} duration */ @@ -81,9 +139,7 @@ qx.Class.define("osparc.FlashMessenger", { }, log: function(logMessage) { - const message = osparc.utils.Utils.isObject(logMessage.message) && "message" in logMessage.message ? - logMessage.message.message : - logMessage.message; + const message = this.self().extractMessage(logMessage); const level = logMessage.level.toUpperCase(); // "DEBUG", "INFO", "WARNING", "ERROR" @@ -115,14 +171,9 @@ qx.Class.define("osparc.FlashMessenger", { } this.__displayedMessagesCount++; - let duration = flashMessage.getDuration(); - if (duration === null) { - const message = flashMessage.getMessage(); - const wordCount = message.split(" ").length; - duration = Math.max(5500, wordCount*500); // An average reader takes 300ms to read a word - } + const duration = flashMessage.getDuration(); if (duration !== 0) { - qx.event.Timer.once(() => this.removeMessage(flashMessage), this, duration); + flashMessage.timer = setTimeout(() => this.removeMessage(flashMessage), duration); } }, diff --git a/services/static-webserver/client/source/class/osparc/MaintenanceTracker.js b/services/static-webserver/client/source/class/osparc/MaintenanceTracker.js index d6aa69a3084..4a8d552deff 100644 --- a/services/static-webserver/client/source/class/osparc/MaintenanceTracker.js +++ b/services/static-webserver/client/source/class/osparc/MaintenanceTracker.js @@ -166,7 +166,7 @@ qx.Class.define("osparc.MaintenanceTracker", { end: null, reason: null }); - const reason = qx.locale.Manager.tr("We are under maintenance. Please check back later"); + const reason = qx.locale.Manager.tr("The service is under maintenance. Please check back later"); qx.core.Init.getApplication().logout(reason); }, diff --git a/services/static-webserver/client/source/class/osparc/NewRelease.js b/services/static-webserver/client/source/class/osparc/NewRelease.js index bac9d1efb25..939c4509234 100644 --- a/services/static-webserver/client/source/class/osparc/NewRelease.js +++ b/services/static-webserver/client/source/class/osparc/NewRelease.js @@ -70,11 +70,11 @@ qx.Class.define("osparc.NewRelease", { }); this._add(introLabel); - const rData = osparc.store.StaticInfo.getInstance().getReleaseData(); - const url = rData["url"] || osparc.utils.LibVersions.getVcsRefUrl(); + const releaseTag = osparc.utils.Utils.getReleaseTag(); + const releaseLink = osparc.utils.Utils.getReleaseLink(); const linkLabel = new osparc.ui.basic.LinkLabel().set({ - value: this.tr("What's new"), - url, + value: this.tr("What's new in ") + releaseTag, + url: releaseLink, font: "link-label-14" }); this._add(linkLabel); diff --git a/services/static-webserver/client/source/class/osparc/NewUITracker.js b/services/static-webserver/client/source/class/osparc/NewUITracker.js index c85fb3f9390..3955b4ef2b5 100644 --- a/services/static-webserver/client/source/class/osparc/NewUITracker.js +++ b/services/static-webserver/client/source/class/osparc/NewUITracker.js @@ -36,7 +36,7 @@ qx.Class.define("osparc.NewUITracker", { msg += "
"; msg += qx.locale.Manager.tr("Click the Reload button to get the latest features."); // permanent message - const flashMessage = osparc.FlashMessenger.getInstance().logAs(msg, "INFO", 0).set({ + const flashMessage = osparc.FlashMessenger.logAs(msg, "INFO", 0).set({ maxWidth: 500 }); const reloadButton = osparc.utils.Utils.reloadNoCacheButton(); diff --git a/services/static-webserver/client/source/class/osparc/Preferences.js b/services/static-webserver/client/source/class/osparc/Preferences.js index 6d5dea18b5d..b2c6977853d 100644 --- a/services/static-webserver/client/source/class/osparc/Preferences.js +++ b/services/static-webserver/client/source/class/osparc/Preferences.js @@ -173,8 +173,7 @@ qx.Class.define("osparc.Preferences", { osparc.Preferences.patchPreference(preferenceId, newValue) .then(() => preferencesSettings.set(preferenceId, newValue)) .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); preferenceField.setValue(oldValue); }) .finally(() => preferenceField.setEnabled(true)); @@ -208,10 +207,7 @@ qx.Class.define("osparc.Preferences", { wallets.forEach(wallet => wallet.setPreferredWallet(wallet.getWalletId() === walletId)); this.setPreferredWalletId(walletId); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err)); }, __patchPreference: function(value, _, propName) { diff --git a/services/static-webserver/client/source/class/osparc/TooSmallDialog.js b/services/static-webserver/client/source/class/osparc/TooSmallDialog.js index b60219820f6..16b2e679118 100644 --- a/services/static-webserver/client/source/class/osparc/TooSmallDialog.js +++ b/services/static-webserver/client/source/class/osparc/TooSmallDialog.js @@ -19,7 +19,7 @@ qx.Class.define("osparc.TooSmallDialog", { extend: osparc.ui.window.SingletonWindow, construct: function() { - this.base(arguments, "too-small-logout", this.tr("Window too small")); + this.base(arguments, "too-small-logout", this.tr("Window size too small")); this.set({ layout: new qx.ui.layout.VBox(10), diff --git a/services/static-webserver/client/source/class/osparc/WatchDog.js b/services/static-webserver/client/source/class/osparc/WatchDog.js index 2eacf138d59..61817320ce0 100644 --- a/services/static-webserver/client/source/class/osparc/WatchDog.js +++ b/services/static-webserver/client/source/class/osparc/WatchDog.js @@ -61,7 +61,7 @@ qx.Class.define("osparc.WatchDog", { init: false, nullable: false, event: "changeOnline", - apply: "_applyOnline" + apply: "__applyOnline" }, heartbeatInterval: { @@ -75,8 +75,8 @@ qx.Class.define("osparc.WatchDog", { members: { __clientHeartbeatWWPinger: null, - _applyOnline: function(value) { - let logo = osparc.navigation.LogoOnOff.getInstance(); + __applyOnline: function(value) { + const logo = osparc.navigation.LogoOnOff.getInstance(); if (logo) { logo.setOnline(value); } diff --git a/services/static-webserver/client/source/class/osparc/activityManager/ActivityManager.js b/services/static-webserver/client/source/class/osparc/activityManager/ActivityManager.js index b313750f99a..05ce9b1ca28 100644 --- a/services/static-webserver/client/source/class/osparc/activityManager/ActivityManager.js +++ b/services/static-webserver/client/source/class/osparc/activityManager/ActivityManager.js @@ -119,15 +119,15 @@ qx.Class.define("osparc.activityManager.ActivityManager", { /* const runButton = new qx.ui.toolbar.Button(this.tr("Run"), "@FontAwesome5Solid/play/14"); actionsPart.add(runButton); - runButton.addListener("execute", () => osparc.FlashMessenger.getInstance().logAs("Not implemented")); + runButton.addListener("execute", () => osparc.FlashMessenger.logAs("Not implemented")); const stopButton = new qx.ui.toolbar.Button(this.tr("Stop"), "@FontAwesome5Solid/stop-circle/14"); actionsPart.add(stopButton); - stopButton.addListener("execute", () => osparc.FlashMessenger.getInstance().logAs("Not implemented")); + stopButton.addListener("execute", () => osparc.FlashMessenger.logAs("Not implemented")); const infoButton = new qx.ui.toolbar.Button(this.tr("Info"), "@FontAwesome5Solid/info/14"); actionsPart.add(infoButton); - infoButton.addListener("execute", () => osparc.FlashMessenger.getInstance().logAs("Not implemented")); + infoButton.addListener("execute", () => osparc.FlashMessenger.logAs("Not implemented")); [runButton, stopButton, infoButton].map(button => this.__tree.bind("selected", button, "enabled", { converter: data => data.length > 0 diff --git a/services/static-webserver/client/source/class/osparc/activityManager/ActivityTree.js b/services/static-webserver/client/source/class/osparc/activityManager/ActivityTree.js index 2baea59d4d9..fc1eb0e8011 100644 --- a/services/static-webserver/client/source/class/osparc/activityManager/ActivityTree.js +++ b/services/static-webserver/client/source/class/osparc/activityManager/ActivityTree.js @@ -241,9 +241,7 @@ qx.Class.define("osparc.activityManager.ActivityTree", { } this.fireEvent("treeUpdated"); }) - .catch(e => { - console.error(e); - }) + .catch(err => console.error(err)) .then(() => { // Give a 2 seconds delay setTimeout(() => { diff --git a/services/static-webserver/client/source/class/osparc/admin/AdminCenter.js b/services/static-webserver/client/source/class/osparc/admin/AdminCenter.js index 36329e85b97..bd821027575 100644 --- a/services/static-webserver/client/source/class/osparc/admin/AdminCenter.js +++ b/services/static-webserver/client/source/class/osparc/admin/AdminCenter.js @@ -24,7 +24,7 @@ qx.Class.define("osparc.admin.AdminCenter", { const miniProfile = osparc.desktop.account.MyAccount.createMiniProfileView().set({ paddingRight: 10 }); - this.addWidgetOnTopOfTheTabs(miniProfile); + this.addWidgetToTabs(miniProfile); this.__addPricingPlansPage(); this.__addMaintenancePage(); diff --git a/services/static-webserver/client/source/class/osparc/admin/Announcements.js b/services/static-webserver/client/source/class/osparc/admin/Announcements.js index 84ca9f99079..85576321eeb 100644 --- a/services/static-webserver/client/source/class/osparc/admin/Announcements.js +++ b/services/static-webserver/client/source/class/osparc/admin/Announcements.js @@ -115,7 +115,7 @@ qx.Class.define("osparc.admin.Announcements", { } if (widgets.length === 0) { const msg = "Select at least one widget"; - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } const announcementData = { "id": osparc.utils.Utils.uuidV4(), diff --git a/services/static-webserver/client/source/class/osparc/announcement/Tracker.js b/services/static-webserver/client/source/class/osparc/announcement/Tracker.js index 368fbd6dd00..42c5790acff 100644 --- a/services/static-webserver/client/source/class/osparc/announcement/Tracker.js +++ b/services/static-webserver/client/source/class/osparc/announcement/Tracker.js @@ -26,7 +26,7 @@ qx.Class.define("osparc.announcement.Tracker", { members: { __checkInterval: null, - checkAnnouncements: async function() { + checkAnnouncements: function() { osparc.data.Resources.get("announcements") .then(announcements => { osparc.announcement.AnnouncementUIFactory.getInstance().setAnnouncementsData((announcements && announcements.length) ? announcements : []); diff --git a/services/static-webserver/client/source/class/osparc/auth/Data.js b/services/static-webserver/client/source/class/osparc/auth/Data.js index 2a4b27a9646..cda4da711d4 100644 --- a/services/static-webserver/client/source/class/osparc/auth/Data.js +++ b/services/static-webserver/client/source/class/osparc/auth/Data.js @@ -26,7 +26,7 @@ qx.Class.define("osparc.auth.Data", { properties: { /** - * Basic authentification with a token + * Basic authentication with a token */ auth: { init: null, diff --git a/services/static-webserver/client/source/class/osparc/auth/LoginPageS4L.js b/services/static-webserver/client/source/class/osparc/auth/LoginPageS4L.js index 1ff204745d9..bc53f7f9e67 100644 --- a/services/static-webserver/client/source/class/osparc/auth/LoginPageS4L.js +++ b/services/static-webserver/client/source/class/osparc/auth/LoginPageS4L.js @@ -34,9 +34,9 @@ qx.Class.define("osparc.auth.LoginPageS4L", { _getBackgroundImage: function() { let backgroundImage = ""; - const defaultBG = `url(${osparc.product.Utils.getProductBackgroundUrl("Sim4Life-head-default.png")}), url(${osparc.product.Utils.getProductBackgroundUrl("clouds_11.png")})`; - const liteBG = `url(${osparc.product.Utils.getProductBackgroundUrl("Sim4Life-head-lite.png")}), url(${osparc.product.Utils.getProductBackgroundUrl("clouds_11.png")})`; - const academyBG = `url(${osparc.product.Utils.getProductBackgroundUrl("Sim4Life-head-academy.png")}), url(${osparc.product.Utils.getProductBackgroundUrl("clouds_11.png")})`; + const defaultBG = `url(${osparc.product.Utils.getBackgroundUrl("Sim4Life-head-default.png")}), url(${osparc.product.Utils.getBackgroundUrl("clouds_11.png")})`; + const liteBG = `url(${osparc.product.Utils.getBackgroundUrl("Sim4Life-head-lite.png")}), url(${osparc.product.Utils.getBackgroundUrl("clouds_11.png")})`; + const academyBG = `url(${osparc.product.Utils.getBackgroundUrl("Sim4Life-head-academy.png")}), url(${osparc.product.Utils.getBackgroundUrl("clouds_11.png")})`; switch (osparc.product.Utils.getProductName()) { case "s4llite": diff --git a/services/static-webserver/client/source/class/osparc/auth/LoginWithDecorators.js b/services/static-webserver/client/source/class/osparc/auth/LoginWithDecorators.js index 2901520fe28..68e67bab29c 100644 --- a/services/static-webserver/client/source/class/osparc/auth/LoginWithDecorators.js +++ b/services/static-webserver/client/source/class/osparc/auth/LoginWithDecorators.js @@ -225,7 +225,7 @@ qx.Class.define("osparc.auth.LoginWithDecorators", { pages.setSelection([resetPassword]); } } else if (urlFragment.params && urlFragment.params.registered) { - osparc.FlashMessenger.getInstance().logAs(this.tr("Your account has been created.
You can now use your credentials to login.")); + osparc.FlashMessenger.logAs(this.tr("Your account has been created.
You can now use your credentials to login.")); } login.addListener("toRegister", () => { diff --git a/services/static-webserver/client/source/class/osparc/auth/Manager.js b/services/static-webserver/client/source/class/osparc/auth/Manager.js index fdd082cff96..85c738cc66b 100644 --- a/services/static-webserver/client/source/class/osparc/auth/Manager.js +++ b/services/static-webserver/client/source/class/osparc/auth/Manager.js @@ -176,13 +176,13 @@ qx.Class.define("osparc.auth.Manager", { } else { const resp = JSON.parse(xhr.responseText); if (resp.error == null) { - reject(this.tr("Login failed")); + reject(this.tr("Unsuccessful Login")); } else { reject(resp.error.message); } } }; - xhr.onerror = () => reject(this.tr("Login failed")); + xhr.onerror = () => reject(this.tr("Unsuccessful Login")); xhr.open("POST", url, true); xhr.setRequestHeader("Content-Type", "application/json"); xhr.send(JSON.stringify(params)); diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/Login2FAValidationCodeView.js b/services/static-webserver/client/source/class/osparc/auth/ui/Login2FAValidationCodeView.js index 27aaf2e6f61..ca2beb0757a 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/Login2FAValidationCodeView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/Login2FAValidationCodeView.js @@ -97,7 +97,7 @@ qx.Class.define("osparc.auth.ui.Login2FAValidationCodeView", { alignY: "middle" })); const resendCodeDesc = new qx.ui.basic.Label().set({ - value: this.tr("Didn't receive the code? Resend code") + value: this.tr("Didn't receive the code? Click to resend") }); resendLayout.add(resendCodeDesc); @@ -125,7 +125,7 @@ qx.Class.define("osparc.auth.ui.Login2FAValidationCodeView", { }); this.restartSMSButton(retryAfter); }) - .catch(err => osparc.FlashMessenger.logAs(err.message, "ERROR")) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => resendCodeSMSBtn.setFetching(false)); }, this); @@ -145,7 +145,7 @@ qx.Class.define("osparc.auth.ui.Login2FAValidationCodeView", { }); this.restartEmailButton(retryAfter); }) - .catch(err => osparc.FlashMessenger.logAs(err.message, "ERROR")) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => resendCodeEmailBtn.setFetching(false)); }, this); this.add(resendLayout); @@ -188,7 +188,7 @@ qx.Class.define("osparc.auth.ui.Login2FAValidationCodeView", { msg = String(msg) || this.tr("Invalid code"); validationCodeTF.setInvalidMessage(msg); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); }; if (this._form.validate()) { diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/LoginView.js b/services/static-webserver/client/source/class/osparc/auth/ui/LoginView.js index 5143bc8dabb..8610da065e0 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/LoginView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/LoginView.js @@ -235,7 +235,7 @@ qx.Class.define("osparc.auth.ui.LoginView", { const twoFactorAuthCbk = (nextStep, message, retryAfter) => { this.__loginBtn.setFetching(false); - osparc.FlashMessenger.getInstance().logAs(message, "INFO"); + osparc.FlashMessenger.logAs(message, "INFO"); this.fireDataEvent("to2FAValidationCode", { userEmail: email.getValue(), nextStep, @@ -251,7 +251,7 @@ qx.Class.define("osparc.auth.ui.LoginView", { const failFun = msg => { this.__loginBtn.setFetching(false); // TODO: can get field info from response here - msg = String(msg) || this.tr("Typed an invalid email or password"); + msg = String(msg) || this.tr("email or password don't look correct"); [email, pass].forEach(item => { item.set({ invalidMessage: msg, @@ -259,7 +259,7 @@ qx.Class.define("osparc.auth.ui.LoginView", { }); }); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); }; const manager = osparc.auth.Manager.getInstance(); diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/RegistrationView.js b/services/static-webserver/client/source/class/osparc/auth/ui/RegistrationView.js index 370f892b902..97749ea860e 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/RegistrationView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/RegistrationView.js @@ -74,10 +74,7 @@ qx.Class.define("osparc.auth.ui.RegistrationView", { }); } }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err)); } // validation @@ -137,10 +134,7 @@ qx.Class.define("osparc.auth.ui.RegistrationView", { this.fireDataEvent("done", log.message); osparc.FlashMessenger.getInstance().log(log); }) - .catch(err => { - const msg = err.message || this.tr("Cannot register user"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Cannot register user"))) .finally(() => submitButton.setFetching(false)); }, diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/RequestAccount.js b/services/static-webserver/client/source/class/osparc/auth/ui/RequestAccount.js index 49140f0630d..8ab05f16cd3 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/RequestAccount.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/RequestAccount.js @@ -408,13 +408,12 @@ qx.Class.define("osparc.auth.ui.RequestAccount", { }; osparc.data.Resources.fetch("auth", "postRequestAccount", params) .then(() => { - const msg = this.tr("The request is being processed, you will hear from us in the coming hours"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + const msg = this.tr("Your request is being processed. You will hear from us soon"); + osparc.FlashMessenger.logAs(msg, "INFO"); this.fireDataEvent("done"); }) .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.__restartCaptcha(); }); }, diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassRequestView.js b/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassRequestView.js index 89f8bafc40b..096ba9967a6 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassRequestView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassRequestView.js @@ -90,9 +90,8 @@ qx.Class.define("osparc.auth.ui.ResetPassRequestView", { osparc.FlashMessenger.getInstance().log(log); }; - const failFun = msg => { - msg = msg || this.tr("Could not request password reset"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const failFun = err => { + osparc.FlashMessenger.logError(err, this.tr("Could not request password reset")); }; manager.resetPasswordRequest(email.getValue(), successFun, failFun, this); diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassView.js b/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassView.js index 7d47d6b14f3..196a0364821 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/ResetPassView.js @@ -102,9 +102,8 @@ qx.Class.define("osparc.auth.ui.ResetPassView", { osparc.FlashMessenger.getInstance().log(log); }; - const failFun = msg => { - msg = msg || this.tr("Could not reset password"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const failFun = err => { + osparc.FlashMessenger.logError(err, this.tr("Could not reset password")); }; const manager = osparc.auth.Manager.getInstance(); diff --git a/services/static-webserver/client/source/class/osparc/auth/ui/VerifyPhoneNumberView.js b/services/static-webserver/client/source/class/osparc/auth/ui/VerifyPhoneNumberView.js index ea57b141db2..65c5996f1f8 100644 --- a/services/static-webserver/client/source/class/osparc/auth/ui/VerifyPhoneNumberView.js +++ b/services/static-webserver/client/source/class/osparc/auth/ui/VerifyPhoneNumberView.js @@ -61,7 +61,7 @@ qx.Class.define("osparc.auth.ui.VerifyPhoneNumberView", { this.add(verificationInfoTitle); const verificationInfoDesc = new qx.ui.basic.Label().set({ - value: this.tr("We will send you a text message to your mobile phone to authenticate you each time you log in."), + value: this.tr("A text message will be sent to your mobile phone for authentication each time you log in."), rich: true, wrap: true }); @@ -138,7 +138,7 @@ qx.Class.define("osparc.auth.ui.VerifyPhoneNumberView", { this.__enableEnterCommand(this.__validateCodeBtn); }) .catch(err => { - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.__verifyPhoneNumberBtn.setFetching(false); this.__itiInput.setEnabled(true); }); @@ -157,13 +157,13 @@ qx.Class.define("osparc.auth.ui.VerifyPhoneNumberView", { this.fireDataEvent("done", log.message); }; - const failFun = msg => { - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const failFun = err => { + osparc.FlashMessenger.logError(err); this.__validateCodeBtn.setFetching(false); // TODO: can get field info from response here - msg = String(msg) || this.tr("Invalid code"); + err = String(err) || this.tr("Invalid code"); this.__validateCodeField.set({ - invalidMessage: msg, + invalidMessage: err, valid: false }); }; @@ -185,7 +185,7 @@ qx.Class.define("osparc.auth.ui.VerifyPhoneNumberView", { retryAfter }); }) - .catch(err => osparc.FlashMessenger.logAs(err.message, "ERROR")) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.__sendViaEmail.setFetching(false)); }, diff --git a/services/static-webserver/client/source/class/osparc/dashboard/CardBase.js b/services/static-webserver/client/source/class/osparc/dashboard/CardBase.js index 6d74de34376..fc5c41c1999 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/CardBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/CardBase.js @@ -55,8 +55,9 @@ qx.Class.define("osparc.dashboard.CardBase", { MODE_APP: "@FontAwesome5Solid/desktop/13", NEW_ICON: "@FontAwesome5Solid/plus/", LOADING_ICON: "@FontAwesome5Solid/circle-notch/", - // Get the default thumbnail for each product else add the image and extension osparc.product.Utils.getProductThumbUrl(Thumbnail-01.png) - PRODUCT_ICON: osparc.product.Utils.getProductThumbUrl(), + PRODUCT_ICON: osparc.product.Utils.getIconUrl(), + // Get the default thumbnail for each product else add the image and extension osparc.product.Utils.getThumbnailUrl(Thumbnail-01.png) + PRODUCT_THUMBNAIL: osparc.product.Utils.getThumbnailUrl(), CARD_PRIORITY: { NEW: 0, @@ -65,6 +66,23 @@ qx.Class.define("osparc.dashboard.CardBase", { LOADER: 3 }, + ICON_SIZE: 32, + + createCardIcon: function() { + const iconSize = osparc.dashboard.CardBase.ICON_SIZE; + const icon = new osparc.ui.basic.Thumbnail(null, iconSize, iconSize).set({ + minHeight: iconSize, + minWidth: iconSize, + }); + icon.getChildControl("image").set({ + anonymous: true, + decorator: "rounded", + minWidth: iconSize, + minHeight: iconSize, + }); + return icon + }, + createTSRLayout: function() { const layout = new qx.ui.container.Composite(new qx.ui.layout.HBox(2).set({ alignY: "middle" @@ -269,6 +287,13 @@ qx.Class.define("osparc.dashboard.CardBase", { apply: "_applyIcon", }, + thumbnail: { + check: "String", + init: null, + nullable: true, + apply: "_applyThumbnail", + }, + resourceData: { check: "Object", nullable: false, @@ -348,11 +373,18 @@ qx.Class.define("osparc.dashboard.CardBase", { workbench: { check: "Object", nullable: true, - apply: "__applyWorkbench" + }, + + services: { + check: "Array", + init: true, + nullable: false, + apply: "__applyServices", + event: "changeServices", }, uiMode: { - check: ["workbench", "guided", "app"], + check: ["workbench", "guided", "app", "standalone"], // "guided" is no longer used nullable: true, apply: "__applyUiMode" }, @@ -453,6 +485,7 @@ qx.Class.define("osparc.dashboard.CardBase", { let owner = null; let workbench = null; let defaultHits = null; + let icon = null; switch (resourceData["resourceType"]) { case "study": uuid = resourceData.uuid ? resourceData.uuid : null; @@ -467,6 +500,7 @@ qx.Class.define("osparc.dashboard.CardBase", { case "service": uuid = resourceData.key ? resourceData.key : null; owner = resourceData.owner ? resourceData.owner : resourceData.contact; + icon = resourceData["icon"] || osparc.dashboard.CardBase.PRODUCT_ICON; defaultHits = 0; break; } @@ -481,7 +515,8 @@ qx.Class.define("osparc.dashboard.CardBase", { lastChangeDate: resourceData.lastChangeDate ? new Date(resourceData.lastChangeDate) : null, trashedAt: resourceData.trashedAt ? new Date(resourceData.trashedAt) : null, trashedBy: resourceData.trashedBy || null, - icon: resourceData.thumbnail || this.self().PRODUCT_ICON, + icon, + thumbnail: resourceData.thumbnail || this.self().PRODUCT_THUMBNAIL, state: resourceData.state ? resourceData.state : {}, classifiers: resourceData.classifiers && resourceData.classifiers ? resourceData.classifiers : [], quality: resourceData.quality ? resourceData.quality : null, @@ -489,6 +524,23 @@ qx.Class.define("osparc.dashboard.CardBase", { hits: resourceData.hits ? resourceData.hits : defaultHits, workbench }); + + if (resourceData["resourceType"] === "study" || resourceData["resourceType"] === "template") { + const params = { + url: { + studyId: this.getResourceData()["uuid"] + } + }; + osparc.data.Resources.fetch("studies", "getServices", params) + .then(resp => { + const services = resp["services"]; + resourceData["services"] = services; + this.setServices(services); + }); + + osparc.study.Utils.guessIcon(resourceData) + .then(iconSource => this.setIcon(iconSource)); + } }, __applyMultiSelectionMode: function(value) { @@ -500,22 +552,20 @@ qx.Class.define("osparc.dashboard.CardBase", { __evalSelectedButton: function() { if ( + this.hasChildControl("menu-selection-stack") && this.hasChildControl("menu-button") && this.hasChildControl("tick-selected") && this.hasChildControl("tick-unselected") ) { - const menuButton = this.getChildControl("menu-button"); - const tick = this.getChildControl("tick-selected"); - const untick = this.getChildControl("tick-unselected"); + const menuButtonStack = this.getChildControl("menu-selection-stack"); if (this.isResourceType("study") && this.isMultiSelectionMode()) { + const tick = this.getChildControl("tick-selected"); + const untick = this.getChildControl("tick-unselected"); const selected = this.getSelected(); - menuButton.setVisibility("excluded"); - tick.setVisibility(selected ? "visible" : "excluded"); - untick.setVisibility(selected ? "excluded" : "visible"); + menuButtonStack.setSelection(selected ? [tick] : [untick]); } else { - menuButton.setVisibility("visible"); - tick.setVisibility("excluded"); - untick.setVisibility("excluded"); + const menuButton = this.getChildControl("menu-button"); + menuButtonStack.setSelection([menuButton]); } } }, @@ -531,6 +581,10 @@ qx.Class.define("osparc.dashboard.CardBase", { throw new Error("Abstract method called!"); }, + _applyThumbnail: function(value, old) { + throw new Error("Abstract method called!"); + }, + _applyTitle: function(value, old) { throw new Error("Abstract method called!"); }, @@ -582,21 +636,16 @@ qx.Class.define("osparc.dashboard.CardBase", { }, __applyUiMode: function(uiMode) { - let source = null; - let toolTipText = null; switch (uiMode) { case "guided": - case "app": - source = osparc.dashboard.CardBase.MODE_APP; - toolTipText = this.tr("App mode"); + case "app": { + const uiModeIcon = this.getChildControl("workbench-mode"); + uiModeIcon.set({ + source: osparc.dashboard.CardBase.MODE_APP, + toolTipText: this.tr("App mode"), + }); break; - } - if (source) { - const uiModeIcon = this.getChildControl("workbench-mode"); - uiModeIcon.set({ - source, - toolTipText, - }); + } } }, @@ -607,39 +656,31 @@ qx.Class.define("osparc.dashboard.CardBase", { } }, - __applyWorkbench: function(workbench) { - if (workbench === null) { - // it is a service - return; - } - - if (this.isResourceType("study") || this.isResourceType("template")) { - this.setEmptyWorkbench(Object.keys(workbench).length === 0); - } + __applyServices: function(services) { + this.setEmptyWorkbench(services.length === 0); // Updatable study - if (osparc.study.Utils.isWorkbenchRetired(workbench)) { + if (osparc.study.Utils.anyServiceRetired(services)) { this.setUpdatable("retired"); - } else if (osparc.study.Utils.isWorkbenchDeprecated(workbench)) { + } else if (osparc.study.Utils.anyServiceDeprecated(services)) { this.setUpdatable("deprecated"); - } else { - const updatable = osparc.study.Utils.isWorkbenchUpdatable(workbench) - if (updatable) { - this.setUpdatable("updatable"); - } + } else if (osparc.study.Utils.anyServiceUpdatable(services)) { + this.setUpdatable("updatable"); } // Block card - const unaccessibleServices = osparc.study.Utils.getInaccessibleServices(workbench) + const unaccessibleServices = osparc.study.Utils.getCantExecuteServices(services); if (unaccessibleServices.length) { this.setBlocked("UNKNOWN_SERVICES"); - let image = "@FontAwesome5Solid/ban/"; - let toolTipText = this.tr("Service info missing"); + const image = "@FontAwesome5Solid/ban/"; + let toolTipText = this.tr("Unaccessible service(s):"); unaccessibleServices.forEach(unSrv => { - toolTipText += "
" + unSrv.key + ":" + unSrv.version; + toolTipText += "
" + unSrv.key + ":" + osparc.service.Utils.extractVersionDisplay(unSrv.release); }); this.__showBlockedCard(image, toolTipText); } + + this.evaluateMenuButtons(); }, __applyEmptyWorkbench: function(isEmpty) { @@ -726,7 +767,7 @@ qx.Class.define("osparc.dashboard.CardBase", { break; case "FAILED": iconSource = "@FontAwesome5Solid/exclamation/10"; - toolTipText = this.tr("Ran with error"); + toolTipText = this.tr("Unsuccessful Run"); borderColor = "error"; break; case "UNKNOWN": @@ -882,6 +923,10 @@ qx.Class.define("osparc.dashboard.CardBase", { if (duplicateButton) { duplicateButton.setEnabled(osparc.study.Utils.canBeDuplicated(resourceData)); } + const convertToPipelineButton = menuButtons.find(menuBtn => "convertToPipelineButton" in menuBtn); + if (convertToPipelineButton) { + convertToPipelineButton.setEnabled(osparc.study.Utils.canBeDuplicated(resourceData)); + } const exportCMISButton = menuButtons.find(menuBtn => "exportCMISButton" in menuBtn); if (exportCMISButton) { exportCMISButton.setEnabled(osparc.study.Utils.canBeExported(resourceData)); @@ -922,9 +967,14 @@ qx.Class.define("osparc.dashboard.CardBase", { } }, - __openMoreOptions: function() { + __openResourceDetails: function(openWindowCB) { const resourceData = this.getResourceData(); const resourceDetails = new osparc.dashboard.ResourceDetails(resourceData); + resourceDetails.addListenerOnce("pagesAdded", () => { + if (openWindowCB in resourceDetails) { + resourceDetails[openWindowCB](); + } + }) const win = osparc.dashboard.ResourceDetails.popUpInWindow(resourceDetails); [ "updateStudy", @@ -951,33 +1001,28 @@ qx.Class.define("osparc.dashboard.CardBase", { }, openData: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openData(); + const resourceData = this.getResourceData(); + osparc.widget.StudyDataManager.popUpInWindow(resourceData["uuid"]); }, openBilling: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openBillingSettings(); + this.__openResourceDetails("openBillingSettings"); }, openAccessRights: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openAccessRights(); + this.__openResourceDetails("openAccessRights"); }, openTags: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openTags(); + this.__openResourceDetails("openTags"); }, __openQualityEditor: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openQuality(); + this.__openResourceDetails("openQuality"); }, __openUpdateServices: function() { - const moreOpts = this.__openMoreOptions(); - moreOpts.openUpdateServices(); + this.__openResourceDetails("openUpdateServices"); }, _getEmptyWorkbenchIcon: function() { diff --git a/services/static-webserver/client/source/class/osparc/dashboard/Dashboard.js b/services/static-webserver/client/source/class/osparc/dashboard/Dashboard.js index b40fd6fc1be..ee1c1bbbe76 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/Dashboard.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/Dashboard.js @@ -38,9 +38,8 @@ qx.Class.define("osparc.dashboard.Dashboard", { construct: function() { this.base(arguments); - this.getChildControl("bar").set({ - visibility: "excluded", - }); + osparc.utils.Utils.setIdToWidget(this.getChildControl("bar"), "dashboardTabs"); + osparc.utils.Utils.setIdToWidget(this, "dashboard"); this.set({ contentPadding: this.self().PADDING, @@ -48,7 +47,8 @@ qx.Class.define("osparc.dashboard.Dashboard", { barPosition: "top" }); - osparc.wrapper.Plotly.getInstance().init(); + // osparc.wrapper.Plotly.getInstance().init(); + // osparc.wrapper.Three.getInstance().init(); osparc.wrapper.Svg.getInstance().init(); osparc.wrapper.JsonDiffPatch.getInstance().init(); osparc.wrapper.JsonTreeViewer.getInstance().init(); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/DataBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/DataBrowser.js index f8ffd3d73d0..2305feaa8a4 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/DataBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/DataBrowser.js @@ -60,7 +60,7 @@ qx.Class.define("osparc.dashboard.DataBrowser", { this.addListener("appear", () => { const treeFolderView = this.getChildControl("tree-folder-view"); - treeFolderView.getChildControl("folder-tree").populateTree(); + treeFolderView.getChildControl("folder-tree").populateLocations(); treeFolderView.getChildControl("folder-viewer").setFolder(treeFolderView.getChildControl("folder-tree").getModel()); }, this); }, @@ -84,7 +84,7 @@ qx.Class.define("osparc.dashboard.DataBrowser", { const foldersTree = treeFolderView.getChildControl("folder-tree"); foldersTree.resetCache(); - foldersTree.populateTree(); + foldersTree.populateLocations(); const folderViewer = treeFolderView.getChildControl("folder-viewer"); folderViewer.resetFolder(); @@ -94,7 +94,7 @@ qx.Class.define("osparc.dashboard.DataBrowser", { // After deleting a file, try to keep the user in the same folder. // If the folder doesn't longer exist, open the closest available parent - const path = fileMetadata["fileUuid"].split("/"); + const pathParts = fileMetadata["fileUuid"].split("/"); const treeFolderView = this.getChildControl("tree-folder-view"); const foldersTree = treeFolderView.getChildControl("folder-tree"); @@ -102,18 +102,18 @@ qx.Class.define("osparc.dashboard.DataBrowser", { const openSameFolder = () => { // drop last, which is the file - path.pop(); - treeFolderView.openPath(path); + pathParts.pop(); + treeFolderView.openPath(pathParts); }; folderViewer.resetFolder(); const locationId = fileMetadata["locationId"]; - const datasetId = path[0]; + const path = pathParts[0]; foldersTree.resetCache(); - foldersTree.populateTree() + foldersTree.populateLocations() .then(datasetPromises => { Promise.all(datasetPromises) - .then(() => foldersTree.requestDatasetFiles(locationId, datasetId)) + .then(() => foldersTree.requestPathItems(locationId, path)) .then(() => openSameFolder()); }) .catch(err => console.error(err)); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/DragWidget.js b/services/static-webserver/client/source/class/osparc/dashboard/DragWidget.js index 4685f93caaf..9fc43ac7304 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/DragWidget.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/DragWidget.js @@ -71,7 +71,7 @@ qx.Class.define("osparc.dashboard.DragWidget", { __dropAllowed: function(allowed) { this.getChildControl("allowed-icon").set({ source: allowed ? "@FontAwesome5Solid/check/14" : "@FontAwesome5Solid/times/14", - textColor: allowed ? "default-button-text" : "danger-red", + textColor: allowed ? "white" : "danger-red", }); }, diff --git a/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js b/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js index 18dad5fc65d..dfd0b2453a3 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/FolderButtonItem.js @@ -253,7 +253,7 @@ qx.Class.define("osparc.dashboard.FolderButtonItem", { const dateBy = this.getChildControl("date-by"); dateBy.set({ date: value, - toolTipText: this.tr("Moved to the bin"), + toolTipText: this.tr("Deleted"), }); } }, @@ -269,9 +269,10 @@ qx.Class.define("osparc.dashboard.FolderButtonItem", { const menuButton = this.getChildControl("menu-button"); menuButton.setVisibility("visible"); - const menu = new qx.ui.menu.Menu(); - menu.setPosition("bottom-right"); - osparc.utils.Utils.prettifyMenu(menu); + const menu = new qx.ui.menu.Menu().set({ + appearance: "menu-wider", + position: "bottom-right", + }); const studyBrowserContext = osparc.store.Store.getInstance().getStudyBrowserContext(); if ( @@ -302,7 +303,7 @@ qx.Class.define("osparc.dashboard.FolderButtonItem", { menu.addSeparator(); - const trashButton = new qx.ui.menu.Button(this.tr("Move to Bin"), "@FontAwesome5Solid/trash/12"); + const trashButton = new qx.ui.menu.Button(this.tr("Delete"), "@FontAwesome5Solid/trash/12"); trashButton.addListener("execute", () => this.fireDataEvent("trashFolderRequested", this.getFolderId()), this); menu.add(trashButton); } else if (studyBrowserContext === "trash") { diff --git a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js index ebda818baf0..34796bcec18 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js @@ -44,11 +44,10 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { ITEM_WIDTH: 190, ITEM_HEIGHT: 220, PADDING: 10, + TITLE_PADDING: 6, SPACING_IN: 5, SPACING: 15, - // TITLE_MAX_HEIGHT: 34, // two lines in Roboto - TITLE_MAX_HEIGHT: 40, // two lines in Manrope - ICON_SIZE: 50, + THUMBNAIL_SIZE: 50, POS: { TITLE: { row: 0, @@ -117,7 +116,6 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { grid.setSpacing(this.self().SPACING_IN); grid.setRowFlex(2, 1); grid.setColumnFlex(0, 1); - grid.setRowMaxHeight(0, this.self().TITLE_MAX_HEIGHT); control = new qx.ui.container.Composite().set({ maxWidth: this.self().ITEM_WIDTH, @@ -138,18 +136,26 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { }); break; } - case "header": - control = new qx.ui.container.Composite(new qx.ui.layout.VBox(5)).set({ - anonymous: true, - allowGrowX: true, - allowShrinkX: false, - alignY: "middle", - padding: this.self().PADDING + case "header": { + const hGrid = new qx.ui.layout.Grid().set({ + spacing: 6, }); - control.set({ - backgroundColor: "background-card-overlay" + hGrid.setRowFlex(0, 1); + hGrid.setColumnFlex(1, 1); + hGrid.setColumnAlign(0, "right", "middle"); + hGrid.setColumnAlign(1, "left", "middle"); + hGrid.setColumnAlign(2, "center", "middle"); + control = new qx.ui.container.Composite().set({ + backgroundColor: "background-card-overlay", + paddingBottom: 6, + paddingRight: 4, + maxWidth: this.self().ITEM_WIDTH, + minHeight: 32 + 6, + maxHeight: this.self().ITEM_HEIGHT }); + control.setLayout(hGrid); break; + } case "body": control = new qx.ui.container.Composite(new qx.ui.layout.VBox(5)).set({ decorator: "main", @@ -163,58 +169,48 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { }); break; case "footer": { - const fgrid = new qx.ui.layout.Grid(); - fgrid.setSpacing(2); - fgrid.setColumnFlex(0, 1); + const fGrid = new qx.ui.layout.Grid(); + fGrid.setSpacing(2); + fGrid.setColumnFlex(0, 1); control = new qx.ui.container.Composite().set({ backgroundColor: "background-card-overlay", padding: this.self().PADDING - 2, maxWidth: this.self().ITEM_WIDTH, maxHeight: this.self().ITEM_HEIGHT }); - control.setLayout(fgrid); + control.setLayout(fGrid); break; } - case "title-row": - control = new qx.ui.container.Composite(new qx.ui.layout.VBox(6)).set({ - anonymous: true, - maxWidth: this.self().ITEM_WIDTH - 2*this.self().PADDING - }); + case "icon": { + control = osparc.dashboard.CardBase.createCardIcon(); layout = this.getChildControl("header"); - layout.addAt(control, 1, { - flex: 1 + layout.add(control, { + column: 0, + row: 0, }); break; + } case "title": control = new qx.ui.basic.Label().set({ textColor: "contrasted-text-light", font: "text-14", - maxWidth: this.self().ITEM_WIDTH - 2*this.self().PADDING, - maxHeight: this.self().TITLE_MAX_HEIGHT - }); - layout = this.getChildControl("title-row"); - layout.addAt(control, 0, { - flex: 1 }); - break; - case "subtitle": - control = new qx.ui.container.Composite(new qx.ui.layout.HBox(6)).set({ - anonymous: true, - height: 20 - }); - layout = this.getChildControl("title-row"); - layout.addAt(control, 1, { - flex: 1 + layout = this.getChildControl("header"); + layout.add(control, { + column: 1, + row: 0, }); break; case "subtitle-icon": { control = new qx.ui.basic.Image().set({ - alignY: "middle", allowGrowX: false, - allowShrinkX: false + allowShrinkX: false, + }); + layout = this.getChildControl("header"); + layout.add(control, { + column: 0, + row: 1, }); - const subtitleLayout = this.getChildControl("subtitle"); - subtitleLayout.addAt(control, 0); break; } case "subtitle-text": { @@ -228,13 +224,14 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { font: "text-12", allowGrowY: false }); - const subtitleLayout = this.getChildControl("subtitle"); - subtitleLayout.addAt(control, 1, { - flex: 1 + layout = this.getChildControl("header"); + layout.add(control, { + column: 1, + row: 1, }); break; } - case "icon": { + case "thumbnail": { layout = this.getChildControl("body"); const maxWidth = this.self().ITEM_WIDTH; control = new osparc.ui.basic.Thumbnail(null, maxWidth, 124); @@ -259,14 +256,15 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { case "project-status": control = new qx.ui.basic.Image().set({ alignY: "middle", - textColor: "status_icon", + textColor: "text", height: 13, width: 13, margin: [0, 1] }); - layout = this.getChildControl("subtitle"); - layout.set({ - visibility: "visible" + layout = this.getChildControl("header"); + layout.add(control, { + column: 2, + row: 1, }); layout.addAt(control, 2); break; @@ -276,27 +274,39 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { // overridden _applyIcon: function(value, old) { - if (value.includes("@FontAwesome5Solid/")) { - value += this.self().ICON_SIZE; + if (value) { const image = this.getChildControl("icon").getChildControl("image"); image.set({ - source: value + source: value, }); + osparc.utils.Utils.setAltToImage(image, "card-icon"); + } + }, - [ - "appear", - "loaded" - ].forEach(eventName => { - image.addListener(eventName, () => this.__fitIconHeight(), this); + // overridden + _applyThumbnail: function(value, old) { + if (qx.util.ResourceManager.getInstance().isFontUri(value)) { + value += this.self().THUMBNAIL_SIZE; + this.getChildControl("thumbnail").set({ + source: value, }); } else { - this.getContentElement().setStyles({ - "background-image": `url(${value})`, - "background-repeat": "no-repeat", - "background-size": "cover", // auto width, 85% height - "background-position": "center center", - "background-origin": "border-box" - }); + let source = osparc.product.Utils.getThumbnailUrl(); + osparc.utils.Utils.checkImageExists(value) + .then(exists => { + if (exists) { + source = value; + } + }) + .finally(() => { + this.getContentElement().setStyles({ + "background-image": `url(${source})`, + "background-repeat": "no-repeat", + "background-size": "cover", // auto width, 85% height + "background-position": "center center", + "background-origin": "border-box" + }); + }); } }, @@ -312,8 +322,8 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { return; }, - __fitIconHeight: function() { - const iconLayout = this.getChildControl("icon"); + __fitThumbnailHeight: function() { + const thumbnailLayout = this.getChildControl("thumbnail"); let maxHeight = this.getHeight() - this.getPaddingTop() - this.getPaddingBottom() - 5; const checkThis = [ "title", @@ -333,17 +343,9 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { }); // maxHeight -= 4; // for Roboto maxHeight -= 18; // for Manrope - iconLayout.getChildControl("image").setMaxHeight(maxHeight); - iconLayout.setMaxHeight(maxHeight); - iconLayout.recheckSize(); - }, - - replaceIcon: function(newIcon) { - const plusIcon = this.getChildControl("icon"); - plusIcon.exclude(); - - const bodyLayout = this.getChildControl("body"); - bodyLayout.add(newIcon, {flex: 1}); + thumbnailLayout.getChildControl("image").setMaxHeight(maxHeight); + thumbnailLayout.setMaxHeight(maxHeight); + thumbnailLayout.recheckSize(); }, /** diff --git a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonItem.js b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonItem.js index 22779e7831b..ad0bf2ec3ae 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonItem.js @@ -91,9 +91,20 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { layout = this.getChildControl("main-layout"); layout.add(control, osparc.dashboard.GridButtonBase.POS.TAGS); break; - case "menu-button": + case "menu-selection-stack": + control = new qx.ui.container.Stack(); + control.set({ + alignX: "center", + alignY: "middle" + }); + this.getChildControl("header").add(control, { + column: 2, + row: 0, + }); + break; + case "menu-button": { this.getChildControl("title").set({ - maxWidth: osparc.dashboard.GridButtonBase.ITEM_WIDTH - 2*osparc.dashboard.GridButtonBase.PADDING - this.self().MENU_BTN_DIMENSIONS + maxWidth: osparc.dashboard.GridButtonBase.ITEM_WIDTH - osparc.dashboard.CardBase.ICON_SIZE - this.self().MENU_BTN_DIMENSIONS - 2, }); control = new qx.ui.form.MenuButton().set({ appearance: "form-button-outlined", @@ -110,12 +121,11 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { "border-radius": `${this.self().MENU_BTN_DIMENSIONS / 2}px` }); osparc.utils.Utils.setIdToWidget(control, "studyItemMenuButton"); - this._add(control, { - top: 8, - right: 8 - }); + const menuSelectionStack = this.getChildControl("menu-selection-stack"); + menuSelectionStack.addAt(control, 0); break; - case "tick-unselected": + } + case "tick-unselected": { control = new qx.ui.basic.Atom().set({ appearance: "form-button-outlined", width: this.self().MENU_BTN_DIMENSIONS, @@ -126,12 +136,11 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { control.getContentElement().setStyles({ "border-radius": `${this.self().MENU_BTN_DIMENSIONS / 2}px` }); - this._add(control, { - top: 8, - right: 8 - }); + const menuSelectionStack = this.getChildControl("menu-selection-stack"); + menuSelectionStack.addAt(control, 1); break; - case "tick-selected": + } + case "tick-selected": { control = new qx.ui.basic.Image().set({ appearance: "form-button", width: this.self().MENU_BTN_DIMENSIONS, @@ -146,11 +155,10 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { control.getContentElement().setStyles({ "border-radius": `${this.self().MENU_BTN_DIMENSIONS / 2}px` }); - this._add(control, { - top: 8, - right: 8 - }); + const menuSelectionStack = this.getChildControl("menu-selection-stack"); + menuSelectionStack.addAt(control, 2); break; + } case "lock-status": control = new osparc.ui.basic.Thumbnail(); this._add(control, { @@ -194,7 +202,7 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { const dateBy = this.getChildControl("date-by"); dateBy.set({ date: value, - toolTipText: this.tr("Moved to the bin"), + toolTipText: this.tr("Deleted"), }); } } @@ -244,10 +252,12 @@ qx.Class.define("osparc.dashboard.GridButtonItem", { _applyTags: function(tags) { if (osparc.data.Permissions.getInstance().canDo("study.tag")) { + const maxTags = 2; const tagsContainer = this.getChildControl("tags"); tagsContainer.setVisibility(tags.length ? "visible" : "excluded"); tagsContainer.removeAll(); - tags.forEach(tag => { + for (let i=0; i<=tags.length && i
/g, " "); const menuButton = new qx.ui.menu.Button().set({ icon: icon || null, label: title, @@ -62,7 +111,7 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", { }); if (infoText) { infoText = osparc.utils.Utils.replaceTokens( - title, + infoText, "replace_me_product_name", osparc.store.StaticInfo.getInstance().getDisplayName() ); @@ -112,27 +161,26 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", { }, __addNewStudyItems: async function() { - await Promise.all([ - osparc.store.Products.getInstance().getNewStudyConfig(), - osparc.data.Resources.get("templates") - ]).then(values => { - const newStudiesData = values[0]; - const templates = values[1]; - if (newStudiesData["categories"]) { - this.__addCategories(newStudiesData["categories"]); - } - newStudiesData["resources"].forEach(newStudyData => { - if (newStudyData["showDisabled"]) { - this.__addDisabledButton(newStudyData); - } else if (newStudyData["resourceType"] === "study") { - this.__addEmptyStudyButton(newStudyData); - } else if (newStudyData["resourceType"] === "template") { - this.__addFromTemplateButton(newStudyData, templates); - } else if (newStudyData["resourceType"] === "service") { - this.__addFromServiceButton(newStudyData); - } - }); - }); + const plusButtonConfig = osparc.store.Products.getInstance().getPlusButtonUiConfig(); + if (plusButtonConfig) { + await osparc.data.Resources.get("templates") + .then(templates => { + if (plusButtonConfig["categories"]) { + this.__addCategories(plusButtonConfig["categories"]); + } + plusButtonConfig["resources"].forEach(buttonConfig => { + if (buttonConfig["showDisabled"]) { + this.__addDisabledButton(buttonConfig); + } else if (buttonConfig["resourceType"] === "study") { + this.__addEmptyStudyButton(buttonConfig); + } else if (buttonConfig["resourceType"] === "template") { + this.__addFromTemplateButton(buttonConfig, templates); + } else if (buttonConfig["resourceType"] === "service") { + this.__addFromServiceButton(buttonConfig); + } + }); + }); + } }, __getLastIdxFromCategory: function(categoryId) { @@ -160,12 +208,10 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", { __addIcon: function(menuButton, resourceInfo, resourceMetadata) { let source = null; - if (resourceInfo && "icon" in resourceInfo) { - // first the one set in the new_studies + if (resourceInfo && resourceInfo["icon"]) { source = resourceInfo["icon"]; - } else if (resourceMetadata && "thumbnail" in resourceMetadata) { - // second the one from the resource - source = resourceMetadata["thumbnail"]; + } else { + source = osparc.utils.Utils.getIconFromResource(resourceMetadata); } if (source) { @@ -195,92 +241,118 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", { } }, - __addDisabledButton: function(newStudyData) { - const menuButton = this.self().createMenuButton(null, newStudyData.title, newStudyData.reason); - osparc.utils.Utils.setIdToWidget(menuButton, newStudyData.idToWidget); + __addDisabledButton: function(buttonConfig) { + const menuButton = this.self().createMenuButton(null, buttonConfig["title"], buttonConfig["reason"]); + osparc.utils.Utils.setIdToWidget(menuButton, buttonConfig["idToWidget"]); menuButton.setEnabled(false); - this.__addIcon(menuButton, newStudyData); - this.__addFromResourceButton(menuButton, newStudyData.category); + this.__addIcon(menuButton, buttonConfig); + this.__addFromResourceButton(menuButton, buttonConfig["category"]); }, - __addEmptyStudyButton: function(newStudyData) { - const menuButton = this.self().createMenuButton(null, newStudyData.title); - osparc.utils.Utils.setIdToWidget(menuButton, newStudyData.idToWidget); + __addEmptyStudyButton: function(buttonConfig) { + const menuButton = this.self().createMenuButton(null, buttonConfig["title"]); + osparc.utils.Utils.setIdToWidget(menuButton, buttonConfig["idToWidget"]); menuButton.addListener("tap", () => { this.fireDataEvent("newEmptyStudyClicked", { - newStudyLabel: newStudyData.newStudyLabel, + newStudyLabel: buttonConfig["newStudyLabel"], }); }); - this.__addIcon(menuButton, newStudyData); - this.__addFromResourceButton(menuButton, newStudyData.category); + this.__addIcon(menuButton, buttonConfig); + this.__addFromResourceButton(menuButton, buttonConfig["category"]); }, - __addFromTemplateButton: function(newStudyData, templates) { - const menuButton = this.self().createMenuButton(null, newStudyData.title); - osparc.utils.Utils.setIdToWidget(menuButton, newStudyData.idToWidget); + __addFromTemplateButton: function(buttonConfig, templates) { + const menuButton = this.self().createMenuButton(null, buttonConfig["title"]); + osparc.utils.Utils.setIdToWidget(menuButton, buttonConfig["idToWidget"]); // disable it until found in templates store menuButton.setEnabled(false); - let templateMetadata = templates.find(t => t.name === newStudyData.expectedTemplateLabel); + let templateMetadata = templates.find(t => t.name === buttonConfig["expectedTemplateLabel"]); if (templateMetadata) { menuButton.setEnabled(true); menuButton.addListener("tap", () => { this.fireDataEvent("newStudyFromTemplateClicked", { templateData: templateMetadata, - newStudyLabel: newStudyData.newStudyLabel, + newStudyLabel: buttonConfig["newStudyLabel"], }); }); - this.__addIcon(menuButton, newStudyData, templateMetadata); - this.__addFromResourceButton(menuButton, newStudyData.category); + this.__addIcon(menuButton, buttonConfig, templateMetadata); + this.__addFromResourceButton(menuButton, buttonConfig["category"]); } }, - __addFromServiceButton: function(newStudyData) { - const menuButton = this.self().createMenuButton(null, newStudyData.title); - osparc.utils.Utils.setIdToWidget(menuButton, newStudyData.idToWidget); - // disable it until found in services store - menuButton.setEnabled(false); + __addFromServiceButton: function(buttonConfig) { + const addListenerToButton = (menuButton, latestMetadata) => { + menuButton.addListener("tap", () => { + this.fireDataEvent("newStudyFromServiceClicked", { + serviceMetadata: latestMetadata, + newStudyLabel: buttonConfig["newStudyLabel"], + }); + }); - const key = newStudyData.expectedKey; - // Include deprecated versions, they should all be updatable to a non deprecated version - const versions = osparc.service.Utils.getVersions(key, false); - if (versions.length && newStudyData) { - // scale to latest compatible - const latestVersion = versions[0]; - const latestCompatible = osparc.service.Utils.getLatestCompatible(key, latestVersion); - osparc.store.Services.getService(latestCompatible["key"], latestCompatible["version"]) - .then(latestMetadata => { - // make sure this one is not deprecated - if (osparc.service.Utils.isDeprecated(latestMetadata)) { - return; - } - menuButton.setEnabled(true); - menuButton.addListener("tap", () => { - this.fireDataEvent("newStudyFromServiceClicked", { - serviceMetadata: latestMetadata, - newStudyLabel: newStudyData.newStudyLabel, - }); + const cb = e => { + this.hide(); + // so that is not consumed by the menu button itself + e.stopPropagation(); + latestMetadata["resourceType"] = "service"; + const resourceDetails = new osparc.dashboard.ResourceDetails(latestMetadata); + const win = osparc.dashboard.ResourceDetails.popUpInWindow(resourceDetails); + resourceDetails.addListener("openService", ev => { + win.close(); + const openServiceData = ev.getData(); + this.fireDataEvent("newStudyFromServiceClicked", { + serviceMetadata: openServiceData, + newStudyLabel: buttonConfig["newStudyLabel"], }); - - const cb = e => { - this.hide(); - // so that is not consumed by the menu button itself - e.stopPropagation(); - latestMetadata["resourceType"] = "service"; - const resourceDetails = new osparc.dashboard.ResourceDetails(latestMetadata); - osparc.dashboard.ResourceDetails.popUpInWindow(resourceDetails); + }); + } + const infoButton = new osparc.ui.basic.IconButton(osparc.ui.hint.InfoHint.INFO_ICON + "/16", cb); + // where the shortcut is supposed to go + // eslint-disable-next-line no-underscore-dangle + menuButton._add(infoButton, {column: 2}); + }; + + if ("expectedKey" in buttonConfig) { + const menuButton = this.self().createMenuButton(null, buttonConfig["title"]); + osparc.utils.Utils.setIdToWidget(menuButton, buttonConfig["idToWidget"]); + // disable it until found in services store + menuButton.setEnabled(false); + + const key = buttonConfig["expectedKey"]; + const latestMetadata = osparc.store.Services.getLatest(key); + if (!latestMetadata) { + return; + } + menuButton.setEnabled(true); + this.__addIcon(menuButton, buttonConfig, latestMetadata); + this.__addFromResourceButton(menuButton, buttonConfig["category"]); + addListenerToButton(menuButton, latestMetadata); + } else if ("myMostUsed" in buttonConfig) { + const excludeFrontend = true; + const excludeDeprecated = true + osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated) + .then(servicesList => { + osparc.service.Utils.sortObjectsBasedOn(servicesList, { + "sort": "hits", + "order": "down" + }); + for (let i=0; i
" + this.tr("It will be permanently deleted after ") + trashDays + " days."; const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ - caption: this.tr("Move to Bin"), - confirmText: this.tr("Move to Bin"), + caption: this.tr("Delete"), + confirmText: this.tr("Delete"), confirmAction: "warning", }); confirmationWin.center(); @@ -647,13 +586,10 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { .then(() => { this.__reloadFolders(); const msg = this.tr("Successfully restored"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); this._resourceFilter.evaluateTrashEmpty(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(err, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) }, _deleteFolderRequested: function(folderId) { @@ -661,7 +597,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { .then(() => { this.__reloadFolders(); const msg = this.tr("Successfully deleted"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); this._resourceFilter.evaluateTrashEmpty(); }) .catch(err => console.error(err)); @@ -939,36 +875,83 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { if (!osparc.product.Utils.hasNewPlusButton()) { switch (osparc.product.Utils.getProductName()) { - case "osparc": - this.__addEmptyStudyPlusButton(); - break; case "tis": case "tiplite": + // this one is different since it groups all new buttons in one new button this.__addTIPPlusButton(); break; - case "s4l": - case "s4lacad": - case "s4llite": - this.__addPlusButtonsFromServices(); + default: + this.__addPlusButtons(); break; } } }, - __addEmptyStudyPlusButton: function() { + __addPlusButtons: function() { + const plusButtonConfig = osparc.store.Products.getInstance().getNewStudiesUiConfig(); + if (plusButtonConfig) { + plusButtonConfig["resources"].forEach(newStudyData => { + if (newStudyData["resourceType"] === "study") { + this.__addEmptyStudyPlusButton(newStudyData); + } else if (newStudyData["resourceType"] === "service") { + this.__addNewStudyFromServiceButton(newStudyData); + } + }); + } + }, + + __addEmptyStudyPlusButton: function(newStudyData) { const mode = this._resourcesContainer.getMode(); - const title = this.tr("Empty") + " " + osparc.product.Utils.getStudyAlias({ + const defTitle = this.tr("Empty") + " " + osparc.product.Utils.getStudyAlias({ firstUpperCase: true - }) - const desc = this.tr("Start with an empty study"); + }); + const title = newStudyData["title"] || defTitle; + const desc = newStudyData["description"] || this.tr("Start with an empty study"); const newEmptyStudyBtn = (mode === "grid") ? new osparc.dashboard.GridButtonNew(title, desc) : new osparc.dashboard.ListButtonNew(title, desc); newEmptyStudyBtn.setCardKey("new-study"); newEmptyStudyBtn.subscribeToFilterGroup("searchBarFilter"); - osparc.utils.Utils.setIdToWidget(newEmptyStudyBtn, "emptyStudyBtn"); - newEmptyStudyBtn.addListener("tap", () => this.__newEmptyStudyBtnClicked("New Study")); + osparc.utils.Utils.setIdToWidget(newEmptyStudyBtn, newStudyData["idToWidget"]); + newEmptyStudyBtn.addListener("tap", () => this.__newEmptyStudyBtnClicked(newStudyData["newStudyLabel"])); this._resourcesContainer.addNonResourceCard(newEmptyStudyBtn); }, + __addNewStudyFromServiceButton: function(newStudyData) { + if ("expectedKey" in newStudyData) { + const key = newStudyData["expectedKey"]; + const latestMetadata = osparc.store.Services.getLatest(key); + if (!latestMetadata) { + return; + } + const title = newStudyData.title + " " + osparc.service.Utils.extractVersionDisplay(latestMetadata); + const desc = newStudyData.description; + const mode = this._resourcesContainer.getMode(); + const newStudyFromServiceButton = (mode === "grid") ? new osparc.dashboard.GridButtonNew(title, desc) : new osparc.dashboard.ListButtonNew(title, desc); + newStudyFromServiceButton.setCardKey("new-"+key); + if (newStudyData["idToWidget"]) { + osparc.utils.Utils.setIdToWidget(newStudyFromServiceButton, newStudyData["idToWidget"]); + } + newStudyFromServiceButton.addListener("tap", () => this.__newStudyFromServiceBtnClicked(latestMetadata["key"], latestMetadata["version"], newStudyData.newStudyLabel)); + this._resourcesContainer.addNonResourceCard(newStudyFromServiceButton); + } else if ("myMostUsed" in newStudyData) { + const excludeFrontend = true; + const excludeDeprecated = true + osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated) + .then(servicesList => { + osparc.service.Utils.sortObjectsBasedOn(servicesList, { + "sort": "hits", + "order": "down" + }); + for (let i=0; i
" + (studyNames.length > 1 ? "They" : "It") + this.tr(` will be permanently deleted after ${trashDays} days.`); const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ - caption: this.tr("Move to Bin"), - confirmText: this.tr("Move to Bin"), + caption: this.tr("Delete"), + confirmText: this.tr("Delete"), confirmAction: "warning", }); osparc.utils.Utils.setIdToWidget(confirmationWin.getConfirmButton(), "confirmDeleteStudyBtn"); @@ -2207,45 +2114,26 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { return confirmationWin; }, - __createConfirmEmptyTrashWindow: function() { - const msg = this.tr("Items in the bin will be permanently deleted"); - const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ - caption: this.tr("Delete"), - confirmText: this.tr("Delete permanently"), - confirmAction: "delete" - }); - return confirmationWin; - }, - // TASKS // - __tasksReceived: function(tasks) { - tasks.forEach(taskData => this._taskDataReceived(taskData)); - }, - - _taskDataReceived: function(taskData) { - // a bit hacky - if (taskData["task_id"].includes("from_study") && !taskData["task_id"].includes("as_template")) { - const interval = 1000; - const pollTasks = osparc.data.PollTasks.getInstance(); - const task = pollTasks.addTask(taskData, interval); - if (task === null) { - return; - } - // ask backend for studyData? + __tasksToCards: function() { + const tasks = osparc.store.PollTasks.getInstance().getDuplicateStudyTasks(); + tasks.forEach(task => { const studyName = ""; this.__taskDuplicateReceived(task, studyName); - } + }); }, __taskDuplicateReceived: function(task, studyName) { const duplicateTaskUI = new osparc.task.Duplicate(studyName); duplicateTaskUI.setTask(task); - duplicateTaskUI.start(); - const duplicatingStudyCard = this.__createDuplicateCard(studyName); - duplicatingStudyCard.setTask(task); - duplicatingStudyCard.subscribeToFilterGroup("searchBarFilter"); - this._resourcesContainer.addNonResourceCard(duplicatingStudyCard); - this.__attachDuplicateEventHandler(task, duplicateTaskUI, duplicatingStudyCard); + + osparc.task.TasksContainer.getInstance().addTaskUI(duplicateTaskUI); + + const cardTitle = this.tr("Duplicating ") + studyName; + const duplicatingStudyCard = this._addTaskCard(task, cardTitle, osparc.task.Duplicate.ICON); + if (duplicatingStudyCard) { + this.__attachDuplicateEventHandler(task, duplicateTaskUI, duplicatingStudyCard); + } }, __attachDuplicateEventHandler: function(task, taskUI, duplicatingStudyCard) { @@ -2253,22 +2141,24 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { if (msg) { osparc.FlashMessenger.logAs(msg, msgLevel); } - taskUI.stop(); + osparc.store.PollTasks.getInstance().removeTask(task); + osparc.task.TasksContainer.getInstance().removeTaskUI(taskUI); this._resourcesContainer.removeNonResourceCard(duplicatingStudyCard); }; task.addListener("taskAborted", () => { const msg = this.tr("Duplication cancelled"); - finished(msg, "INFO"); + finished(msg, "WARNING"); }); task.addListener("resultReceived", e => { - finished(); + const msg = this.tr("Duplication completed"); + finished(msg, "INFO"); const duplicatedStudyData = e.getData(); this._updateStudyData(duplicatedStudyData); }); task.addListener("pollingError", e => { const err = e.getData(); - const msg = this.tr("Something went wrong Duplicating the study
") + err.message; + const msg = this.tr("Something went wrong while duplicating the study
") + err.message; finished(msg, "ERROR"); }); } diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js index afe3d47bd95..3a6ba34f6b3 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js @@ -45,7 +45,7 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { "locationChanged": "qx.event.type.Data", "workspaceUpdated": "qx.event.type.Data", "deleteWorkspaceRequested": "qx.event.type.Data", - "emptyTrashRequested": "qx.event.type.Event", + "trashEmptied": "qx.event.type.Event", }, properties: { @@ -193,13 +193,28 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { break; } case "empty-trash-button": { - control = new qx.ui.form.Button(this.tr("Empty Bin"), "@FontAwesome5Solid/trash/14").set({ + control = new osparc.ui.form.FetchButton(this.tr("Delete all"), "@FontAwesome5Solid/trash/14").set({ appearance: "danger-button", allowGrowY: false, alignY: "middle", - visibility: "excluded", // Not yet implemented }); - control.addListener("execute", () => this.fireEvent("emptyTrashRequested")); + control.addListener("execute", () => { + const win = this.__createConfirmEmptyTrashWindow(); + win.center(); + win.open(); + win.addListener("close", () => { + control.setFetching(true); + if (win.getConfirmed()) { + osparc.data.Resources.fetch("trash", "delete") + .then(() => { + this.fireEvent("trashEmptied") + }) + .finally(() => control.setFetching(false)); + } else { + control.setFetching(false) + } + }, this); + }); this._addAt(control, this.self().POS.EMPTY_TRASH_BUTTON); break; } @@ -208,14 +223,26 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { return control || this.base(arguments, id); }, - __titleTapped: function() { - const workspaceId = this.getCurrentWorkspaceId(); - const folderId = null; - this.setCurrentFolderId(folderId); - this.fireDataEvent("locationChanged", { - workspaceId, - folderId, + __createConfirmEmptyTrashWindow: function() { + const msg = this.tr("All items will be permanently deleted"); + const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete"), + confirmText: this.tr("Delete permanently"), + confirmAction: "delete" }); + return confirmationWin; + }, + + __titleTapped: function() { + if (osparc.store.Store.getInstance().getStudyBrowserContext() === "studiesAndFolders") { + const workspaceId = this.getCurrentWorkspaceId(); + const folderId = null; + this.setCurrentFolderId(folderId); + this.fireDataEvent("locationChanged", { + workspaceId, + folderId, + }); + } }, __buildLayout: function() { @@ -276,10 +303,10 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { break; case "trash": { this.__setIcon("@FontAwesome5Solid/trash/20"); - title.setValue(this.tr("Bin")); + title.setValue(this.tr("Recently Deleted")); const trashDays = osparc.store.StaticInfo.getInstance().getTrashRetentionDays(); description.set({ - value: this.tr(`Items in the Bin will be permanently deleted after ${trashDays} days.`), + value: this.tr(`Items here will be permanently deleted after ${trashDays} days.`), visibility: "visible", }); break; diff --git a/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js index e9443c4ecbb..bcaebf17aa6 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/TemplateBrowser.js @@ -23,16 +23,6 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { this.base(arguments); }, - properties: { - multiSelection: { - check: "Boolean", - init: false, - nullable: false, - event: "changeMultiSelection", - apply: "__applyMultiSelection" - } - }, - members: { __updateAllButton: null, @@ -85,6 +75,8 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { }, __reloadTemplates: function() { + this.__tasksToCards(); + osparc.data.Resources.getInstance().getAllPages("templates") .then(templates => this.__setResourcesToList(templates)) .catch(err => { @@ -131,118 +123,6 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { this.resetSelection(); }, - _createStudyFromTemplate: function(templateData) { - if (!this._checkLoggedIn()) { - return; - } - - const studyAlias = osparc.product.Utils.getStudyAlias({firstUpperCase: true}); - this._showLoadingPage(this.tr("Creating ") + (templateData.name || studyAlias)); - - if (osparc.desktop.credits.Utils.areWalletsEnabled()) { - const studyOptions = new osparc.study.StudyOptions(); - // they will be patched once the study is created - studyOptions.setPatchStudy(false); - studyOptions.setStudyData(templateData); - studyOptions.getChildControl("open-button").setLabel(this.tr("New")); - const win = osparc.study.StudyOptions.popUpInWindow(studyOptions); - win.moveItUp(); - const cancelStudyOptions = () => { - this._hideLoadingPage(); - win.close(); - } - win.addListener("cancel", () => cancelStudyOptions()); - studyOptions.addListener("cancel", () => cancelStudyOptions()); - studyOptions.addListener("startStudy", () => { - const newName = studyOptions.getChildControl("title-field").getValue(); - const walletSelection = studyOptions.getChildControl("wallet-selector").getSelection(); - const nodesPricingUnits = studyOptions.getChildControl("study-pricing-units").getNodePricingUnits(); - win.close(); - - this._showLoadingPage(this.tr("Creating ") + (newName || studyAlias)); - osparc.study.Utils.createStudyFromTemplate(templateData, this._loadingPage) - .then(newStudyData => { - const studyId = newStudyData["uuid"]; - const openCB = () => { - this._hideLoadingPage(); - }; - const cancelCB = () => { - this._hideLoadingPage(); - const params = { - url: { - studyId - } - }; - osparc.data.Resources.fetch("studies", "delete", params); - }; - - const promises = []; - // patch the name - if (newStudyData["name"] !== newName) { - promises.push(osparc.study.StudyOptions.updateName(newStudyData, newName)); - } - // patch the wallet - if (walletSelection.length && walletSelection[0]["walletId"]) { - const walletId = walletSelection[0]["walletId"]; - promises.push(osparc.study.StudyOptions.updateWallet(newStudyData["uuid"], walletId)); - } - // patch the pricing units - // the nodeIds are coming from the original template, they need to be mapped to the newStudy - const workbench = newStudyData["workbench"]; - const nodesIdsListed = []; - Object.keys(workbench).forEach(nodeId => { - const node = workbench[nodeId]; - if (osparc.study.StudyPricingUnits.includeInList(node)) { - nodesIdsListed.push(nodeId); - } - }); - nodesPricingUnits.forEach((nodePricingUnits, idx) => { - const selectedPricingUnitId = nodePricingUnits.getPricingUnits().getSelectedUnitId(); - if (selectedPricingUnitId) { - const nodeId = nodesIdsListed[idx]; - const pricingPlanId = nodePricingUnits.getPricingPlanId(); - promises.push(osparc.study.NodePricingUnits.patchPricingUnitSelection(studyId, nodeId, pricingPlanId, selectedPricingUnitId)); - } - }); - - Promise.all(promises) - .then(() => { - win.close(); - const showStudyOptions = false; - this._startStudyById(studyId, openCB, cancelCB, showStudyOptions); - }); - }) - .catch(err => { - this._hideLoadingPage(); - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - console.error(err); - }); - }); - } else { - osparc.study.Utils.createStudyFromTemplate(templateData, this._loadingPage) - .then(newStudyData => { - const studyId = newStudyData["uuid"]; - const openCB = () => this._hideLoadingPage(); - const cancelCB = () => { - this._hideLoadingPage(); - const params = { - url: { - studyId - } - }; - osparc.data.Resources.fetch("studies", "delete", params); - }; - const isStudyCreation = true; - this._startStudyById(studyId, openCB, cancelCB, isStudyCreation); - }) - .catch(err => { - this._hideLoadingPage(); - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - console.error(err); - }); - } - }, - // LAYOUT // _createLayout: function() { this._createSearchBar(); @@ -326,7 +206,7 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { const templatePromises = []; for (const nodeId in studyData["workbench"]) { const node = studyData["workbench"][nodeId]; - const latestCompatible = osparc.service.Utils.getLatestCompatible(node["key"], node["version"]); + const latestCompatible = osparc.store.Services.getLatestCompatible(node["key"], node["version"]); if (latestCompatible && (node["key"] !== latestCompatible["key"] || node["version"] !== latestCompatible["version"])) { const patchData = {}; if (node["key"] !== latestCompatible["key"]) { @@ -335,17 +215,13 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { if (node["version"] !== latestCompatible["version"]) { patchData["version"] = latestCompatible["version"]; } - templatePromises.push(osparc.info.StudyUtils.patchNodeData(uniqueTemplateData, nodeId, patchData)); + templatePromises.push(osparc.store.Study.patchNodeData(uniqueTemplateData, nodeId, patchData)); } } Promise.all(templatePromises) .then(() => this._updateTemplateData(uniqueTemplateData)) .catch(err => { - if ("message" in err) { - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - } else { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong"), "ERROR"); - } + osparc.FlashMessenger.logError(err); }); } }, @@ -390,7 +266,7 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { return null; } - const editButton = new qx.ui.menu.Button(this.tr("Edit")); + const editButton = new qx.ui.menu.Button(this.tr("Open")); editButton.addListener("execute", () => this.__editTemplate(templateData), this); return editButton; }, @@ -450,17 +326,14 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { const arCopy = osparc.utils.Utils.deepCloneObject(studyData["accessRights"]); // remove collaborator delete arCopy[myGid]; - operationPromise = osparc.info.StudyUtils.patchStudyData(studyData, "accessRights", arCopy); + operationPromise = osparc.store.Study.patchStudyData(studyData, "accessRights", arCopy); } else { // delete study operationPromise = osparc.store.Store.getInstance().deleteStudy(studyData.uuid); } operationPromise .then(() => this.__removeFromTemplateList(studyData.uuid)) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(err, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err)); }, __removeFromTemplateList: function(templateId) { @@ -473,18 +346,40 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { // MENU // // TASKS // + __tasksToCards: function() { + const tasks = osparc.store.PollTasks.getInstance().getPublishTemplateTasks(); + tasks.forEach(task => { + const studyName = ""; + this.taskToTemplateReceived(task, studyName); + }); + }, + + taskToTemplateReceived: function(task, studyName) { + const toTemplateTaskUI = new osparc.task.ToTemplate(studyName); + toTemplateTaskUI.setTask(task); + + osparc.task.TasksContainer.getInstance().addTaskUI(toTemplateTaskUI); + + const cardTitle = this.tr("Publishing ") + studyName; + const toTemplateCard = this._addTaskCard(task, cardTitle, osparc.task.ToTemplate.ICON); + if (toTemplateCard) { + this.__attachToTemplateEventHandler(task, toTemplateTaskUI, toTemplateCard); + } + }, + __attachToTemplateEventHandler: function(task, taskUI, toTemplateCard) { const finished = (msg, msgLevel) => { if (msg) { osparc.FlashMessenger.logAs(msg, msgLevel); } - taskUI.stop(); + osparc.store.PollTasks.getInstance().removeTask(task); + osparc.task.TasksContainer.getInstance().removeTaskUI(taskUI); this._resourcesContainer.removeNonResourceCard(toTemplateCard); }; task.addListener("taskAborted", () => { const msg = this.tr("Study to Template cancelled"); - finished(msg, "INFO"); + finished(msg, "WARNING"); }); task.addListener("updateReceived", e => { const updateData = e.getData(); @@ -499,53 +394,16 @@ qx.Class.define("osparc.dashboard.TemplateBrowser", { } }, this); task.addListener("resultReceived", e => { - finished(); + const msg = this.tr("Template created"); + finished(msg, "INFO"); this.reloadResources(); }); task.addListener("pollingError", e => { const err = e.getData(); - const msg = this.tr("Something went wrong Publishing the study
") + err.message; + const msg = this.tr("Something went wrong while publishing the study
") + err.message; finished(msg, "ERROR"); }); }, - - _taskDataReceived: function(taskData) { - // a bit hacky - if (taskData["task_id"].includes("from_study") && taskData["task_id"].includes("as_template")) { - const interval = 1000; - const pollTasks = osparc.data.PollTasks.getInstance(); - const task = pollTasks.addTask(taskData, interval); - if (task === null) { - return; - } - // ask backend for studyData? - const studyName = ""; - this.taskToTemplateReceived(task, studyName); - } - }, - - taskToTemplateReceived: function(task, studyName) { - const toTemplateTaskUI = new osparc.task.ToTemplate(studyName); - toTemplateTaskUI.setTask(task); - toTemplateTaskUI.start(); - const toTemplateCard = this.__createToTemplateCard(studyName); - toTemplateCard.setTask(task); - this.__attachToTemplateEventHandler(task, toTemplateTaskUI, toTemplateCard); - }, - - __createToTemplateCard: function(studyName) { - const isGrid = this._resourcesContainer.getMode() === "grid"; - const toTemplateCard = isGrid ? new osparc.dashboard.GridButtonPlaceholder() : new osparc.dashboard.ListButtonPlaceholder(); - toTemplateCard.buildLayout( - this.tr("Publishing ") + studyName, - osparc.task.ToTemplate.ICON + (isGrid ? "60" : "24"), - null, - true - ); - toTemplateCard.subscribeToFilterGroup("searchBarFilter"); - this._resourcesContainer.addNonResourceCard(toTemplateCard); - return toTemplateCard; - } // TASKS // } }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js index a712666f7fd..80f08813217 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js @@ -51,11 +51,11 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { nullable: true }, - icon: { + thumbnail: { check: "String", init: null, nullable: true, - apply: "_applyIcon", + apply: "__applyThumbnail", }, resourceType: { @@ -184,7 +184,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { flex: 1 }); break; - case "icon": { + case "thumbnail": { layout = this.getChildControl("body"); const maxWidth = this.self().ITEM_WIDTH; control = new osparc.ui.basic.Thumbnail(null, maxWidth, 124); @@ -269,9 +269,8 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { return layout; }, - // overridden - _applyIcon: function(value) { - const image = this.getChildControl("icon").getChildControl("image"); + __applyThumbnail: function(value) { + const image = this.getChildControl("thumbnail").getChildControl("image"); if ( value.includes("@FontAwesome5Solid/") || value.includes("@MaterialIcons/") diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js index e2a5ab98086..d645d5799bc 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonItem.js @@ -161,7 +161,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { workspace.bind("workspaceId", this, "workspaceId"); workspace.bind("name", this, "title"); workspace.bind("description", this, "description"); - workspace.bind("thumbnail", this, "icon", { + workspace.bind("thumbnail", this, "thumbnail", { converter: thumbnail => thumbnail ? thumbnail : osparc.store.Workspaces.iconPath(-1) }); workspace.bind("accessRights", this, "accessRights"); @@ -188,9 +188,10 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { const menuButton = this.getChildControl("menu-button"); menuButton.setVisibility("visible"); - const menu = new qx.ui.menu.Menu(); - menu.setPosition("bottom-right"); - osparc.utils.Utils.prettifyMenu(menu); + const menu = new qx.ui.menu.Menu().set({ + appearance: "menu-wider", + position: "bottom-right", + }); const studyBrowserContext = osparc.store.Store.getInstance().getStudyBrowserContext(); if ( @@ -217,7 +218,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { menu.addSeparator(); - const trashButton = new qx.ui.menu.Button(this.tr("Move to Bin"), "@FontAwesome5Solid/trash/12"); + const trashButton = new qx.ui.menu.Button(this.tr("Delete"), "@FontAwesome5Solid/trash/12"); trashButton.addListener("execute", () => this.__trashWorkspaceRequested(), this); menu.add(trashButton); } else if (studyBrowserContext === "trash") { @@ -263,7 +264,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { const dateBy = this.getChildControl("date-by"); dateBy.set({ date: value, - toolTipText: this.tr("Moved to the bin"), + toolTipText: this.tr("Deleted"), }); } }, @@ -300,11 +301,11 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonItem", { __trashWorkspaceRequested: function() { const trashDays = osparc.store.StaticInfo.getInstance().getTrashRetentionDays(); - let msg = this.tr("Are you sure you want to move the Workspace and all its content to the Bin?"); + let msg = this.tr("Are you sure you want to delete the Workspace and all its content?"); msg += "
" + this.tr("It will be permanently deleted after ") + trashDays + " days."; const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ - caption: this.tr("Move to Bin"), - confirmText: this.tr("Move to Bin"), + caption: this.tr("Delete"), + confirmText: this.tr("Delete"), confirmAction: "delete" }); confirmationWin.center(); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js index aa8425858a8..8545d665811 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js @@ -37,7 +37,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonNew", { const title = this.getChildControl("title"); title.setValue(this.tr("New Workspace")); - this.setIcon(osparc.dashboard.CardBase.NEW_ICON); + this.setThumbnail(osparc.dashboard.CardBase.NEW_ICON); this.getChildControl("header").set({ opacity: 1 diff --git a/services/static-webserver/client/source/class/osparc/data/Converters.js b/services/static-webserver/client/source/class/osparc/data/Converters.js index c5cae8bd25c..d567b63e561 100644 --- a/services/static-webserver/client/source/class/osparc/data/Converters.js +++ b/services/static-webserver/client/source/class/osparc/data/Converters.js @@ -24,40 +24,6 @@ qx.Class.define("osparc.data.Converters", { type: "static", statics: { - __mergeFileTreeChildren: function(one, two) { - let newDir = true; - for (let i=0; i
Please try again later and/or contact support"; + // NOTE: a temporary solution to avoid duplicate information + if (!message.includes("contact") && !message.includes("try")) { + message += "
Please try again later and/or contact support"; + } } const err = Error(message ? message : `Error while trying to fetch ${endpoint} ${resource}`); if (status) { diff --git a/services/static-webserver/client/source/class/osparc/data/model/IframeHandler.js b/services/static-webserver/client/source/class/osparc/data/model/IframeHandler.js index 2ca74c47274..850672e9657 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/IframeHandler.js +++ b/services/static-webserver/client/source/class/osparc/data/model/IframeHandler.js @@ -94,18 +94,27 @@ qx.Class.define("osparc.data.model.IframeHandler", { osparc.utils.Utils.setIdToWidget(iframe.getIframe(), "iframe_"+this.getNode().getNodeId()); if (osparc.product.Utils.isProduct("s4llite")) { iframe.setShowToolbar(false); + } else { + this.getStudy().getUi().bind("mode", iframe, "showToolbar", { + converter: mode => mode !== "standalone" + }); } - iframe.addListener("restart", () => this.__restartIFrame(), this); + iframe.addListener("restart", () => this.restartIFrame(), this); iframe.getDiskUsageIndicator().setCurrentNode(this.getNode()) this.setIFrame(iframe); }, __initLoadingPage: function() { - const showZoomMaximizeButton = !osparc.product.Utils.isProduct("s4llite"); - const loadingPage = new osparc.ui.message.Loading(showZoomMaximizeButton); - loadingPage.set({ + const loadingPage = new osparc.ui.message.Loading().set({ header: this.__getLoadingPageHeader() }); + if (osparc.product.Utils.isProduct("s4llite")) { + loadingPage.setShowToolbar(false); + } else { + this.getStudy().getUi().bind("mode", loadingPage, "showToolbar", { + converter: mode => mode !== "standalone" + }); + } const node = this.getNode(); const thumbnail = node.getMetaData()["thumbnail"]; @@ -115,7 +124,9 @@ qx.Class.define("osparc.data.model.IframeHandler", { node.addListener("changeLabel", () => loadingPage.setHeader(this.__getLoadingPageHeader()), this); const nodeStatus = node.getStatus(); - const sequenceWidget = nodeStatus.getProgressSequence().getWidgetForLoadingPage(); + const sequenceWidget = nodeStatus.getProgressSequence().getWidgetForLoadingPage().set({ + width: 400 + }); nodeStatus.bind("interactive", sequenceWidget, "visibility", { converter: state => ["pending", "pulling", "starting", "connecting"].includes(state) ? "visible" : "excluded" }); @@ -159,7 +170,7 @@ qx.Class.define("osparc.data.model.IframeHandler", { if ("status" in err && err.status === 406) { errorMsg = node.getKey() + ":" + node.getVersion() + "is retired"; node.getStatus().setInteractive("retired"); - osparc.FlashMessenger.getInstance().logAs(node.getLabel() + this.tr(" is retired"), "ERROR"); + osparc.FlashMessenger.logAs(node.getLabel() + this.tr(" is retired"), "ERROR"); } const errorMsgData = { nodeId: node.getNodeId(), @@ -181,7 +192,7 @@ qx.Class.define("osparc.data.model.IframeHandler", { this.__unresponsiveRetries--; } else { node.getStatus().setInteractive("failed"); - osparc.FlashMessenger.getInstance().logAs(this.tr("There was an error starting") + " " + node.getLabel(), "ERROR"); + osparc.FlashMessenger.logAs(this.tr("There was an issue starting") + " " + node.getLabel(), "ERROR"); } }); }, @@ -352,7 +363,7 @@ qx.Class.define("osparc.data.model.IframeHandler", { node.fireDataEvent("showInLogger", msgData); // will switch to iframe's content - this.__restartIFrame(); + this.restartIFrame(); if (!node.isDynamicV2()) { node.callRetrieveInputs(); } @@ -374,7 +385,7 @@ qx.Class.define("osparc.data.model.IframeHandler", { } }, - __restartIFrame: function() { + restartIFrame: function() { const node = this.getNode(); if (node.getServiceUrl() !== null) { // restart button pushed diff --git a/services/static-webserver/client/source/class/osparc/data/model/LicensedItem.js b/services/static-webserver/client/source/class/osparc/data/model/LicensedItem.js new file mode 100644 index 00000000000..b184523a35d --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/data/model/LicensedItem.js @@ -0,0 +1,185 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2025 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.data.model.LicensedItem", { + extend: qx.core.Object, + + /** + * @param licensedItemData {Object} Object containing the serialized LicensedItem Data + */ + construct: function(licensedItemData) { + this.base(arguments); + + let thumbnail = ""; + let date = null; + let licensedResources = []; + if (licensedItemData["licensedResources"]) { + if (licensedItemData["licensedResources"].length) { + const firstItem = licensedItemData["licensedResources"][0]["source"]; + if (firstItem["thumbnail"]) { + thumbnail = firstItem["thumbnail"]; + } + if (firstItem["features"] && firstItem["features"]["date"]) { + date = firstItem["features"]["date"]; + } + } + licensedItemData["licensedResources"].forEach(licensedRsrc => { + const licensedItemResource = new osparc.data.model.LicensedItemResource(licensedRsrc["source"]); + if (licensedItemData["termsOfUseUrl"]) { + licensedItemResource.set({ + termsOfUseUrl: licensedItemData["termsOfUseUrl"], + }) + } + licensedResources.push(licensedItemResource); + }); + } + let categoryIcon = "@FontAwesome5Solid/shopping-bag/20"; + if (licensedItemData.categoryIcon) { + categoryIcon = licensedItemData.categoryIcon; + } else if (qx.util.ResourceManager.getInstance().has(`osparc/market/${licensedItemData.categoryId}.svg`)) { + categoryIcon = `osparc/market/${licensedItemData.categoryId}.svg`; + } + + this.set({ + licensedItemId: licensedItemData.licensedItemId, + categoryId: licensedItemData.categoryId, + categoryDisplay: licensedItemData.categoryDisplay, + categoryIcon: categoryIcon, + pricingPlanId: licensedItemData.pricingPlanId, + key: licensedItemData.key, + version: licensedItemData.version, + thumbnail: thumbnail, + displayName: licensedItemData.displayName, + date: new Date(date), + licensedResources: licensedResources, + seats: licensedItemData.seats || [], + }); + }, + + properties: { + licensedItemId: { + check: "String", + nullable: false, + init: null, + event: "changeLicensedItemId", + }, + + categoryId: { + check: "String", + nullable: true, + init: null, + event: "changeCategoryId", + }, + + categoryDisplay: { + check: "String", + nullable: true, + init: null, + event: "changeCategoryDisplay", + }, + + categoryIcon: { + check: "String", + nullable: true, + init: null, + event: "changeCategoryIcon", + }, + + pricingPlanId: { + check: "Number", + nullable: false, + init: null, + event: "changePricingPlanId", + }, + + key: { + check: "String", + nullable: false, + init: null, + event: "changeKey", + }, + + version: { + check: "String", + nullable: false, + init: null, + event: "changeVersion", + }, + + thumbnail: { + check: "String", + nullable: true, + init: null, + event: "changeThumbnail", + }, + + displayName: { + check: "String", + nullable: false, + init: null, + event: "changeDisplayName", + }, + + date: { + check: "Date", + nullable: false, + init: null, + event: "changeDate", + }, + + licensedResources: { + check: "Array", + nullable: false, + init: [], + event: "changeLicensedResources", + }, + + seats: { + check: "Array", + nullable: false, + init: [], + event: "changeSeats", + }, + }, + + statics: { + addSeatsFromPurchases: function(licensedItems, purchases) { + // reset seats + Object.values(licensedItems).forEach(licensedItem => licensedItem.setSeats([])); + // populate seats + purchases.forEach(purchase => { + const { + key, + version, + } = purchase; + Object.values(licensedItems).forEach(licensedItem => { + if (licensedItem.getKey() === key && licensedItem.getVersion() <= version) { + licensedItem.getSeats().push({ + licensedItemId: purchase["licensedItemId"], + licensedItemPurchaseId: purchase["licensedItemPurchaseId"], + numOfSeats: purchase["numOfSeats"], + expireAt: new Date(purchase["expireAt"]), + }); + } + }); + }) + }, + }, + + members: { + } +}); diff --git a/services/static-webserver/client/source/class/osparc/data/model/LicensedItemResource.js b/services/static-webserver/client/source/class/osparc/data/model/LicensedItemResource.js new file mode 100644 index 00000000000..fa1a141677b --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/data/model/LicensedItemResource.js @@ -0,0 +1,159 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2025 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.data.model.LicensedItemResource", { + extend: qx.core.Object, + + /** + * @param licensedItemResourceData {Object} Object containing the serialized LicensedItem Data + */ + construct: function(licensedItemResourceData) { + this.base(arguments); + + let description = licensedItemResourceData["description"] || ""; + let title = ""; + let subtitle = null; + description = description.replace(/SPEAG/g, " "); // remove SPEAG substring + const delimiter = " - "; + let titleAndSubtitle = description.split(delimiter); + if (titleAndSubtitle.length > 0) { + title = titleAndSubtitle[0]; + titleAndSubtitle.shift(); + } + if (titleAndSubtitle.length > 0) { + subtitle = titleAndSubtitle.join(delimiter); + } + + const manufacturerData = {}; + if (licensedItemResourceData["thumbnail"]) { + if (licensedItemResourceData["thumbnail"].includes("itis.swiss")) { + manufacturerData["label"] = "IT'IS Foundation"; + manufacturerData["link"] = "https://itis.swiss/virtual-population/"; + manufacturerData["icon"] = "https://media.licdn.com/dms/image/v2/C4D0BAQE_FGa66IyvrQ/company-logo_200_200/company-logo_200_200/0/1631341490431?e=2147483647&v=beta&t=7f_IK-ArGjPrz-1xuWolAT4S2NdaVH-e_qa8hsKRaAc"; + } else if (licensedItemResourceData["thumbnail"].includes("speag.swiss")) { + manufacturerData["label"] = "Speag"; + manufacturerData["link"] = "https://speag.swiss/products/em-phantoms/overview-2/"; + manufacturerData["icon"] = "https://media.licdn.com/dms/image/v2/D4E0BAQG2CYG28KAKbA/company-logo_200_200/company-logo_200_200/0/1700045977122/schmid__partner_engineering_ag_logo?e=2147483647&v=beta&t=6CZb1jjg5TnnzQWkrZBS9R3ebRKesdflg-_xYi4dwD8"; + } + } + + this.set({ + modelId: licensedItemResourceData.id, + description: description, + title: title, + subtitle: subtitle, + thumbnail: licensedItemResourceData.thumbnail || "", + features: licensedItemResourceData.features || {}, + doi: licensedItemResourceData.doi || null, + termsOfUseUrl: licensedItemResourceData.termsOfUseUrl || null, + manufacturerLabel: manufacturerData.label || null, + manufacturerLink: manufacturerData.link || null, + manufacturerIcon: manufacturerData.icon || null, + }); + }, + + properties: { + modelId: { + check: "Number", + nullable: false, + init: null, + event: "changeModelId", + }, + + description: { + check: "String", + nullable: false, + init: null, + event: "changeDescription", + }, + + title: { + check: "String", + nullable: false, + init: null, + event: "changeTitle", + }, + + subtitle: { + check: "String", + nullable: true, + init: null, + event: "changeSubtitle", + }, + + thumbnail: { + check: "String", + nullable: false, + init: null, + event: "changeThumbnail", + }, + + features: { + check: "Object", + nullable: false, + init: null, + event: "changeFeatures", + }, + + doi: { + check: "String", + nullable: true, + init: null, + event: "changeDoi", + }, + + termsOfUseUrl: { + check: "String", + nullable: true, + init: null, + event: "changeTermsOfUseUrl", + }, + + manufacturerLabel: { + check: "String", + nullable: true, + init: null, + event: "changeManufacturerLabel", + }, + + manufacturerLink: { + check: "String", + nullable: true, + init: null, + event: "changeManufacturerLink", + }, + + manufacturerIcon: { + check: "String", + nullable: true, + init: null, + event: "changeManufacturerIcon", + }, + }, + + statics: { + longName: function(licensedResource) { + const name = licensedResource.getFeatures()["name"] || licensedResource.getSubtitle(); + const version = licensedResource.getFeatures()["version"] || ""; + const functionality = licensedResource.getFeatures()["functionality"] || "Static"; + return `${name} ${version}, ${functionality}`; + }, + }, + + members: { + } +}); diff --git a/services/static-webserver/client/source/class/osparc/data/model/Node.js b/services/static-webserver/client/source/class/osparc/data/model/Node.js index 25e6c86e2b6..048f69ebfc0 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Node.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Node.js @@ -480,13 +480,15 @@ qx.Class.define("osparc.data.model.Node", { } } - this.__initLogger(); - - this.initIframeHandler(); - if (this.isParameter()) { this.__initParameter(); } + + if (osparc.store.Store.getInstance().getCurrentStudy()) { + // do not initialize the logger and iframe if the study isn't open + this.__initLogger(); + this.initIframeHandler(); + } }, populateNodeUIData: function(nodeUIData) { @@ -528,7 +530,7 @@ qx.Class.define("osparc.data.model.Node", { }, initIframeHandler: function() { - if (this.isDynamic()) { + if (this.isDynamic() && this.__iframeHandler === null) { this.__iframeHandler = new osparc.data.model.IframeHandler(this.getStudy(), this); } }, @@ -549,7 +551,9 @@ qx.Class.define("osparc.data.model.Node", { return this.getIframeHandler() ? this.getIframeHandler().getLoadingPage() : null; }, - __applyPropsForm: function() { + __applyPropsForm: function(propsForm) { + osparc.utils.Utils.setIdToWidget(propsForm, "settingsForm_" + this.getNodeId()); + const checkIsPipelineRunning = () => { const isPipelineRunning = this.getStudy().isPipelineRunning(); this.getPropsForm().setEnabled(!isPipelineRunning); @@ -928,7 +932,7 @@ qx.Class.define("osparc.data.model.Node", { .then(() => this.checkState()) .catch(err => { if ("status" in err && (err.status === 409 || err.status === 402)) { - osparc.FlashMessenger.getInstance().logAs(err.message, "WARNING"); + osparc.FlashMessenger.logAs(err.message, "WARNING"); } else { console.error(err); } @@ -1061,7 +1065,7 @@ qx.Class.define("osparc.data.model.Node", { console.error(failure, error); const errorMsgData = { nodeId: this.getNodeId(), - msg: "Failed retrieving inputs", + msg: "Unsuccessful input retrieval", level: "ERROR" }; this.fireDataEvent("showInLogger", errorMsgData); @@ -1203,7 +1207,7 @@ qx.Class.define("osparc.data.model.Node", { if (!["int"].includes(type)) { return; } - const newMetadata = osparc.service.Utils.getParameterMetadata("integer"); + const newMetadata = osparc.store.Services.getParameterMetadata("integer"); if (newMetadata) { const value = this.__getInputData()["linspace_start"]; const label = this.getLabel(); @@ -1220,7 +1224,7 @@ qx.Class.define("osparc.data.model.Node", { if (!["int"].includes(type)) { return; } - const metadata = osparc.service.Utils.getLatest("simcore/services/frontend/data-iterator/int-range") + const metadata = osparc.store.Services.getLatest("simcore/services/frontend/data-iterator/int-range") if (metadata) { const value = this.__getOutputData("out_1"); const label = this.getLabel(); diff --git a/services/static-webserver/client/source/class/osparc/data/model/NodeProgressSequence.js b/services/static-webserver/client/source/class/osparc/data/model/NodeProgressSequence.js index eec18e05f5b..bb67236294e 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/NodeProgressSequence.js +++ b/services/static-webserver/client/source/class/osparc/data/model/NodeProgressSequence.js @@ -123,7 +123,7 @@ qx.Class.define("osparc.data.model.NodeProgressSequence", { createDisclaimerText: function() { const disclaimerText = new qx.ui.basic.Atom().set({ - label: qx.locale.Manager.tr("Please be patient, this process can take a few minutes ..."), + label: qx.locale.Manager.tr("Please wait, this process may take a few minutes ..."), padding: [20, 10], gap: 15, icon: "@FontAwesome5Solid/exclamation-triangle/16", @@ -186,8 +186,11 @@ qx.Class.define("osparc.data.model.NodeProgressSequence", { getProgress: function(report) { if (report.unit) { + const attempt = ("attempt" in report && report["attempt"] > 1) ? `(attempt ${report["attempt"]}) ` : ""; + const currentValue = osparc.utils.Utils.bytesToSize(report["actual_value"], 1, false); + const totalValue = osparc.utils.Utils.bytesToSize(report["total"], 1, false) return { - progressLabel: `${osparc.utils.Utils.bytesToSize(report["actual_value"], 1, false)} / ${osparc.utils.Utils.bytesToSize(report["total"], 1, false)}`, + progressLabel: `${attempt}${currentValue} / ${totalValue}`, value: report["actual_value"] / report["total"] } } diff --git a/services/static-webserver/client/source/class/osparc/data/model/Study.js b/services/static-webserver/client/source/class/osparc/data/model/Study.js index 6a511f726b7..8f63b1bc79e 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Study.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Study.js @@ -301,20 +301,6 @@ qx.Class.define("osparc.data.model.Study", { return false; }, - hasSlideshow: function(studyData) { - if ("ui" in studyData && "slideshow" in studyData["ui"] && Object.keys(studyData["ui"]["slideshow"]).length) { - return true; - } - return false; - }, - - getUiMode: function(studyData) { - if ("ui" in studyData && "mode" in studyData["ui"]) { - return studyData["ui"]["mode"]; - } - return null; - }, - getOutputValue: function(studyData, nodeId, portId) { if ("workbench" in studyData && nodeId in studyData["workbench"] && @@ -354,17 +340,6 @@ qx.Class.define("osparc.data.model.Study", { "RETRY" ].includes(state); }, - - __isAnyLinkedNodeMissing: function(studyData) { - const existingNodeIds = Object.keys(studyData["workbench"]); - const linkedNodeIds = osparc.data.model.Workbench.getLinkedNodeIds(studyData["workbench"]); - const allExist = linkedNodeIds.every(linkedNodeId => existingNodeIds.includes(linkedNodeId)); - return !allExist; - }, - - isCorrupt: function(studyData) { - return this.__isAnyLinkedNodeMissing(studyData); - }, }, members: { @@ -512,7 +487,7 @@ qx.Class.define("osparc.data.model.Study", { // Do not listen to output related backend updates if the node is a frontend node. // The frontend controls its output values, progress and states. // If a File Picker is uploading a file, the backend could override the current state with some older state. - if (node && nodeData && !osparc.data.model.Node.isFrontend(node)) { + if (node && nodeData && !osparc.data.model.Node.isFrontend(node.getMetaData())) { node.setOutputData(nodeData.outputs); if ("progress" in nodeData) { const progress = Number.parseInt(nodeData["progress"]); @@ -637,6 +612,19 @@ qx.Class.define("osparc.data.model.Study", { return parameters; }, + getNonFrontendNodes: function() { + const nodes = this.getWorkbench().getNodes(); + return Object.values(nodes).filter(node => node.isComputational() || node.isDynamic()); + }, + + isOnlyNodeDynamic: function() { + const validNodes = this.getNonFrontendNodes(); + if (validNodes.length === 1) { + return validNodes[0].isDynamic(); + } + return null; + }, + hasSlideshow: function() { return !this.getUi().getSlideshow().isEmpty(); }, diff --git a/services/static-webserver/client/source/class/osparc/data/model/StudyUI.js b/services/static-webserver/client/source/class/osparc/data/model/StudyUI.js index d6451f158e3..fcc2f7bd9f8 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/StudyUI.js +++ b/services/static-webserver/client/source/class/osparc/data/model/StudyUI.js @@ -63,7 +63,7 @@ qx.Class.define("osparc.data.model.StudyUI", { }, mode: { - check: ["workbench", "guided", "app"], + check: ["workbench", "guided", "app", "standalone"], // "guided" is no longer used init: "workbench", nullable: true, event: "changeMode", diff --git a/services/static-webserver/client/source/class/osparc/data/model/User.js b/services/static-webserver/client/source/class/osparc/data/model/User.js index 7294987345c..47d665f847d 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/User.js +++ b/services/static-webserver/client/source/class/osparc/data/model/User.js @@ -30,10 +30,20 @@ qx.Class.define("osparc.data.model.User", { const userId = ("id" in userData) ? parseInt(userData["id"]) : parseInt(userData["userId"]); const groupId = ("gid" in userData) ? parseInt(userData["gid"]) : parseInt(userData["groupId"]); - const username = userData["userName"]; + const username = userData["userName"] || "-"; const email = ("login" in userData) ? userData["login"] : userData["email"]; - const firstName = ("first_name" in userData) ? userData["first_name"] : userData["firstName"]; - const lastName = ("last_name" in userData) ? userData["last_name"] : userData["lastName"]; + let firstName = ""; + if (userData["first_name"]) { + firstName = userData["first_name"]; + } else if (userData["firstName"]) { + firstName = userData["firstName"]; + } + let lastName = ""; + if (userData["last_name"]) { + lastName = userData["last_name"]; + } else if (userData["lastName"]) { + lastName = userData["lastName"]; + } let description = [firstName, lastName].join(" ").trim(); // the null values will be replaced by empty strings if (email) { if (description) { @@ -50,7 +60,7 @@ qx.Class.define("osparc.data.model.User", { lastName, email, thumbnail, - label: username, + label: userData["userName"] || description, description, }); }, diff --git a/services/static-webserver/client/source/class/osparc/data/model/Workbench.js b/services/static-webserver/client/source/class/osparc/data/model/Workbench.js index d442cb03b63..55c64406cd4 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Workbench.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Workbench.js @@ -283,11 +283,12 @@ qx.Class.define("osparc.data.model.Workbench", { createNode: async function(key, version) { if (!osparc.data.Permissions.getInstance().canDo("study.node.create", true)) { - osparc.FlashMessenger.getInstance().logAs(qx.locale.Manager.tr("You are not allowed to add nodes"), "ERROR"); + const msg = qx.locale.Manager.tr("You are not allowed to add nodes"); + osparc.FlashMessenger.logError(msg); return null; } if (this.getStudy().isPipelineRunning()) { - osparc.FlashMessenger.getInstance().logAs(this.self().CANT_ADD_NODE, "ERROR"); + osparc.FlashMessenger.logError(this.self().CANT_ADD_NODE); return null; } @@ -330,7 +331,7 @@ qx.Class.define("osparc.data.model.Workbench", { level: "ERROR" }; this.fireDataEvent("showInLogger", errorMsgData); - osparc.FlashMessenger.getInstance().logAs(errorMsg, "ERROR"); + osparc.FlashMessenger.logError(errorMsg); return null; } }, @@ -426,7 +427,7 @@ qx.Class.define("osparc.data.model.Workbench", { }, __filePickerNodeRequested: async function(nodeId, portId, file) { - const filePickerMetadata = osparc.service.Utils.getFilePicker(); + const filePickerMetadata = osparc.store.Services.getFilePicker(); const filePicker = await this.createNode(filePickerMetadata["key"], filePickerMetadata["version"]); if (filePicker === null) { return; @@ -459,7 +460,7 @@ qx.Class.define("osparc.data.model.Workbench", { } else { this.removeNode(filePickerId); const msg = qx.locale.Manager.tr("File couldn't be assigned"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); } }); }, @@ -469,7 +470,7 @@ qx.Class.define("osparc.data.model.Workbench", { // create a new ParameterNode const type = osparc.utils.Ports.getPortType(requesterNode.getMetaData()["inputs"], portId); - const parameterMetadata = osparc.service.Utils.getParameterMetadata(type); + const parameterMetadata = osparc.store.Services.getParameterMetadata(type); if (parameterMetadata) { const parameterNode = await this.createNode(parameterMetadata["key"], parameterMetadata["version"]); if (parameterNode === null) { @@ -487,7 +488,7 @@ qx.Class.define("osparc.data.model.Workbench", { if (requesterNode.getPropsForm().addPortLink(portId, pmId, "out_1") !== true) { this.removeNode(pmId); const msg = qx.locale.Manager.tr("Parameter couldn't be assigned"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); } this.fireEvent("reloadModel"); } @@ -499,7 +500,7 @@ qx.Class.define("osparc.data.model.Workbench", { // create a new ProbeNode const requesterPortMD = requesterNode.getMetaData()["outputs"][portId]; const type = osparc.utils.Ports.getPortType(requesterNode.getMetaData()["outputs"], portId); - const probeMetadata = osparc.service.Utils.getProbeMetadata(type); + const probeMetadata = osparc.store.Services.getProbeMetadata(type); if (probeMetadata) { const probeNode = await this.createNode(probeMetadata["key"], probeMetadata["version"]); if (probeNode === null) { @@ -519,7 +520,7 @@ qx.Class.define("osparc.data.model.Workbench", { if (probeNode.getPropsForm().addPortLink("in_1", nodeId, portId) !== true) { this.removeNode(probeId); const msg = qx.locale.Manager.tr("Probe couldn't be assigned"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); } this.fireEvent("reloadModel"); } @@ -536,7 +537,7 @@ qx.Class.define("osparc.data.model.Workbench", { return false; } if (this.getStudy().isPipelineRunning()) { - osparc.FlashMessenger.getInstance().logAs(this.self().CANT_DELETE_NODE, "ERROR"); + osparc.FlashMessenger.logAs(this.self().CANT_DELETE_NODE, "ERROR"); return false; } diff --git a/services/static-webserver/client/source/class/osparc/desktop/MainPage.js b/services/static-webserver/client/source/class/osparc/desktop/MainPage.js index d3eff170ad6..a17b42e9a8d 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/MainPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/MainPage.js @@ -50,13 +50,13 @@ qx.Class.define("osparc.desktop.MainPage", { const navBar = this.__navBar = new osparc.navigation.NavigationBar(); navBar.populateLayout(); navBar.addListener("backToDashboardPressed", () => this.__backToDashboardPressed(), this); - navBar.addListener("downloadStudyLogs", () => this.__downloadStudyLogs(), this); + navBar.addListener("openLogger", () => this.__openLogger(), this); this._add(navBar); // Some resources request before building the main stack osparc.MaintenanceTracker.getInstance().startTracker(); osparc.CookieExpirationTracker.getInstance().startTracker(); - osparc.NewUITracker.getInstance().startTracker(); + // osparc.NewUITracker.getInstance().startTracker(); const store = osparc.store.Store.getInstance(); const preloadPromises = []; @@ -67,6 +67,11 @@ qx.Class.define("osparc.desktop.MainPage", { } preloadPromises.push(store.getAllClassifiers(true)); preloadPromises.push(osparc.store.Tags.getInstance().fetchTags()); + preloadPromises.push(osparc.store.Products.getInstance().fetchUiConfig()); + preloadPromises.push(osparc.store.PollTasks.getInstance().fetchTasks()); + if (osparc.utils.DisabledPlugins.isJobsEnabled()) { + preloadPromises.push(osparc.store.Jobs.getInstance().fetchJobs()); + } Promise.all(preloadPromises) .then(() => { const mainStack = this.__createMainStack(); @@ -171,9 +176,9 @@ qx.Class.define("osparc.desktop.MainPage", { } }, - __downloadStudyLogs: function() { + __openLogger: function() { if (this.__studyEditor) { - this.__studyEditor.getStudyLogger().downloadLogs(); + osparc.ui.window.Window.popUpInWindow(this.__studyEditor.getStudyLogger(), this.tr("Platform logger"), 950, 650); } }, @@ -199,6 +204,11 @@ qx.Class.define("osparc.desktop.MainPage", { __createDashboardLayout: function() { const dashboard = this.__dashboard = new osparc.dashboard.Dashboard(); + const tabsBar = dashboard.getChildControl("bar"); + tabsBar.set({ + paddingBottom: 6 + }); + this.__navBar.addDashboardTabButtons(tabsBar); const dashboardLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox(5)); dashboardLayout.add(dashboard, { flex: 1 @@ -213,7 +223,7 @@ qx.Class.define("osparc.desktop.MainPage", { __publishTemplate: function(data) { const text = this.tr("Started template creation and added to the background tasks"); - osparc.FlashMessenger.getInstance().logAs(text, "INFO"); + osparc.FlashMessenger.logAs(text, "INFO"); const params = { url: { @@ -226,7 +236,7 @@ qx.Class.define("osparc.desktop.MainPage", { pollTask: true }; const fetchPromise = osparc.data.Resources.fetch("studies", "postToTemplate", params, options); - const pollTasks = osparc.data.PollTasks.getInstance(); + const pollTasks = osparc.store.PollTasks.getInstance(); const interval = 1000; pollTasks.createPollingTask(fetchPromise, interval) .then(task => { @@ -236,12 +246,12 @@ qx.Class.define("osparc.desktop.MainPage", { } task.addListener("resultReceived", e => { const templateData = e.getData(); - osparc.info.StudyUtils.addCollaborators(templateData, data["accessRights"]); + osparc.store.Study.addCollaborators(templateData, data["accessRights"]); }); }) .catch(errMsg => { - const msg = this.tr("Something went wrong Duplicating the study
") + errMsg; - osparc.FlashMessenger.logAs(msg, "ERROR"); + const msg = this.tr("Something went wrong while duplicating the study
") + errMsg; + osparc.FlashMessenger.logError(msg); }); }, @@ -294,7 +304,7 @@ qx.Class.define("osparc.desktop.MainPage", { osparc.data.Resources.fetch("snapshots", "checkout", params) .then(snapshotResp => { if (!snapshotResp) { - const msg = this.tr("Snapshot not found"); + const msg = this.tr("No snapshot found"); throw new Error(msg); } const params2 = { @@ -302,7 +312,7 @@ qx.Class.define("osparc.desktop.MainPage", { "studyId": studyId } }; - osparc.data.Resources.getOne("studies", params2) + osparc.data.Resources.fetch("studies", "getOne", params2) .then(studyData => { if (!studyData) { const msg = this.tr("Study not found"); @@ -312,7 +322,7 @@ qx.Class.define("osparc.desktop.MainPage", { }); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.__showDashboard(); return; }); @@ -342,7 +352,7 @@ qx.Class.define("osparc.desktop.MainPage", { } }; // OM TODO. DO NOT ADD ITERATIONS TO STUDIES CACHE - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(studyData => { if (!studyData) { const msg = this.tr("Iteration not found"); @@ -351,7 +361,7 @@ qx.Class.define("osparc.desktop.MainPage", { osparc.desktop.MainPageHandler.getInstance().loadStudy(studyData); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.__showDashboard(); return; }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js b/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js index ee935adab67..577ddece319 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js +++ b/services/static-webserver/client/source/class/osparc/desktop/MainPageDesktop.js @@ -62,6 +62,8 @@ qx.Class.define("osparc.desktop.MainPageDesktop", { } preloadPromises.push(store.getAllClassifiers(true)); preloadPromises.push(osparc.store.Tags.getInstance().fetchTags()); + preloadPromises.push(osparc.store.Products.getInstance().fetchUiConfig()); + preloadPromises.push(osparc.store.PollTasks.getInstance().fetchTasks()); Promise.all(preloadPromises) .then(() => { const desktopCenter = new osparc.desktop.credits.DesktopCenter(); diff --git a/services/static-webserver/client/source/class/osparc/desktop/MainPageHandler.js b/services/static-webserver/client/source/class/osparc/desktop/MainPageHandler.js index 6e34be5d88c..40174728275 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/MainPageHandler.js +++ b/services/static-webserver/client/source/class/osparc/desktop/MainPageHandler.js @@ -69,7 +69,7 @@ qx.Class.define("osparc.desktop.MainPageHandler", { "studyId": studyId } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(studyData => { if (!studyData) { const msg = qx.locale.Manager.tr("Study not found"); @@ -78,7 +78,7 @@ qx.Class.define("osparc.desktop.MainPageHandler", { this.loadStudy(studyData); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.showDashboard(); return; }); @@ -103,22 +103,26 @@ qx.Class.define("osparc.desktop.MainPageHandler", { } // check if it's corrupt - if (osparc.data.model.Study.isCorrupt(studyData)) { + if (osparc.study.Utils.isCorrupt(studyData)) { const msg = `${qx.locale.Manager.tr("We encountered an issue with the")} ${studyAlias}
${qx.locale.Manager.tr("Please contact support.")}`; throw new Error(msg); } this.setLoadingPageHeader(qx.locale.Manager.tr("Loading ") + studyData.name); this.showLoadingPage(); - const inaccessibleServices = osparc.study.Utils.getInaccessibleServices(studyData["workbench"]) - if (inaccessibleServices.length) { - const msg = osparc.study.Utils.getInaccessibleServicesMsg(inaccessibleServices, studyData["workbench"]); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - this.showDashboard(); - return; - } - this.showStudyEditor(); - this.__studyEditor.setStudyData(studyData); + + osparc.store.Services.getStudyServicesMetadata(studyData) + .finally(() => { + const inaccessibleServices = osparc.store.Services.getInaccessibleServices(studyData["workbench"]) + if (inaccessibleServices.length) { + const msg = osparc.store.Services.getInaccessibleServicesMsg(inaccessibleServices, studyData["workbench"]); + osparc.FlashMessenger.getInstance().logError(msg); + this.showDashboard(); + return; + } + this.showStudyEditor(); + this.__studyEditor.setStudyData(studyData); + }); } } }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js b/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js index e05a37f56a1..52c33d94be9 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js +++ b/services/static-webserver/client/source/class/osparc/desktop/SlideshowView.js @@ -72,7 +72,7 @@ qx.Class.define("osparc.desktop.SlideshowView", { const nodeId = e.getData(); this.__hideNode(nodeId); }, this); - slideshowToolbar.addListener("slidesStop", () => this.fireEvent("slidesStop"), this); + slideshowToolbar.addListener("slidesStop", () => this.getStudy().getUi().setMode("workbench"), this); this._add(slideshowToolbar); const mainView = this.__mainView = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ @@ -109,7 +109,6 @@ qx.Class.define("osparc.desktop.SlideshowView", { }, events: { - "slidesStop": "qx.event.type.Event", "startPartialPipeline": "qx.event.type.Data", "stopPipeline": "qx.event.type.Event", "backToDashboardPressed": "qx.event.type.Event", @@ -131,12 +130,6 @@ qx.Class.define("osparc.desktop.SlideshowView", { apply: "__applyMaximized", event: "changeMaximized" }, - - pageContext: { - check: ["guided", "app"], - nullable: false, - init: "guided" - } }, statics: { @@ -270,9 +263,6 @@ qx.Class.define("osparc.desktop.SlideshowView", { view = new osparc.node.slideshow.NodeView(); } view.setNode(node); - if (node.isDynamic()) { - view.getSettingsLayout().setVisibility(this.getPageContext() === "app" ? "excluded" : "visible"); - } } this.__connectMaximizeEvents(node); this.__styleView(node, view); @@ -377,7 +367,6 @@ qx.Class.define("osparc.desktop.SlideshowView", { }); } } - this.setPageContext("app"); this.__slideshowToolbar.populateButtons(true); const currentNodeId = this.getStudy().getUi().getCurrentNodeId(); const isValid = slideshow.getPosition(currentNodeId) !== -1; diff --git a/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js b/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js index 3e6a0c943e2..deeb4f38372 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/static-webserver/client/source/class/osparc/desktop/StudyEditor.js @@ -37,8 +37,6 @@ qx.Class.define("osparc.desktop.StudyEditor", { }); workbenchView.addListener("slidesEdit", () => this.__editSlides(), this); - workbenchView.addListener("slidesAppStart", () => this.setPageContext(osparc.navigation.NavigationBar.PAGE_CONTEXT[2]), this); - slideshowView.addListener("slidesStop", () => this.setPageContext(osparc.navigation.NavigationBar.PAGE_CONTEXT[1]), this); workbenchView.addListener("takeSnapshot", () => this.__takeSnapshot(), this); workbenchView.addListener("takeSnapshot", () => this.__takeSnapshot(), this); @@ -72,7 +70,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { const startStopButtons = workbenchView.getStartStopButtons(); startStopButtons.addListener("startPipeline", () => this.__startPipeline([]), this); startStopButtons.addListener("startPartialPipeline", () => { - const partialPipeline = this.getPageContext() === "workbench" ? this.__workbenchView.getSelectedNodeIDs() : this.__slideshowView.getSelectedNodeIDs(); + const partialPipeline = this.getStudy().getUi().getMode() === "app" ? this.__slideshowView.getSelectedNodeIDs() : this.__workbenchView.getSelectedNodeIDs(); this.__startPipeline(partialPipeline); }, this); startStopButtons.addListener("stopPipeline", () => this.__stopPipeline(), this); @@ -103,23 +101,14 @@ qx.Class.define("osparc.desktop.StudyEditor", { apply: "__applyStudy", event: "changeStudy" }, - - pageContext: { - check: ["workbench", "guided", "app"], - init: null, - nullable: false, - event: "changePageContext", - apply: "__applyPageContext" - } }, statics: { AUTO_SAVE_INTERVAL: 3000, - READ_ONLY_TEXT: qx.locale.Manager.tr("You do not have writing permissions.
Your changes will not be saved.") + READ_ONLY_TEXT: qx.locale.Manager.tr("You do not have writing permissions.
Your changes will not be saved."), }, members: { - __study: null, __settingStudy: null, __viewsStack: null, __workbenchView: null, @@ -145,7 +134,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { "studyId": studyData.uuid } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(latestStudyData => { const study = new osparc.data.model.Study(latestStudyData); this.setStudy(study); @@ -179,6 +168,10 @@ qx.Class.define("osparc.desktop.StudyEditor", { } }, this); } + + study.getUi().addListener("changeMode", e => { + this.__uiModeChanged(e.getData(), e.getOldData()); + }); }) .catch(err => { console.error(err); @@ -212,7 +205,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { msg += "
" + err["message"]; } } - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); this.fireEvent("forceBackToDashboard"); }) .finally(() => this._hideLoadingPage()); @@ -240,7 +233,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { let msg = this.tr("The Study contains more than ") + maxNumber + this.tr(" Interactive services."); msg += "
"; msg += this.tr("Please start them manually."); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } } @@ -248,24 +241,12 @@ qx.Class.define("osparc.desktop.StudyEditor", { this.__startAutoSaveTimer(); } else { const msg = this.self().READ_ONLY_TEXT; - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } - const pageContext = study.getUi().getMode(); - switch (pageContext) { - case "guided": - case "app": - this.__slideshowView.startSlides(); - break; - default: - this.__workbenchView.openFirstNode(); - break; - } - this.addListener("changePageContext", e => { - const pageCxt = e.getData(); - study.getUi().setMode(pageCxt); - }); - this.setPageContext(pageContext); + + const uiMode = study.getUi().getMode(); + this.__uiModeChanged(uiMode); const workbench = study.getWorkbench(); workbench.addListener("retrieveInputs", e => { @@ -401,7 +382,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { const usedWallet = store.getWallets().find(wallet => wallet.getWalletId() === walletId); const walletName = usedWallet.getName(); const text = `Wallet "${walletName}", running your service(s) has run out of credits. Stopping service(s) gracefully.`; - osparc.FlashMessenger.getInstance().logAs(this.tr(text), "ERROR", flashMessageDisplayDuration); + osparc.FlashMessenger.logError(this.tr(text), null, flashMessageDisplayDuration); }, this); } }, @@ -425,7 +406,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { const node = workbench.getNode(nodeId); const label = node.getLabel(); const text = `New inputs for service ${label}. Please reload to refresh service.`; - osparc.FlashMessenger.getInstance().logAs(text, "INFO"); + osparc.FlashMessenger.logAs(text, "INFO"); } } }, this); @@ -556,7 +537,8 @@ qx.Class.define("osparc.desktop.StudyEditor", { }, __editSlides: function() { - if (this.getPageContext() !== osparc.navigation.NavigationBar.PAGE_CONTEXT[1]) { + if (this.getStudy().getUi().getMode() !== "workbench") { + // if the user is not in "workbench" mode, return return; } @@ -627,9 +609,9 @@ qx.Class.define("osparc.desktop.StudyEditor", { }, this); } else if (e.getTarget().getStatus() == "402") { const msg = await e.getTarget().getResponse().error.errors[0].message; - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } else { - this.getStudyLogger().error(null, "Failed submitting pipeline"); + this.getStudyLogger().error(null, "Unsuccessful pipeline submission"); } this.getStudy().setPipelineRunning(false); }, this); @@ -719,21 +701,41 @@ qx.Class.define("osparc.desktop.StudyEditor", { return this.__workbenchView.getLogger(); }, - __applyPageContext: function(newCtxt) { - switch (newCtxt) { - case "workbench": - this.__viewsStack.setSelection([this.__workbenchView]); - if (this.getStudy() && this.getStudy().getUi()) { - this.__workbenchView.nodeSelected(this.getStudy().getUi().getCurrentNodeId()); - } - break; + __uiModeChanged: function(newUIMode, oldUIMode) { + switch (newUIMode) { case "guided": case "app": this.__viewsStack.setSelection([this.__slideshowView]); - if (this.getStudy() && this.getStudy().getUi()) { - this.__slideshowView.startSlides(); + this.__slideshowView.startSlides(); + break; + case "standalone": { + this.__viewsStack.setSelection([this.__workbenchView]); + this.__workbenchView.openFirstNode(); + break; + } + case "workbench": + default: { + this.__viewsStack.setSelection([this.__workbenchView]); + if (oldUIMode === "standalone") { + // in this transition, show workbenchUI + this.__workbenchView.setMaximized(false); + // uncomment this when we release the osparc<->s4l integration + // this.__workbenchView.showPipeline(); + } else { + const currentNodeId = this.getStudy().getUi().getCurrentNodeId(); + if (currentNodeId && this.getStudy().getWorkbench().getNode(currentNodeId)) { + const node = this.getStudy().getWorkbench().getNode(currentNodeId); + if (node && node.isDynamic()) { + this.__workbenchView.fullscreenNode(currentNodeId); + } else { + this.__workbenchView.nodeSelected(currentNodeId); + } + } else { + this.__workbenchView.openFirstNode(); + } } break; + } } }, @@ -764,7 +766,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { const store = osparc.store.Store.getInstance(); store.getSnapshots().push(data); }) - .catch(err => osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR")); + .catch(err => osparc.FlashMessenger.logError(err)); win.close(); }, this); @@ -893,8 +895,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { if ("status" in error && error.status === 409) { console.log("Flash message blocked"); // Workaround for osparc-issues #1189 } else { - console.error(error); - osparc.FlashMessenger.getInstance().logAs(this.tr("Error saving the study"), "ERROR"); + osparc.FlashMessenger.logError(this.tr("Error saving the study")); } this.getStudyLogger().error(null, "Error updating pipeline"); // Need to throw the error to be able to handle it later diff --git a/services/static-webserver/client/source/class/osparc/desktop/StudyEditorIdlingTracker.js b/services/static-webserver/client/source/class/osparc/desktop/StudyEditorIdlingTracker.js index 3ffac59d401..4c8575f0442 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/StudyEditorIdlingTracker.js +++ b/services/static-webserver/client/source/class/osparc/desktop/StudyEditorIdlingTracker.js @@ -42,7 +42,7 @@ qx.Class.define("osparc.desktop.StudyEditorIdlingTracker", { __updateFlashMessage: function(timeoutSec) { if (this.__idleFlashMessage === null) { - this.__idleFlashMessage = osparc.FlashMessenger.getInstance().logAs(qx.locale.Manager.tr("Are you still there?"), "WARNING", timeoutSec*1000); + this.__idleFlashMessage = osparc.FlashMessenger.logAs(qx.locale.Manager.tr("Are you still there?"), "WARNING", timeoutSec*1000); } let msg = qx.locale.Manager.tr("Are you still there?") + "
"; diff --git a/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js b/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js index a735b2532d6..e10cc5f62fd 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/static-webserver/client/source/class/osparc/desktop/WorkbenchView.js @@ -49,16 +49,13 @@ qx.Class.define("osparc.desktop.WorkbenchView", { decorateSlider: function(slider) { slider.set({ width: 2, - backgroundColor: "#007fd4", // Visual Studio blue + backgroundColor: "visual-blue", opacity: 1 }); }, openNodeDataManager: function(node) { - const nodeDataManager = new osparc.widget.NodeDataManager(null, node.getNodeId()); - const win = osparc.ui.window.Window.popUpInWindow(nodeDataManager, node.getLabel(), 900, 600).set({ - appearance: "service-window" - }); + const win = osparc.widget.StudyDataManager.popUpInWindow(node.getStudy().getUuid(), node.getNodeId(), node.getLabel()); const closeBtn = win.getChildControl("close-button"); osparc.utils.Utils.setIdToWidget(closeBtn, "nodeDataManagerCloseBtn"); } @@ -70,7 +67,6 @@ qx.Class.define("osparc.desktop.WorkbenchView", { "expandNavBar": "qx.event.type.Event", "backToDashboardPressed": "qx.event.type.Event", "slidesEdit": "qx.event.type.Event", - "slidesAppStart": "qx.event.type.Event", "annotationRectStart": "qx.event.type.Event", "takeSnapshot": "qx.event.type.Event", "showSnapshots": "qx.event.type.Event", @@ -81,7 +77,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { properties: { study: { check: "osparc.data.model.Study", - apply: "_applyStudy", + apply: "__applyStudy", nullable: false }, @@ -244,7 +240,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { return sidePanelsNewWidth; }, - _applyStudy: function(study) { + __applyStudy: function(study) { if (study) { this.__initViews(); this.__connectEvents(); @@ -302,9 +298,12 @@ qx.Class.define("osparc.desktop.WorkbenchView", { if (study === null) { return; } + this.__initPrimaryColumn(); this.__initSecondaryColumn(); this.__initMainView(); + + this.setMaximized(false); }, __initPrimaryColumn: function() { @@ -370,7 +369,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { dragMechanism: true, hideRoot: true }); - filesTree.populateTree(); + filesTree.populateLocations(); const storagePage = this.__storagePage = this.__createTabPage("@FontAwesome5Solid/database", this.tr("Storage"), filesTree, this.self().PRIMARY_COL_BG_COLOR); tabViewPrimary.add(storagePage); @@ -443,12 +442,12 @@ qx.Class.define("osparc.desktop.WorkbenchView", { appearance: "form-button-outlined", label: this.tr("App Mode"), toolTipText: this.tr("Start App Mode"), - icon: "@FontAwesome5Solid/play/14", + icon: osparc.dashboard.CardBase.MODE_APP, marginRight: 10, marginTop: 7, ...osparc.navigation.NavigationBar.BUTTON_OPTIONS }); - startAppButtonTB.addListener("execute", () => this.fireEvent("slidesAppStart")); + startAppButtonTB.addListener("execute", () => study.getUi().setMode("app")); topBar.add(startAppButtonTB); const collapseWithUserMenu = this.__collapseWithUserMenu = new osparc.desktop.CollapseWithUserMenu(); @@ -497,10 +496,9 @@ qx.Class.define("osparc.desktop.WorkbenchView", { studyTreeItem.addListener("changeSelectedNode", () => { nodesTree.resetSelection(); - this.__populateSecondaryColumn(this.getStudy()); - this.__evalIframe(); - this.__openWorkbenchTab(); - this.__loggerView.setCurrentNodeId(null); + this.showPipeline(); + + this.getStudy().getUi().setCurrentNodeId(this.getStudy().getUuid()); }); nodesTree.addListener("changeSelectedNode", e => { studyTreeItem.resetSelection(); @@ -514,6 +512,8 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.__loggerView.setCurrentNodeId(nodeId); this.__workbenchUI.nodeSelected(nodeId); this.fireDataEvent("changeSelectedNode", nodeId); + + this.getStudy().getUi().setCurrentNodeId(nodeId); }); if (this.__workbenchUIConnected === null) { @@ -529,9 +529,13 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.__evalIframe(node); this.__loggerView.setCurrentNodeId(nodeId); this.fireDataEvent("changeSelectedNode", nodeId); + + this.getStudy().getUi().setCurrentNodeId(nodeId); } else { // empty selection this.__studyTreeItem.selectStudyItem(); + + this.getStudy().getUi().setCurrentNodeId(this.getStudy().getUuid()); } }); workbenchUI.addListener("nodeSelected", e => { @@ -545,6 +549,8 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.__populateSecondaryColumn(node); this.__openIframeTab(node); this.__loggerView.setCurrentNodeId(nodeId); + + this.getStudy().getUi().setCurrentNodeId(nodeId); } }, this); } @@ -553,16 +559,8 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const nodeId = e.getData(); if (nodeId) { studyTreeItem.resetSelection(); - const workbench = this.getStudy().getWorkbench(); - const node = workbench.getNode(nodeId); - if (node) { - this.__populateSecondaryColumn(node); - this.__openIframeTab(node); - node.getLoadingPage().maximizeIFrame(true); - node.getIFrame().maximizeIFrame(true); - } - this.__loggerView.setCurrentNodeId(nodeId); - this.__workbenchUI.nodeSelected(nodeId); + this.fullscreenNode(nodeId); + this.getStudy().getUi().setCurrentNodeId(nodeId); } }, this); nodesTree.addListener("removeNode", e => { @@ -839,11 +837,11 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const startAppBtn = this.__startAppButton = new qx.ui.form.Button().set({ label: this.tr("Start"), - icon: "@FontAwesome5Solid/play/14", + icon: osparc.dashboard.CardBase.MODE_APP, toolTipText: this.tr("Start App Mode"), height: buttonsHeight }); - startAppBtn.addListener("execute", () => this.fireEvent("slidesAppStart"), this); + startAppBtn.addListener("execute", () => this.getStudy().getUi().setMode("app"), this); slideshowButtons.add(startAppBtn); this.__evalSlidesButtons(); @@ -1017,7 +1015,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { // HEADER const nodeMetadata = node.getMetaData(); - const version = osparc.service.Utils.getVersionDisplay(nodeMetadata["key"], nodeMetadata["version"]); + const version = osparc.store.Services.getVersionDisplay(nodeMetadata["key"], nodeMetadata["version"]); const header = new qx.ui.basic.Label(`${nodeMetadata["name"]} ${version}`).set({ paddingLeft: 5 }); @@ -1183,21 +1181,38 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.__nodesTree.nodeSelected(this.__currentNodeId); }, + showPipeline: function() { + this.__populateSecondaryColumn(this.getStudy()); + this.__evalIframe(); + this.__openWorkbenchTab(); + this.__loggerView.setCurrentNodeId(null); + + this.getStudy().getUi().setCurrentNodeId(this.getStudy().getUuid()); + }, + + fullscreenNode: function(nodeId) { + const node = this.getStudy().getWorkbench().getNode(nodeId); + if (node && node.isDynamic()) { + qx.event.Timer.once(() => { + this.__populateSecondaryColumn(node); + this.__openIframeTab(node); + node.getIFrame().maximizeIFrame(true); + }, this, 10); + } + this.__loggerView.setCurrentNodeId(nodeId); + this.__workbenchUI.nodeSelected(nodeId); + }, + openFirstNode: function() { - const nodes = this.getStudy().getWorkbench().getNodes(); - const validNodes = Object.values(nodes).filter(node => node.isComputational() || node.isDynamic()); + const validNodes = this.getStudy().getNonFrontendNodes(); if (validNodes.length === 1 && validNodes[0].isDynamic()) { const dynamicNode = validNodes[0]; - this.nodeSelected(dynamicNode.getNodeId()); - qx.event.Timer.once(() => { - this.__openIframeTab(dynamicNode); - dynamicNode.getLoadingPage().maximizeIFrame(true); - dynamicNode.getIFrame().maximizeIFrame(true); - }, this, 10); - return; + this.fullscreenNode(dynamicNode.getNodeId()); + this.getStudy().getUi().setCurrentNodeId(dynamicNode.getNodeId()); + } else { + this.setMaximized(false); + this.nodeSelected(this.getStudy().getUuid()); } - this.setMaximized(false); - this.nodeSelected(this.getStudy().getUuid()); } } }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/account/DeleteAccount.js b/services/static-webserver/client/source/class/osparc/desktop/account/DeleteAccount.js index 79c54bbc1a9..8c3d301923e 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/account/DeleteAccount.js +++ b/services/static-webserver/client/source/class/osparc/desktop/account/DeleteAccount.js @@ -116,13 +116,10 @@ qx.Class.define("osparc.desktop.account.DeleteAccount", { osparc.data.Resources.fetch("auth", "unregister", params) .then(() => { const msg = this.tr(`You account will be deleted in ${retentionDays} days`); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); this.fireEvent("deleted"); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => deleteBtn.setFetching(false)); } } diff --git a/services/static-webserver/client/source/class/osparc/desktop/account/MyAccount.js b/services/static-webserver/client/source/class/osparc/desktop/account/MyAccount.js index 0df1dc1d8be..a77f891f328 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/account/MyAccount.js +++ b/services/static-webserver/client/source/class/osparc/desktop/account/MyAccount.js @@ -24,12 +24,24 @@ qx.Class.define("osparc.desktop.account.MyAccount", { const miniProfile = osparc.desktop.account.MyAccount.createMiniProfileView().set({ paddingRight: 10 }); - this.addWidgetOnTopOfTheTabs(miniProfile); + this.addWidgetToTabs(miniProfile); this.__profilePage = this.__addProfilePage(); - if (osparc.data.Permissions.getInstance().canDo("usage.all.read")) { - this.__usagePage = this.__addUsagePage(); + // show Usage in My Account if wallets are not enabled. If they are enabled it will be in the BIlling Center + if (!osparc.desktop.credits.Utils.areWalletsEnabled()) { + if (osparc.data.Permissions.getInstance().canDo("usage.all.read")) { + this.__usagePage = this.__addUsagePage(); + } + } + + this.__addGeneralSettings(); + this.__addConfirmationSettings(); + if (osparc.product.Utils.showPreferencesTokens()) { + this.__addTokensPage(); + } + if (osparc.data.Permissions.getInstance().canDo("user.tag")) { + this.__addTagsPage(); } }, @@ -96,6 +108,7 @@ qx.Class.define("osparc.desktop.account.MyAccount", { members: { __profilePage: null, __usagePage: null, + __tagsPage: null, __addProfilePage: function() { const title = this.tr("Profile"); @@ -112,10 +125,39 @@ qx.Class.define("osparc.desktop.account.MyAccount", { const page = this.addTab(title, iconSrc, usageOverview); return page; }, + __addGeneralSettings: function() { + const title = this.tr("Settings"); + const iconSrc = "@FontAwesome5Solid/cogs/22"; + const generalPage = new osparc.desktop.preferences.pages.GeneralPage(); + this.addTab(title, iconSrc, generalPage); + }, - openProfile: function() { - this._openPage(this.__profilePage); - return true; - } + __addConfirmationSettings: function() { + const title = this.tr("Confirmations"); + const iconSrc = "@FontAwesome5Solid/question-circle/22"; + const confirmPage = new osparc.desktop.preferences.pages.ConfirmationsPage(); + this.addTab(title, iconSrc, confirmPage); + }, + + __addTokensPage: function() { + const title = this.tr("API Keys/Tokens"); + const iconSrc = "@FontAwesome5Solid/exchange-alt/22"; + const tokensPage = new osparc.desktop.preferences.pages.TokensPage(); + this.addTab(title, iconSrc, tokensPage); + }, + + __addTagsPage: function() { + const title = this.tr("Create/Edit Tags"); + const iconSrc = "@FontAwesome5Solid/tags/22"; + const tagsPage = new osparc.desktop.preferences.pages.TagsPage(); + const page = this.__tagsPage = this.addTab(title, iconSrc, tagsPage); + osparc.utils.Utils.setIdToWidget(page.getChildControl("button"), "preferencesTagsTabBtn"); + }, + + openTags: function() { + if (this.__tagsPage) { + this._openPage(this.__tagsPage); + } + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/account/MyAccountWindow.js b/services/static-webserver/client/source/class/osparc/desktop/account/MyAccountWindow.js index a86eeeb9507..3dcaefa8160 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/account/MyAccountWindow.js +++ b/services/static-webserver/client/source/class/osparc/desktop/account/MyAccountWindow.js @@ -21,7 +21,7 @@ qx.Class.define("osparc.desktop.account.MyAccountWindow", { construct: function() { this.base(arguments, "credits", this.tr("My Account")); - const width = 900; + const width = 990; const height = 700; const maxHeight = 700; this.set({ @@ -46,8 +46,8 @@ qx.Class.define("osparc.desktop.account.MyAccountWindow", { members: { __myAccount: null, - openProfile: function() { - return this.__myAccount.openProfile(); - } + openTags: function() { + this.__myAccount.openTags(); + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js b/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js index 53fcc95d0b8..e522d53975f 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js @@ -31,12 +31,13 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { this._setLayout(new qx.ui.layout.VBox(15)); + this.__userProfileData = {}; + this.__userPrivacyData = {}; + this.__fetchProfile(); this._add(this.__createProfileUser()); - if (osparc.utils.Utils.isDevelopmentPlatform()) { - this._add(this.__createPrivacySection()); - } + this._add(this.__createPrivacySection()); if (osparc.store.StaticInfo.getInstance().is2FARequired()) { this._add(this.__create2FASection()); } @@ -47,8 +48,12 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { members: { __userProfileData: null, __userProfileModel: null, + __userProfileRenderer: null, + __updateProfileBtn: null, __userPrivacyData: null, __userPrivacyModel: null, + __updatePrivacyBtn: null, + __userProfileForm: null, __fetchProfile: function() { osparc.data.Resources.getOne("profile", {}, null, false) @@ -56,9 +61,7 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { this.__setDataToProfile(profile); this.__setDataToPrivacy(profile["privacy"]); }) - .catch(err => { - console.error(err); - }); + .catch(err => console.error(err)); }, __setDataToProfile: function(data) { @@ -72,16 +75,29 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { "expirationDate": data["expirationDate"] || null, }); } + this.__updateProfileBtn.setEnabled(false); }, __setDataToPrivacy: function(privacyData) { if (privacyData) { this.__userPrivacyData = privacyData; this.__userPrivacyModel.set({ + "hideUsername": "hideUsername" in privacyData ? privacyData["hideUsername"] : false, "hideFullname": "hideFullname" in privacyData ? privacyData["hideFullname"] : true, "hideEmail": "hideEmail" in privacyData ? privacyData["hideEmail"] : true, }); + + const visibleIcon = "@FontAwesome5Solid/eye/12"; + const hiddenIcon = "@FontAwesome5Solid/eye-slash/12"; + const icons = { + 0: this.__userPrivacyModel.getHideUsername() ? hiddenIcon : visibleIcon, + 1: this.__userPrivacyModel.getHideFullname() ? hiddenIcon : visibleIcon, + 2: this.__userPrivacyModel.getHideFullname() ? hiddenIcon : visibleIcon, + 3: this.__userPrivacyModel.getHideEmail() ? hiddenIcon : visibleIcon, + }; + this.__userProfileRenderer.setIcons(icons); } + this.__updatePrivacyBtn.setEnabled(false); }, __createProfileUser: function() { @@ -108,12 +124,13 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { readOnly: true }); - const form = new qx.ui.form.Form(); - form.add(username, "Username", null, "username"); - form.add(firstName, "First Name", null, "firstName"); - form.add(lastName, "Last Name", null, "lastName"); - form.add(email, "Email", null, "email"); - box.add(new qx.ui.form.renderer.Single(form)); + const profileForm = this.__userProfileForm = new qx.ui.form.Form(); + profileForm.add(username, "Username", null, "username"); + profileForm.add(firstName, "First Name", null, "firstName"); + profileForm.add(lastName, "Last Name", null, "lastName"); + profileForm.add(email, "Email", null, "email"); + const singleWithIcon = this.__userProfileRenderer = new osparc.ui.form.renderer.SingleWithIcon(profileForm); + box.add(singleWithIcon); const expirationLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox(5)).set({ paddingLeft: 16, @@ -125,7 +142,7 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { expirationLayout.add(expirationDateLabel); const expirationDate = new qx.ui.basic.Label(); expirationLayout.add(expirationDate); - const infoLabel = this.tr("Please contact us by email:
"); + const infoLabel = this.tr("Please contact us via email:
"); const infoExtension = new osparc.ui.hint.InfoHint(infoLabel); const supportEmail = osparc.store.VendorInfo.getInstance().getSupportEmail(); infoExtension.setHintText(infoLabel + supportEmail); @@ -170,21 +187,23 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { namesValidator.add(firstName, qx.util.Validate.regExp(/[^\.\d]+/), this.tr("Avoid dots or numbers in text")); namesValidator.add(lastName, qx.util.Validate.regExp(/^$|[^\.\d]+/), this.tr("Avoid dots or numbers in text")); // allow also empty last name - const updateBtn = new qx.ui.form.Button("Update Profile").set({ + const updateProfileBtn = this.__updateProfileBtn = new qx.ui.form.Button().set({ + label: this.tr("Update Profile"), appearance: "form-button", alignX: "right", - allowGrowX: false + allowGrowX: false, + enabled: false, }); - box.add(updateBtn); + box.add(updateProfileBtn); - updateBtn.addListener("execute", () => { + updateProfileBtn.addListener("execute", () => { if (!osparc.data.Permissions.getInstance().canDo("user.user.update", true)) { this.__resetUserData(); return; } const patchData = {}; - if (this.__userProfileData["username"] !== model.getUsername()) { + if (this.__userProfileData["userName"] !== model.getUsername()) { patchData["userName"] = model.getUsername(); } if (this.__userProfileData["first_name"] !== model.getFirstName()) { @@ -204,61 +223,83 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { this.__setDataToProfile(Object.assign(this.__userProfileData, params.data)); osparc.auth.Manager.getInstance().updateProfile(this.__userProfileData); const msg = this.tr("Profile updated"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); }) .catch(err => { this.__resetUserData(); - const msg = err.message || this.tr("Failed to update profile"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - console.error(err); + osparc.FlashMessenger.logError(err, this.tr("Unsuccessful profile update")); }); } } }); + const profileFields = [ + username, + firstName, + lastName, + ] + const valueChanged = () => { + const anyChanged = + username.getValue() !== this.__userProfileData["userName"] || + firstName.getValue() !== this.__userProfileData["first_name"] || + lastName.getValue() !== this.__userProfileData["last_name"]; + updateProfileBtn.setEnabled(anyChanged); + }; + valueChanged(); + profileFields.forEach(privacyField => privacyField.addListener("changeValue", () => valueChanged())); + return box; }, __createPrivacySection: function() { + // binding to a model + const defaultModel = { + "hideUsername": false, + "hideFullname": true, + "hideEmail": true, + }; + + const privacyModel = this.__userPrivacyModel = qx.data.marshal.Json.createModel(defaultModel, true); + const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("Privacy")); box.set({ alignX: "left", maxWidth: 500 }); - const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("For Privacy reasons, you might want to hide your Full Name and/or the email to other users")); + const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("For Privacy reasons, you might want to hide some personal data.")); box.add(label); + const hideUsername = new qx.ui.form.CheckBox().set({ + value: defaultModel.hideUsername + }); const hideFullname = new qx.ui.form.CheckBox().set({ - value: true + value: defaultModel.hideFullname }); const hideEmail = new qx.ui.form.CheckBox().set({ - value: true + value: defaultModel.hideEmail }); - const form = new qx.ui.form.Form(); - form.add(hideFullname, "Hide Full Name", null, "hideFullname"); - form.add(hideEmail, "Hide email", null, "hideEmail"); - box.add(new qx.ui.form.renderer.Single(form)); + const privacyForm = new qx.ui.form.Form(); + privacyForm.add(hideUsername, "Hide Username", null, "hideUsername"); + privacyForm.add(hideFullname, "Hide Full Name", null, "hideFullname"); + privacyForm.add(hideEmail, "Hide Email", null, "hideEmail"); + box.add(new qx.ui.form.renderer.Single(privacyForm)); - // binding to a model - const raw = { - "hideFullname": true, - "hideEmail": true, - }; + const privacyModelCtrl = new qx.data.controller.Object(privacyModel); + privacyModelCtrl.addTarget(hideUsername, "value", "hideUsername", true); + privacyModelCtrl.addTarget(hideFullname, "value", "hideFullname", true); + privacyModelCtrl.addTarget(hideEmail, "value", "hideEmail", true); - const model = this.__userPrivacyModel = qx.data.marshal.Json.createModel(raw); - const controller = new qx.data.controller.Object(model); - controller.addTarget(hideFullname, "value", "hideFullname", true); - controller.addTarget(hideEmail, "value", "hideEmail", true); - - const privacyBtn = new qx.ui.form.Button("Update Privacy").set({ + const updatePrivacyBtn = this.__updatePrivacyBtn = new qx.ui.form.Button().set({ + label: this.tr("Update Privacy"), appearance: "form-button", alignX: "right", - allowGrowX: false + allowGrowX: false, + enabled: false, }); - box.add(privacyBtn); - privacyBtn.addListener("execute", () => { + box.add(updatePrivacyBtn); + updatePrivacyBtn.addListener("execute", () => { if (!osparc.data.Permissions.getInstance().canDo("user.user.update", true)) { this.__resetPrivacyData(); return; @@ -266,11 +307,27 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { const patchData = { "privacy": {} }; - if (this.__userPrivacyData["hideFullname"] !== model.getHideFullname()) { - patchData["privacy"]["hideFullname"] = model.getHideFullname(); + if (this.__userPrivacyData["hideUsername"] !== privacyModel.getHideUsername()) { + patchData["privacy"]["hideUsername"] = privacyModel.getHideUsername(); + } + if (this.__userPrivacyData["hideFullname"] !== privacyModel.getHideFullname()) { + patchData["privacy"]["hideFullname"] = privacyModel.getHideFullname(); } - if (this.__userPrivacyData["hideEmail"] !== model.getHideEmail()) { - patchData["privacy"]["hideEmail"] = model.getHideEmail(); + if (this.__userPrivacyData["hideEmail"] !== privacyModel.getHideEmail()) { + patchData["privacy"]["hideEmail"] = privacyModel.getHideEmail(); + } + + if ( + "hideFullname" in patchData["privacy"] && + patchData["privacy"]["hideFullname"] === false && + this.__userProfileData["first_name"] === null + ) { + this.__userProfileForm.getItem("firstName").set({ + invalidMessage: qx.locale.Manager.tr("Name is required"), + valid: false + }); + osparc.FlashMessenger.logAs(this.tr("Set the Name first"), "WARNING"); + return; } if (Object.keys(patchData["privacy"]).length) { @@ -281,22 +338,50 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { .then(() => { this.__setDataToPrivacy(Object.assign(this.__userPrivacyData, params.data["privacy"])); const msg = this.tr("Privacy updated"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); }) .catch(err => { this.__resetPrivacyData(); - const msg = err.message || this.tr("Failed to update privacy"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - console.error(err); + osparc.FlashMessenger.logError(err, this.tr("Unsuccessful privacy update")); }); } }); + const optOutMessage = new qx.ui.basic.Atom().set({ + label: this.tr("If all searchable fields are hidden, you will not be discoverable."), + icon: "@FontAwesome5Solid/exclamation-triangle/14", + gap: 8, + allowGrowX: false, + }); + optOutMessage.getChildControl("icon").setTextColor("warning-yellow") + box.add(optOutMessage); + + const privacyFields = [ + hideUsername, + hideFullname, + hideEmail, + ] + const valueChanged = () => { + const anyChanged = + hideUsername.getValue() !== this.__userPrivacyData["hideUsername"] || + hideFullname.getValue() !== this.__userPrivacyData["hideFullname"] || + hideEmail.getValue() !== this.__userPrivacyData["hideEmail"]; + updatePrivacyBtn.setEnabled(anyChanged); + + if (privacyFields.every(privacyField => privacyField.getValue())) { + optOutMessage.show(); + } else { + optOutMessage.exclude(); + } + }; + valueChanged(); + privacyFields.forEach(privacyField => privacyField.addListener("changeValue", () => valueChanged())); + return box; }, __create2FASection: function() { - const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("2 Factor Authentication")); + const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("Two-Factor Authentication")); const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("Set your preferred method to use for two-factor authentication when signing in:")); box.add(label); @@ -334,7 +419,7 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { if (selectedId === "DISABLED") { const discourageTitle = this.tr("You are about to disable the 2FA"); const discourageText = this.tr("\ - The 2 Factor Authentication is one more measure to prevent hackers from accessing your account with an additional layer of security. \ + The Two-Factor Authentication is one more measure to prevent hackers from accessing your account with an additional layer of security. \ When you sign in, 2FA helps make sure that your resources and personal information stays private, safe and secure.\ "); const win = new osparc.ui.window.Confirmation(discourageTitle).set({ @@ -431,8 +516,8 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { }); }) .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Failed to reset password"), "ERROR"); + const msg = this.tr("Unsuccessful password reset"); + osparc.FlashMessenger.logError(err, msg); [currentPassword, newPassword, confirm].forEach(item => { item.resetValue(); }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/AutoRecharge.js b/services/static-webserver/client/source/class/osparc/desktop/credits/AutoRecharge.js index 2265aff3e9a..c5609b789e0 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/AutoRecharge.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/AutoRecharge.js @@ -290,7 +290,7 @@ qx.Class.define("osparc.desktop.credits.AutoRecharge", { .then(arData => { this.__populateForm(arData); wallet.setAutoRecharge(arData); - osparc.FlashMessenger.getInstance().logAs(successfulMsg, "INFO"); + osparc.FlashMessenger.logAs(successfulMsg, "INFO"); this.fireEvent("close"); }) .finally(() => fetchButton.setFetching(false)); diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/BillingCenter.js b/services/static-webserver/client/source/class/osparc/desktop/credits/BillingCenter.js index d27acc99fe0..3093df0cd7d 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/BillingCenter.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/BillingCenter.js @@ -24,7 +24,7 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", { const miniWallet = this.self().createMiniWalletView().set({ paddingRight: 10 }); - this.addWidgetOnTopOfTheTabs(miniWallet); + this.addWidgetToTabs(miniWallet); this.__walletsPage = this.__addWalletsPage(); this.__paymentMethodsPage = this.__addPaymentMethodsPage(); @@ -36,22 +36,33 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", { if (osparc.product.Utils.showS4LStore()) { this.__addPurchasesPage(); - this.__addCheckoutsPage(); + // For now, do not add checkouts page + // this.__addCheckoutsPage(); } }, statics: { createMiniWalletView: function() { - const layout = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)).set({ + const layout = new qx.ui.container.Composite(new qx.ui.layout.VBox(6)).set({ alignX: "center", minWidth: 120, maxWidth: 150 }); - const store = osparc.store.Store.getInstance(); + const walletName = new qx.ui.basic.Label().set({ + alignX: "center" + }); + layout.add(walletName); const creditsIndicator = new osparc.desktop.credits.CreditsIndicator(); - store.bind("contextWallet", creditsIndicator, "wallet"); layout.add(creditsIndicator); + const store = osparc.store.Store.getInstance(); + store.bind("contextWallet", walletName, "value", { + converter: wallet => wallet.getName() + }); + store.bind("contextWallet", walletName, "toolTipText", { + converter: wallet => wallet.getName() + }); + store.bind("contextWallet", creditsIndicator, "wallet"); layout.add(new qx.ui.core.Spacer(15, 15)); @@ -102,7 +113,7 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", { __addPurchasesPage: function() { const title = this.tr("Purchases"); - const iconSrc = "@FontAwesome5Solid/list/22"; + const iconSrc = "@FontAwesome5Solid/shopping-bag/22"; const purchases = new osparc.desktop.credits.Purchases(); const page = this.addTab(title, iconSrc, purchases); return page; diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsInput.js b/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsInput.js index 25a7086599d..3e8a3a30717 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsInput.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsInput.js @@ -57,17 +57,18 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsInput", { paddingRight: 0 }); this.__totalInput = totalInput; - amountInput.addListener("changeValue", e => { - const value = e.getData(); + const amountChanged = value => { totalInput.setValue(value ? 1 * (value * this.__pricePerCredit).toFixed(2) + this.__currencySymbol : "-"); this.fireDataEvent("input", this.getValues()); - }); + } + amountInput.getChildControl("textfield").addListener("input", e => amountChanged(e.getData())); + amountInput.addListener("changeValue", e => amountChanged(e.getData())); this._add(totalContainer); osparc.store.Store.getInstance().getMinimumAmount() .then(minimum => { amountInput.set({ - maximum: 10000, + maximum: 100000, minimum: Math.ceil(minimum/this.__pricePerCredit), value: Math.ceil(minimum/this.__pricePerCredit) }); @@ -81,7 +82,7 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsInput", { const input = new qx.ui.form.TextField().set({ appearance: "appmotion-buy-credits-input", textAlign: "center", - width: 90, + width: 100, ...inputProps }); const label = new qx.ui.basic.Label(labelText); @@ -96,7 +97,7 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsInput", { })); const input = new qx.ui.form.Spinner().set({ appearance: "appmotion-buy-credits-spinner", - width: 100, + width: 110, ...inputProps }); input.getChildControl("textfield").set({ diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsStepper.js b/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsStepper.js index f569fb05b34..9789fd0fbc6 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsStepper.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/BuyCreditsStepper.js @@ -59,8 +59,7 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsStepper", { }); }) .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); this.__form.setFetching(false); }) } else { @@ -79,10 +78,7 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsStepper", { } }); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.__form.setFetching(false)); } }); @@ -95,14 +91,14 @@ qx.Class.define("osparc.desktop.credits.BuyCreditsStepper", { const msg = this.tr("Payment ") + osparc.utils.Utils.onlyFirstsUp(paymentData.completedStatus); switch (paymentData.completedStatus) { case "SUCCESS": - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); + osparc.FlashMessenger.logAs(msg, "INFO"); break; case "PENDING": - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); break; case "CANCELED": case "FAILED": - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logError(msg); break; default: console.error("completedStatus unknown"); diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTable.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTable.js index a757d6e3f2f..b506beca86d 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTable.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTable.js @@ -80,7 +80,7 @@ qx.Class.define("osparc.desktop.credits.CheckoutsTable", { id: "user", column: 6, label: qx.locale.Manager.tr("User"), - width: 100 + width: 150 }, } } diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTableModel.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTableModel.js index fe59f001d04..8e25d5740b7 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTableModel.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CheckoutsTableModel.js @@ -122,19 +122,16 @@ qx.Class.define("osparc.desktop.credits.CheckoutsTableModel", { return Promise.all([ licensedItemsStore.getLicensedItems(), licensedItemsStore.getCheckedOutLicensedItems(walletId, urlParams), - licensedItemsStore.getVipModels(), ]) .then(values => { const licensedItems = values[0]; const checkoutsItems = values[1]; - const vipModels = values[2]; const data = []; const checkoutsCols = osparc.desktop.credits.CheckoutsTable.COLS; checkoutsItems.forEach(checkoutsItem => { const licensedItemId = checkoutsItem["licensedItemId"]; - const licensedItem = licensedItems.find(licItem => licItem["licensedItemId"] === licensedItemId); - const vipModel = vipModels.find(vipMdl => licensedItem && (vipMdl["modelId"] == licensedItem["name"])); + const licensedItem = licensedItems[licensedItemId]; let start = ""; let duration = ""; if (checkoutsItem["startedAt"]) { @@ -146,11 +143,11 @@ qx.Class.define("osparc.desktop.credits.CheckoutsTableModel", { data.push({ [checkoutsCols.CHECKOUT_ID.id]: checkoutsItem["licensedItemCheckoutId"], [checkoutsCols.ITEM_ID.id]: licensedItemId, - [checkoutsCols.ITEM_LABEL.id]: vipModel ? vipModel["name"] : "unknown model", + [checkoutsCols.ITEM_LABEL.id]: licensedItem ? licensedItem.getDisplayName() : "unknown model", [checkoutsCols.START.id]: start, [checkoutsCols.DURATION.id]: duration, [checkoutsCols.SEATS.id]: checkoutsItem["numOfSeats"], - [checkoutsCols.USER.id]: checkoutsItem["userId"], + [checkoutsCols.USER.id]: checkoutsItem["userEmail"], }); }); return data; diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsIndicatorButton.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsIndicatorButton.js index 37092f5fd47..25ae3083b79 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsIndicatorButton.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsIndicatorButton.js @@ -84,11 +84,10 @@ qx.Class.define("osparc.desktop.credits.CreditsIndicatorButton", { }, __handleOutsideEvent: function(event) { - const offset = 30; - if ( - !osparc.utils.Utils.isMouseOnElement(this.__creditsContainer, event, offset) && - !osparc.utils.Utils.isMouseOnElement(this, event, offset) - ) { + const offset = 0; + const onContainer = osparc.utils.Utils.isMouseOnElement(this.__creditsContainer, event, offset); + const onButton = osparc.utils.Utils.isMouseOnElement(this, event, offset); + if (!onContainer && !onButton) { this.__hideCreditsContainer(); } }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsPerService.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsPerService.js index 182924d225b..e08659fa8f3 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsPerService.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsPerService.js @@ -89,8 +89,9 @@ qx.Class.define("osparc.desktop.credits.CreditsPerService", { this._add(uiEntry); }); } else { - const nothingFound = new qx.ui.basic.Label(this.tr("No usage found")).set({ - font: "text-14" + const nothingFound = new qx.ui.basic.Label(this.tr("No usage records found")).set({ + font: "text-14", + padding: 20, }); this._add(nothingFound); } diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsServiceListItem.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsServiceListItem.js index b12da620b97..0e5181f5631 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsServiceListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsServiceListItem.js @@ -31,12 +31,13 @@ qx.Class.define("osparc.desktop.credits.CreditsServiceListItem", { const icon = this.getChildControl("icon"); const name = this.getChildControl("title"); - const serviceMetadata = osparc.service.Utils.getLatest(serviceKey); + const serviceMetadata = osparc.store.Services.getLatest(serviceKey); if (serviceMetadata) { - icon.setSource(serviceMetadata["thumbnail"] ? serviceMetadata["thumbnail"] : osparc.dashboard.CardBase.PRODUCT_ICON); + const source = osparc.utils.Utils.getIconFromResource(serviceMetadata); + icon.setSource(source); name.setValue(serviceMetadata["name"]); } else { - icon.setSource(osparc.dashboard.CardBase.PRODUCT_ICON); + icon.setSource(osparc.dashboard.CardBase.PRODUCT_THUMBNAIL); const serviceName = serviceKey.split("/").pop(); name.setValue(serviceName); } diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsSummary.js b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsSummary.js index 45461b39b9f..b6fa37b07ac 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsSummary.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/CreditsSummary.js @@ -27,6 +27,7 @@ qx.Class.define("osparc.desktop.credits.CreditsSummary", { appearance: "floating-menu", padding: 8, maxWidth: this.self().WIDTH, + minHeight: 150, zIndex: osparc.utils.Utils.FLOATING_Z_INDEX, }); osparc.utils.Utils.setIdToWidget(this, "creditsSummary"); diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTable.js b/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTable.js index 8cd31c0287c..8eeb4d7a28b 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTable.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTable.js @@ -87,7 +87,7 @@ qx.Class.define("osparc.desktop.credits.PurchasesTable", { id: "user", column: 7, label: qx.locale.Manager.tr("User"), - width: 100 + width: 150 }, } } diff --git a/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTableModel.js b/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTableModel.js index b1b054071ce..97502433ff2 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTableModel.js +++ b/services/static-webserver/client/source/class/osparc/desktop/credits/PurchasesTableModel.js @@ -121,28 +121,25 @@ qx.Class.define("osparc.desktop.credits.PurchasesTableModel", { return Promise.all([ licensedItemsStore.getLicensedItems(), licensedItemsStore.getPurchasedLicensedItems(walletId, urlParams), - licensedItemsStore.getVipModels(), ]) .then(values => { const licensedItems = values[0]; const purchasesItems = values[1]; - const vipModels = values[2]; const data = []; const purchasesCols = osparc.desktop.credits.PurchasesTable.COLS; purchasesItems.forEach(purchasesItem => { const licensedItemId = purchasesItem["licensedItemId"]; - const licensedItem = licensedItems.find(licItem => licItem["licensedItemId"] === licensedItemId); - const vipModel = vipModels.find(vipMdl => licensedItem && (vipMdl["modelId"] == licensedItem["name"])); + const licensedItem = licensedItems[licensedItemId]; data.push({ [purchasesCols.PURCHASE_ID.id]: purchasesItem["licensedItemPurchaseId"], [purchasesCols.ITEM_ID.id]: licensedItemId, - [purchasesCols.ITEM_LABEL.id]: vipModel ? vipModel["name"] : "unknown model", + [purchasesCols.ITEM_LABEL.id]: licensedItem ? licensedItem.getDisplayName() : "unknown model", [purchasesCols.START.id]: osparc.utils.Utils.formatDateAndTime(new Date(purchasesItem["startAt"])), [purchasesCols.END.id]: osparc.utils.Utils.formatDateAndTime(new Date(purchasesItem["expireAt"])), [purchasesCols.SEATS.id]: purchasesItem["numOfSeats"], [purchasesCols.COST.id]: purchasesItem["pricingUnitCost"] ? ("-" + parseFloat(purchasesItem["pricingUnitCost"]).toFixed(2)) : "", // show it negative - [purchasesCols.USER.id]: purchasesItem["purchasedByUser"], + [purchasesCols.USER.id]: purchasesItem["userEmail"], }); }); return data; diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js index 2f0a016a8df..c7f8ca40578 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js @@ -243,8 +243,8 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const canIDelete = organization.getAccessRights()["delete"]; const introText = canIWrite ? - this.tr("You can add new members and change their roles.") : - this.tr("You can't add new members to this Organization. Please contact an Administrator or Manager."); + this.tr("You can add new members and assign roles.") : + this.tr("You cannot add new members to this Organization. Please contact an Administrator or Manager."); this.__introLabel.setValue(introText); this.__addMembersButton.set({ @@ -342,17 +342,13 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { request .then(newMember => { const text = newMemberIdentifier + this.tr(" successfully added"); - osparc.FlashMessenger.getInstance().logAs(text); + osparc.FlashMessenger.logAs(text); this.__reloadOrgMembers(); // push 'NEW_ORGANIZATION' notification osparc.notification.Notifications.postNewOrganization(newMember.getUserId(), orgId); }) - .catch(err => { - const errorMessage = err["message"] || this.tr("Something went wrong adding the user"); - osparc.FlashMessenger.getInstance().logAs(errorMessage, "ERROR"); - console.error(err); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while adding the user"))); }, __promoteToUser: function(listedMember) { @@ -364,12 +360,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.patchMember(this.__currentOrg.getGroupId(), listedMember["id"], newAccessRights) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[1].label}`)); + osparc.FlashMessenger.logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[1].label}`)); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[1].label, "ERROR"); - console.error(err); + const msg = this.tr("Something went wrong while promoting to ") + osparc.data.Roles.ORG[1].label; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -387,12 +383,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { if (msg === undefined) { msg = this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[0].label}`); } - osparc.FlashMessenger.getInstance().logAs(msg); + osparc.FlashMessenger.logAs(msg); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[0].label, "ERROR"); - console.error(err); + const errorMsg = this.tr("Something went wrong while demoting to ") + osparc.data.Roles.ORG[0].label; + osparc.FlashMessenger.logError(err, errorMsg); }); }, @@ -407,12 +403,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.patchAccessRights(orgId, userId, newAccessRights) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[2].label}`)); + osparc.FlashMessenger.logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[2].label}`)); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[2].label, "ERROR"); - console.error(err); + const msg = this.tr("Something went wrong while promoting to ") + osparc.data.Roles.ORG[2].label; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -427,12 +423,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.patchAccessRights(orgId, userId, newAccessRights) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[3].label}`)); + osparc.FlashMessenger.logAs(this.tr(`Successfully promoted to ${osparc.data.Roles.ORG[3].label}`)); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong promoting to ") + osparc.data.Roles.ORG[3].label, "ERROR"); - console.error(err); + const msg = this.tr("Something went wrong while promoting to ") + osparc.data.Roles.ORG[3].label; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -447,12 +443,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.patchAccessRights(orgId, userId, newAccessRights) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[1].label}`)); + osparc.FlashMessenger.logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[1].label}`)); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[1].label, "ERROR"); - console.error(err); + const msg = this.tr("Something went wrong while demoting to ") + osparc.data.Roles.ORG[1].label; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -467,12 +463,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.patchAccessRights(orgId, userId, newAccessRights) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[3].label}`)); + osparc.FlashMessenger.logAs(this.tr(`Successfully demoted to ${osparc.data.Roles.ORG[3].label}`)); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong demoting to ") + osparc.data.Roles.ORG[3].label, "ERROR"); - console.error(err); + const msg =this.tr("Something went wrong while demoting to ") + osparc.data.Roles.ORG[3].label; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -480,12 +476,12 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { const groupsStore = osparc.store.Groups.getInstance(); return groupsStore.removeMember(this.__currentOrg.getGroupId(), listedMember["id"]) .then(() => { - osparc.FlashMessenger.getInstance().logAs(listedMember["name"] + this.tr(" successfully removed")); + osparc.FlashMessenger.logAs(listedMember["name"] + this.tr(" successfully removed")); this.__reloadOrgMembers(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + listedMember["name"], "ERROR"); - console.error(err); + const msg = this.tr("Something went wrong while removing ") + listedMember["name"]; + osparc.FlashMessenger.logError(err, msg); }); }, @@ -512,7 +508,7 @@ qx.Class.define("osparc.desktop.organizations.MembersList", { } else if (isThereAnyManager) { rUSure += `
There is no ${osparc.data.Roles.ORG[3].label} in this Organization.`; } - rUSure += "
" + this.tr("If you Leave, the page will be reloaded."); + rUSure += "
" + this.tr("If you leave, the page will reload."); const confirmationWin = new osparc.ui.window.Confirmation(rUSure).set({ caption: this.tr("Leave Organization"), confirmText: this.tr("Leave"), diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js index 8b3781d7a12..bfc757d2880 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationDetails.js @@ -71,7 +71,7 @@ qx.Class.define("osparc.desktop.organizations.OrganizationDetails", { const titleLayout = this.__titleLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox(5)); const prevBtn = new qx.ui.form.Button().set({ - toolTipText: this.tr("Back to Organizations list"), + toolTipText: this.tr("Return to Organizations list"), icon: "@FontAwesome5Solid/arrow-left/20", backgroundColor: "transparent" }); @@ -115,14 +115,14 @@ qx.Class.define("osparc.desktop.organizations.OrganizationDetails", { const thumbnail = orgEditor.getThumbnail(); osparc.store.Groups.getInstance().patchOrganization(groupId, name, description, thumbnail) .then(() => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully edited")); + osparc.FlashMessenger.logAs(name + this.tr(" successfully edited")); button.setFetching(false); win.close(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong editing ") + name, "ERROR"); + const msg = this.tr("Something went wrong while editing ") + name; + osparc.FlashMessenger.logError(err, msg); button.setFetching(false); - console.error(err); }); }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js index c86917fed36..dd488f9b328 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/OrganizationsList.js @@ -234,8 +234,8 @@ qx.Class.define("osparc.desktop.organizations.OrganizationsList", { this.reloadOrganizations(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong deleting ") + name, "ERROR"); - console.error(err); + const errorMsg = this.tr("Something went wrong while deleting ") + name; + osparc.FlashMessenger.logError(err, errorMsg); }) .finally(() => { win.close(); @@ -251,18 +251,16 @@ qx.Class.define("osparc.desktop.organizations.OrganizationsList", { const groupsStore = osparc.store.Groups.getInstance(); groupsStore.postOrganization(name, description, thumbnail) .then(org => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully created")); - button.setFetching(false); + osparc.FlashMessenger.logAs(name + this.tr(" successfully created")); // open it this.reloadOrganizations(org.getGroupId()); }) .catch(err => { - const errorMessage = err["message"] || this.tr("Something went wrong creating ") + name; - osparc.FlashMessenger.getInstance().logAs(errorMessage, "ERROR"); - button.setFetching(false); - console.error(err); + const msg = this.tr("Something went wrong while creating ") + name; + osparc.FlashMessenger.logError(err, msg); }) .finally(() => { + button.setFetching(false); win.close(); }); }, @@ -274,14 +272,14 @@ qx.Class.define("osparc.desktop.organizations.OrganizationsList", { const thumbnail = orgEditor.getThumbnail(); osparc.store.Groups.getInstance().patchOrganization(groupId, name, description, thumbnail) .then(() => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully edited")); + osparc.FlashMessenger.logAs(name + this.tr(" successfully edited")); button.setFetching(false); win.close(); }) .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong editing ") + name, "ERROR"); + const msg = this.tr("Something went wrong while editing ") + name; + osparc.FlashMessenger.logError(err, msg); button.setFetching(false); - console.error(err); }); } } diff --git a/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js b/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js index fec8d6d85e6..2f0635eba58 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/organizations/ServicesList.js @@ -128,7 +128,7 @@ qx.Class.define("osparc.desktop.organizations.ServicesList", { const orgServiceCopy = osparc.utils.Utils.deepCloneObject(orgService); orgServiceCopy["orgId"] = groupId; if (orgServiceCopy["thumbnail"] === null) { - orgServiceCopy["thumbnail"] = osparc.dashboard.CardBase.PRODUCT_ICON; + orgServiceCopy["thumbnail"] = osparc.dashboard.CardBase.PRODUCT_THUMBNAIL; } servicesModel.append(qx.data.marshal.Json.createModel(orgServiceCopy)); }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js index 49dbab989ed..936f032db04 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js @@ -47,7 +47,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodDetails", { members: { __buildLayout: function(paymentMethodData) { [ - [this.tr("Holder name"), paymentMethodData["cardHolderName"]], + [this.tr("Card Holder name"), paymentMethodData["cardHolderName"]], [this.tr("Type"), paymentMethodData["cardType"]], [this.tr("Number"), paymentMethodData["cardNumberMasked"]], [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]] diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js index 9342f10f066..90c3348826e 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodListItem.js @@ -182,7 +182,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodListItem", { }, __deletePressed: function() { - const msg = this.tr("Are you sure you want to delete the Payment Method?"); + const msg = this.tr("Are you sure you want to delete this Payment Method?"); const win = new osparc.ui.window.Confirmation(msg).set({ caption: this.tr("Delete Payment Method"), confirmText: this.tr("Delete"), diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethods.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethods.js index fa90ceddbe2..14b64b3a8ea 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethods.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethods.js @@ -118,7 +118,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethods", { __windowClosed: function(paymentMethodId) { const msg = this.tr("The window was closed. Try again and follow the instructions inside the opened window."); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); this.__cancelPaymentMethod(paymentMethodId); }, @@ -165,11 +165,8 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethods", { }) .finally(() => this.__fetchingMsg.setVisibility("excluded")) .catch(err => { - console.error(err) - osparc.FlashMessenger.getInstance().logAs( - this.tr("We could not retrieve your saved payment methods. Please try again later."), - "ERROR" - ); + const msg = this.tr("Could not retrieve your saved payment methods. Please try again later."); + osparc.FlashMessenger.logError(err, msg); }); }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/Preferences.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/Preferences.js deleted file mode 100644 index 31f8de5590b..00000000000 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/Preferences.js +++ /dev/null @@ -1,73 +0,0 @@ -/* ************************************************************************ - - osparc - the simcore frontend - - https://osparc.io - - Copyright: - 2018 IT'IS Foundation, https://itis.swiss - - License: - MIT: https://opensource.org/licenses/MIT - - Authors: - * Odei Maiz (odeimaiz) - -************************************************************************ */ - -qx.Class.define("osparc.desktop.preferences.Preferences", { - extend: osparc.ui.window.TabbedView, - - construct: function() { - this.base(arguments); - - this.__addGeneralSettings(); - this.__addConfirmationSettings(); - if (osparc.product.Utils.showPreferencesTokens()) { - this.__addTokensPage(); - } - if (osparc.data.Permissions.getInstance().canDo("user.tag")) { - this.__addTagsPage(); - } - }, - - members: { - __tagsPage: null, - - __addGeneralSettings: function() { - const title = this.tr("General Settings"); - const iconSrc = "@FontAwesome5Solid/cogs/22"; - const generalPage = new osparc.desktop.preferences.pages.GeneralPage(); - this.addTab(title, iconSrc, generalPage); - }, - - __addConfirmationSettings: function() { - const title = this.tr("Confirmation Settings"); - const iconSrc = "@FontAwesome5Solid/question-circle/22"; - const confirmPage = new osparc.desktop.preferences.pages.ConfirmationsPage(); - this.addTab(title, iconSrc, confirmPage); - }, - - __addTokensPage: function() { - const title = this.tr("API Keys/Tokens"); - const iconSrc = "@FontAwesome5Solid/exchange-alt/22"; - const tokensPage = new osparc.desktop.preferences.pages.TokensPage(); - this.addTab(title, iconSrc, tokensPage); - }, - - __addTagsPage: function() { - const title = this.tr("Create/Edit Tags"); - const iconSrc = "@FontAwesome5Solid/tags/22"; - const tagsPage = new osparc.desktop.preferences.pages.TagsPage(); - const page = this.__tagsPage = this.addTab(title, iconSrc, tagsPage); - osparc.utils.Utils.setIdToWidget(page.getChildControl("button"), "preferencesTagsTabBtn"); - }, - - openTags: function() { - if (this.__tagsPage) { - this._openPage(this.__tagsPage); - return true; - } - }, - } -}); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/PreferencesWindow.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/PreferencesWindow.js deleted file mode 100644 index 889a1e54c2e..00000000000 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/PreferencesWindow.js +++ /dev/null @@ -1,54 +0,0 @@ -/* ************************************************************************ - - osparc - the simcore frontend - - https://osparc.io - - Copyright: - 2018 IT'IS Foundation, https://itis.swiss - - License: - MIT: https://opensource.org/licenses/MIT - - Authors: - * Pedro Crespo (pcrespov) - -************************************************************************ */ - -qx.Class.define("osparc.desktop.preferences.PreferencesWindow", { - extend: osparc.ui.window.TabbedWindow, - - construct: function() { - this.base(arguments, "preferences", this.tr("Preferences")); - - const closeBtn = this.getChildControl("close-button"); - osparc.utils.Utils.setIdToWidget(closeBtn, "preferencesWindowCloseBtn"); - - const width = 750; - const height = 660; - this.set({ - width, - height - }); - - const preferences = this.__preferences = new osparc.desktop.preferences.Preferences(); - this._setTabbedView(preferences); - }, - - statics: { - openWindow: function() { - const preferencesWindow = new osparc.desktop.preferences.PreferencesWindow(); - preferencesWindow.center(); - preferencesWindow.open(); - return preferencesWindow; - } - }, - - members: { - __preferences: null, - - openTags: function() { - return this.__preferences.openTags(); - } - } -}); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/BasePage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/BasePage.js index a32ec216d70..449546236ca 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/BasePage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/BasePage.js @@ -30,7 +30,17 @@ qx.Class.define("osparc.desktop.preferences.pages.BasePage", { paddingLeft: 15 }); - this.__showLabelOnTab(title) + this.__showLabelOnTab(title); + + const tabButton = this.getChildControl("button"); + if (tabButton.getIcon() && tabButton.getIcon().includes(".svg")) { + tabButton.getChildControl("icon").set({ + minWidth: 24, + minHeight: 24, + scale: true, + }); + osparc.ui.basic.SVGImage.setColorToImage(tabButton.getChildControl("icon"), "text"); + } }, members: { diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ConfirmationsPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ConfirmationsPage.js index 7b9dc943e63..c883c6bd37f 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ConfirmationsPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/ConfirmationsPage.js @@ -114,7 +114,7 @@ qx.Class.define("osparc.desktop.preferences.pages.ConfirmationsPage", { cbConfirmStopNode.addListener("changeValue", e => osparc.Preferences.patchPreferenceField("confirmStopNode", cbConfirmStopNode, e.getData())); box.add(cbConfirmStopNode); - const cbSnapNodeToGrid = new qx.ui.form.CheckBox(this.tr("Snap Node to grid")); + const cbSnapNodeToGrid = new qx.ui.form.CheckBox(this.tr("Snap Node to Grid")); preferencesSettings.bind("snapNodeToGrid", cbSnapNodeToGrid, "value"); cbSnapNodeToGrid.addListener("changeValue", e => osparc.Preferences.patchPreferenceField("snapNodeToGrid", cbSnapNodeToGrid, e.getData())); box.add(cbSnapNodeToGrid); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/GeneralPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/GeneralPage.js index 70095507e49..6f02c5a65b9 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/GeneralPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/GeneralPage.js @@ -137,7 +137,7 @@ qx.Class.define("osparc.desktop.preferences.pages.GeneralPage", { __addLowDiskSpaceSetting: function() { const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("Low Disk Space Threshold")); - const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("Set the warning Threshold for low Disk Space availability."), "text-13-italic"); + const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("Set the warning Threshold for Low Disk Space availability"), "text-13-italic"); box.add(label); const form = new qx.ui.form.Form(); const diskUsageSpinner = new qx.ui.form.Spinner().set({ diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TagsPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TagsPage.js index f69ae38e4ee..6ef62565d05 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TagsPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TagsPage.js @@ -26,64 +26,63 @@ qx.Class.define("osparc.desktop.preferences.pages.TagsPage", { this._add(new qx.ui.core.Spacer(null, 10)); - this.__container = new qx.ui.container.Composite(new qx.ui.layout.VBox(10)); - this.__container.set({ - paddingLeft: 10 - }); - const scroll = new qx.ui.container.Scroll(this.__container); - this._add(scroll); - - this.__createComponents(); + this.__renderLayout(); }, members: { - __container: null, - __addTagButton: null, - __tagItems: null, + __tagsContainer: null, + + __renderLayout: async function() { + // Tags + this.__tagsContainer = new qx.ui.container.Composite(new qx.ui.layout.VBox(10)); + this.__tagsContainer.set({ + paddingLeft: 10 + }); + const tagContainerScroll = new qx.ui.container.Scroll(this.__tagsContainer); + this._add(tagContainerScroll, { + flex: 1 + }); - __createComponents: function() { - this.__addTagButton = new qx.ui.form.Button().set({ + const tags = osparc.store.Tags.getInstance().getTags(); + for (const tag of tags) { + await osparc.store.Tags.getInstance().fetchAccessRights(tag); + } + const tagItems = tags.map(tag => new osparc.form.tag.TagItem().set({tag})); + tagItems.forEach(tagItem => { + this.__tagsContainer.add(tagItem); + this.__attachTagItemEvents(tagItem); + }); + + // New tag Button + const addTagButton = new qx.ui.form.Button().set({ appearance: "form-button-outlined", label: this.tr("New Tag"), icon: "@FontAwesome5Solid/plus/14" }); - osparc.utils.Utils.setIdToWidget(this.__addTagButton, "addTagBtn"); - const tags = osparc.store.Tags.getInstance().getTags(); - this.__tagItems = tags.map(tag => new osparc.form.tag.TagItem().set({tag})); - this.__renderLayout(); - this.__attachEventHandlers(); - }, - - __renderLayout: function() { - this.__container.removeAll(); + osparc.utils.Utils.setIdToWidget(addTagButton, "addTagBtn"); + addTagButton.addListener("execute", () => { + const newItem = new osparc.form.tag.TagItem().set({ + mode: osparc.form.tag.TagItem.modes.EDIT + }); + this.__tagsContainer.add(newItem); + this.__attachTagItemEvents(newItem); - // Print tag items - this.__tagItems.forEach(tagItem => this.__container.add(tagItem)); + // scroll down + const height = tagContainerScroll.getSizeHint().height; + tagContainerScroll.scrollToY(height); + }); // New tag button const buttonContainer = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ alignX: "center" })); - buttonContainer.add(new qx.ui.core.Spacer(null, 10)); - buttonContainer.add(this.__addTagButton); - this.__container.add(buttonContainer); - }, - - __attachEventHandlers: function() { - this.__addTagButton.addListener("execute", () => { - const itemCount = this.__container.getChildren().length; - const newItem = new osparc.form.tag.TagItem().set({ - mode: osparc.form.tag.TagItem.modes.EDIT - }); - this.__attachTagItemEvents(newItem); - this.__container.addAt(newItem, Math.max(0, itemCount - 1)); - }); - this.__tagItems.forEach(tagItem => this.__attachTagItemEvents(tagItem)); + buttonContainer.add(addTagButton); + this._add(buttonContainer); }, __attachTagItemEvents: function(tagItem) { - tagItem.addListener("cancelNewTag", e => this.__container.remove(e.getTarget()), this); - tagItem.addListener("deleteTag", e => this.__container.remove(e.getTarget())); + tagItem.addListener("cancelNewTag", e => this.__tagsContainer.remove(e.getTarget()), this); + tagItem.addListener("deleteTag", e => this.__tagsContainer.remove(e.getTarget())); } } }); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js index 6d5ae0e5258..b22542d6d4d 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/pages/TokensPage.js @@ -97,11 +97,7 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { showAPIKeyWindow.center(); showAPIKeyWindow.open(); }) - .catch(err => { - const errorMsg = err.message || this.tr("Cannot create API Key"); - osparc.FlashMessenger.getInstance().logAs(errorMsg, "ERROR"); - console.error(err); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Cannot create API Key"))) .finally(() => this.__requestAPIKeyBtn.setFetching(false)); }, this); createAPIKeyWindow.open(); @@ -160,11 +156,7 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { }; osparc.data.Resources.fetch("apiKeys", "delete", params) .then(() => this.__rebuildAPIKeysList()) - .catch(err => { - const errorMsg = err.message || this.tr("Cannot delete API Key"); - osparc.FlashMessenger.getInstance().logAs(errorMsg, "ERROR"); - console.error(err) - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Cannot delete API Key"))); } }, this); }, @@ -181,12 +173,12 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { __createTokensSection: function() { // layout - const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("External Service Tokens")); + const box = osparc.ui.window.TabbedView.createSectionBox(this.tr("API Tokens for External Services")); - const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("Enter the API tokens to access external services.")); + const label = osparc.ui.window.TabbedView.createHelpLabel(this.tr("Provide the API tokens needed to access external services.")); box.add(label); - const validTokensGB = this.__validTokensGB = osparc.ui.window.TabbedView.createSectionBox(this.tr("Existing Tokens")); + const validTokensGB = this.__validTokensGB = osparc.ui.window.TabbedView.createSectionBox(this.tr("Current Tokens")); box.add(validTokensGB); const supportedExternalsGB = this.__supportedExternalsGB = osparc.ui.window.TabbedView.createSectionBox(this.tr("Supported services")).set({ @@ -288,7 +280,7 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { return; } - const msg = this.tr("Do you want to delete the Token?"); + const msg = this.tr("Are you sure you want to delete this token?"); const win = new osparc.ui.window.Confirmation(msg).set({ caption: this.tr("Delete Token"), confirmText: this.tr("Delete"), @@ -334,13 +326,13 @@ qx.Class.define("osparc.desktop.preferences.pages.TokensPage", { const newTokenKey = new qx.ui.form.TextField(); newTokenKey.set({ - placeholder: this.tr("Input your token key") + placeholder: this.tr("Enter your token key") }); form.add(newTokenKey, this.tr("Key")); const newTokenSecret = new qx.ui.form.TextField(); newTokenSecret.set({ - placeholder: this.tr("Input your token secret") + placeholder: this.tr("Enter your token secret") }); form.add(newTokenSecret, this.tr("Secret")); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js index 3540f23b518..ca5ba07cd70 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js @@ -17,7 +17,7 @@ qx.Class.define("osparc.desktop.preferences.window.CreateAPIKey", { extend: osparc.desktop.preferences.window.APIKeyBase, construct: function() { - const caption = this.tr("Create API Key"); + const caption = this.tr("Generate API Key"); const infoText = this.tr("Key names must be unique."); this.base(arguments, caption, infoText); @@ -48,8 +48,8 @@ qx.Class.define("osparc.desktop.preferences.window.CreateAPIKey", { if (date) { // allow only future dates if (new Date() > new Date(date)) { - const msg = this.tr("Choose a future date"); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + const msg = this.tr("Select a future date"); + osparc.FlashMessenger.logAs(msg, "WARNING"); expirationDate.resetValue(); } else { expirationDate.setDateFormat(dateFormat); diff --git a/services/static-webserver/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js b/services/static-webserver/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js index 6c7209d3ecc..8154c1f6296 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js +++ b/services/static-webserver/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js @@ -18,7 +18,7 @@ qx.Class.define("osparc.desktop.preferences.window.ShowAPIKey", { construct: function(key, secret, baseUrl) { const caption = this.tr("API Key"); - const infoText = this.tr("For your protection, store your access keys securely and do not share them. You will not be able to access the key again once this window is closed."); + const infoText = this.tr("For your security, store your access keys safely. You will not be able to access them again after closing this window."); this.base(arguments, caption, infoText); this.set({ diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js index 963126fe3f8..90fa5fbfb69 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/MemberListItem.js @@ -40,7 +40,7 @@ qx.Class.define("osparc.desktop.wallets.MemberListItem", { // highlight me const email = osparc.auth.Data.getInstance().getEmail(); - if (email === value) { + if (value && value.includes(email)) { this.addState("selected"); } }, diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletDetails.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletDetails.js index a1dde9756de..d55f6ebd35a 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletDetails.js @@ -66,7 +66,7 @@ qx.Class.define("osparc.desktop.wallets.WalletDetails", { const titleLayout = this.__titleLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox(5)); const prevBtn = new qx.ui.form.Button().set({ - toolTipText: this.tr("Back to Credit Accounts list"), + toolTipText: this.tr("Return to Credit Accounts list"), icon: "@FontAwesome5Solid/arrow-left/20", backgroundColor: "transparent" }); @@ -120,14 +120,13 @@ qx.Class.define("osparc.desktop.wallets.WalletDetails", { }; osparc.data.Resources.fetch("wallets", "put", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully edited")); + osparc.FlashMessenger.logAs(name + this.tr(" successfully edited")); const wallet = osparc.desktop.credits.Utils.getWallet(walletId); wallet.set(params.data); }) .catch(err => { - console.error(err); - const msg = err.message || (this.tr("Something went wrong editing ") + name); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const msg = this.tr("Something went wrong while editing ") + name; + osparc.FlashMessenger.logError(err, msg); }) .finally(() => { button.setFetching(false); diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js index 2c2a70a7f07..837de69ffe3 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletListItem.js @@ -81,6 +81,7 @@ qx.Class.define("osparc.desktop.wallets.WalletListItem", { control = new qx.ui.basic.Label().set({ font: "text-14" }); + control.bind("value", control, "toolTipText"); this._add(control, { row: 0, column: 0, @@ -140,9 +141,8 @@ qx.Class.define("osparc.desktop.wallets.WalletListItem", { osparc.data.Resources.fetch("wallets", "put", params) .then(() => found.setStatus(newStatus)) .catch(err => { - console.error(err); - const msg = err.message || (this.tr("Something went wrong updating the state")); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const msg = this.tr("Something went wrong while updating the state"); + osparc.FlashMessenger.logError(err, msg); }); } }, this); diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletsList.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletsList.js index c8f1198eabd..5ed1afa1372 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletsList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/WalletsList.js @@ -199,14 +199,13 @@ qx.Class.define("osparc.desktop.wallets.WalletsList", { }; osparc.data.Resources.fetch("wallets", "put", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully edited")); + osparc.FlashMessenger.logAs(name + this.tr(" successfully edited")); const wallet = osparc.desktop.credits.Utils.getWallet(walletId); wallet.set(params.data); }) .catch(err => { - console.error(err); - const msg = err.message || this.tr("Something went wrong updating the Credit Account"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + const msg = this.tr("Something went wrong while updating the Credit Account"); + osparc.FlashMessenger.logError(err, msg); }) .finally(() => { walletEditor.setIsFetching(false); diff --git a/services/static-webserver/client/source/class/osparc/editor/MarkdownEditor.js b/services/static-webserver/client/source/class/osparc/editor/MarkdownEditor.js index a8785cebc8a..2d08dc0bcf6 100644 --- a/services/static-webserver/client/source/class/osparc/editor/MarkdownEditor.js +++ b/services/static-webserver/client/source/class/osparc/editor/MarkdownEditor.js @@ -31,7 +31,7 @@ qx.Class.define("osparc.editor.MarkdownEditor", { this.getChildControl("preview-markdown"); this.getChildControl("subtitle").set({ - value: this.tr("Supports Markdown") + value: this.tr("Markdown supported") }); }, diff --git a/services/static-webserver/client/source/class/osparc/editor/ThumbnailEditor.js b/services/static-webserver/client/source/class/osparc/editor/ThumbnailEditor.js index 227ecfe61ff..c345b376263 100644 --- a/services/static-webserver/client/source/class/osparc/editor/ThumbnailEditor.js +++ b/services/static-webserver/client/source/class/osparc/editor/ThumbnailEditor.js @@ -65,7 +65,7 @@ qx.Class.define("osparc.editor.ThumbnailEditor", { sanitizeUrl: function(dirty) { const clean = osparc.wrapper.DOMPurify.getInstance().sanitize(dirty); if ((dirty && dirty !== clean) || (clean !== "" && !osparc.utils.Utils.isValidHttpUrl(clean))) { - osparc.FlashMessenger.getInstance().logAs(qx.locale.Manager.tr("Error checking link"), "WARNING"); + osparc.FlashMessenger.logAs(qx.locale.Manager.tr("Error checking link"), "WARNING"); return null; } return clean; diff --git a/services/static-webserver/client/source/class/osparc/editor/ThumbnailSuggestions.js b/services/static-webserver/client/source/class/osparc/editor/ThumbnailSuggestions.js index 188485676ac..c51dace40c5 100644 --- a/services/static-webserver/client/source/class/osparc/editor/ThumbnailSuggestions.js +++ b/services/static-webserver/client/source/class/osparc/editor/ThumbnailSuggestions.js @@ -209,15 +209,18 @@ qx.Class.define("osparc.editor.ThumbnailSuggestions", { this.setSuggestions(this.__thumbnails); }, - thumbnailTapped: function(thumbnail) { + __thumbnailTapped: function(thumbnail) { + // reset decoration + const unselectedBorderColor = qx.theme.manager.Color.getInstance().resolve("text"); + const unselectedBGColor = qx.theme.manager.Color.getInstance().resolve("fab-background"); this.getChildren().forEach(thumbnailImg => { - osparc.utils.Utils.updateBorderColor(thumbnailImg, qx.theme.manager.Color.getInstance().resolve("box-shadow")); - osparc.utils.Utils.addBackground(thumbnailImg, qx.theme.manager.Color.getInstance().resolve("fab-background")); + osparc.utils.Utils.updateBorderColor(thumbnailImg, unselectedBorderColor); + osparc.utils.Utils.addBackground(thumbnailImg, unselectedBGColor); }); - const color = qx.theme.manager.Color.getInstance().resolve("background-selected-dark"); - const bgColor = qx.theme.manager.Color.getInstance().resolve("background-selected"); - osparc.utils.Utils.updateBorderColor(thumbnail, color); - osparc.utils.Utils.addBackground(thumbnail, bgColor); + const selectedBorderColor = qx.theme.manager.Color.getInstance().resolve("strong-main"); + const selectedBGColor = qx.theme.manager.Color.getInstance().resolve("background-selected"); + osparc.utils.Utils.updateBorderColor(thumbnail, selectedBorderColor); + osparc.utils.Utils.addBackground(thumbnail, selectedBGColor); this.fireDataEvent("thumbnailTapped", { type: thumbnail.thumbnailType || "templateThumbnail", source: thumbnail.thumbnailFileUrl || thumbnail.getSource() @@ -238,9 +241,7 @@ qx.Class.define("osparc.editor.ThumbnailSuggestions", { thumbnail.thumbnailFileUrl = suggestion.fileUrl || suggestion; thumbnail.addListener("mouseover", () => thumbnail.set({decorator: "thumbnail-selected"}), this); thumbnail.addListener("mouseout", () => thumbnail.set({decorator: "thumbnail"}), this); - thumbnail.addListener("tap", () => { - this.thumbnailTapped(thumbnail); - }, this); + thumbnail.addListener("tap", () => this.__thumbnailTapped(thumbnail), this); this.add(thumbnail); }); } diff --git a/services/static-webserver/client/source/class/osparc/editor/WorkspaceEditor.js b/services/static-webserver/client/source/class/osparc/editor/WorkspaceEditor.js index 4e151d4e433..ff557ddc59e 100644 --- a/services/static-webserver/client/source/class/osparc/editor/WorkspaceEditor.js +++ b/services/static-webserver/client/source/class/osparc/editor/WorkspaceEditor.js @@ -221,10 +221,7 @@ qx.Class.define("osparc.editor.WorkspaceEditor", { }; osparc.store.Workspaces.getInstance().putWorkspace(this.getWorkspace().getWorkspaceId(), updateData) .then(() => this.fireEvent("workspaceUpdated")) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => editButton.setFetching(false)); } }, @@ -233,10 +230,7 @@ qx.Class.define("osparc.editor.WorkspaceEditor", { if (this.__creatingWorkspace) { osparc.store.Workspaces.getInstance().deleteWorkspace(this.getWorkspace().getWorkspaceId()) .then(() => this.fireEvent("workspaceDeleted")) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err)); } this.fireEvent("cancel"); }, diff --git a/services/static-webserver/client/source/class/osparc/file/FileDownloadLink.js b/services/static-webserver/client/source/class/osparc/file/FileDownloadLink.js index bbb10785079..637257b0c1e 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileDownloadLink.js +++ b/services/static-webserver/client/source/class/osparc/file/FileDownloadLink.js @@ -45,7 +45,7 @@ qx.Class.define("osparc.file.FileDownloadLink", { this.fireDataEvent("fileLinkAdded", downloadLink); } else { downloadLinkField.resetValue(); - osparc.FlashMessenger.getInstance().logAs(this.tr("Error checking link"), "WARNING"); + osparc.FlashMessenger.logAs(this.tr("An issue occurred while checking link"), "WARNING"); } }, this); }, diff --git a/services/static-webserver/client/source/class/osparc/file/FileDrop.js b/services/static-webserver/client/source/class/osparc/file/FileDrop.js index 106a6c66b51..92b4f15206c 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileDrop.js +++ b/services/static-webserver/client/source/class/osparc/file/FileDrop.js @@ -39,7 +39,7 @@ qx.Class.define("osparc.file.FileDrop", { let msg = "
" + qx.locale.Manager.tr("Please zip all files together."), + ONE_FILE_ONLY: qx.locale.Manager.tr("Only one file can be uploaded at a time.") + "
" + qx.locale.Manager.tr("Please compress all files into a single zip file."), }, events: { @@ -301,10 +301,10 @@ qx.Class.define("osparc.file.FileDrop", { pos: this.__pointerFileEventToScreenPos(e) }); } else { - osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); + osparc.FlashMessenger.logError(osparc.file.FileDrop.ONE_FILE_ONLY); } } else { - osparc.FlashMessenger.getInstance().logAs(this.tr("Folders are not accepted. You might want to upload a zip file."), "ERROR"); + osparc.FlashMessenger.logError(this.tr("Folders are not accepted. Please upload a zip file instead.")); } } }, diff --git a/services/static-webserver/client/source/class/osparc/file/FileLabelWithActions.js b/services/static-webserver/client/source/class/osparc/file/FileLabelWithActions.js index 43113f880e2..cdd8c0c0e5f 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileLabelWithActions.js +++ b/services/static-webserver/client/source/class/osparc/file/FileLabelWithActions.js @@ -108,23 +108,11 @@ qx.Class.define("osparc.file.FileLabelWithActions", { setItemSelected: function(selectedItem) { if (selectedItem) { + this.__selection = [selectedItem]; const isFile = osparc.file.FilesTree.isFile(selectedItem); this.getChildControl("download-button").setEnabled(isFile); - this.getChildControl("delete-button").setEnabled(isFile); - const selectedLabel = this.getChildControl("selected-label"); - if (isFile) { - this.__selection = [selectedItem]; - selectedLabel.set({ - value: selectedItem.getLabel(), - toolTipText: selectedItem.getFileId() - }); - } else { - this.__selection = []; - selectedLabel.set({ - value: "", - toolTipText: "" - }); - } + this.getChildControl("delete-button").setEnabled(true); // folders can also be deleted + this.getChildControl("selected-label").setValue(selectedItem.getLabel()); } else { this.resetSelection(); } @@ -138,7 +126,7 @@ qx.Class.define("osparc.file.FileLabelWithActions", { } else { const selectedLabel = this.getChildControl("selected-label"); selectedLabel.set({ - value: multiSelectionData.length + " files" + value: multiSelectionData.length + " items" }); } } else { @@ -168,60 +156,87 @@ qx.Class.define("osparc.file.FileLabelWithActions", { } }, + __retrieveURLAndDownloadFile: function(file) { + const fileId = file.getFileId(); + const locationId = file.getLocation(); + osparc.utils.Utils.retrieveURLAndDownload(locationId, fileId) + .then(data => { + if (data) { + osparc.DownloadLinkTracker.getInstance().downloadLinkUnattended(data.link, data.fileName); + } + }); + }, + __deleteSelected: function() { + const toBeDeleted = []; + let isFolderSelected = false; if (this.isMultiSelect()) { - const requests = []; this.__selection.forEach(selection => { - if (selection && osparc.file.FilesTree.isFile(selection)) { - const request = this.__deleteFile(selection); - if (request) { - requests.push(request); + if (selection) { + toBeDeleted.push(selection); + if (osparc.file.FilesTree.isDir(selection)) { + isFolderSelected = true; } } }); - Promise.all(requests) - .then(datas => { - if (datas.length) { - this.fireDataEvent("fileDeleted", datas[0]); - osparc.FlashMessenger.getInstance().logAs(this.tr("Files successfully deleted"), "INFO"); - } - }); - requests } else if (this.__selection.length) { const selection = this.__selection[0]; - if (selection && osparc.file.FilesTree.isFile(selection)) { - const request = this.__deleteFile(selection); - if (request) { - request - .then(data => { - this.fireDataEvent("fileDeleted", data); - osparc.FlashMessenger.getInstance().logAs(this.tr("File successfully deleted"), "INFO"); - }); + if (selection) { + toBeDeleted.push(selection); + if (osparc.file.FilesTree.isDir(selection)) { + isFolderSelected = true; } } } + + let msg = this.tr("This action cannot be undone."); + msg += isFolderSelected ? ("
"+this.tr("All contents within the folders will be deleted.")) : ""; + msg += "
" + this.tr("Do you want to proceed?"); + const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ + caption: this.tr("Delete"), + confirmText: this.tr("Delete"), + confirmAction: "delete" + }); + confirmationWin.center(); + confirmationWin.open(); + confirmationWin.addListener("close", () => { + if (confirmationWin.getConfirmed()) { + this.__doDeleteSelected(toBeDeleted); + } + }, this); }, - __retrieveURLAndDownloadFile: function(file) { - const fileId = file.getFileId(); - const locationId = file.getLocation(); - osparc.utils.Utils.retrieveURLAndDownload(locationId, fileId) - .then(data => { - if (data) { - osparc.DownloadLinkTracker.getInstance().downloadLinkUnattended(data.link, data.fileName); + __doDeleteSelected: function(toBeDeleted) { + const requests = []; + toBeDeleted.forEach(selection => { + if (selection) { + let request = null; + if (osparc.file.FilesTree.isFile(selection)) { + request = this.__deleteItem(selection.getFileId(), selection.getLocation()); + } else { + request = this.__deleteItem(selection.getPath(), selection.getLocation()); + } + if (request) { + requests.push(request); + } + } + }); + Promise.all(requests) + .then(datas => { + if (datas.length) { + this.fireDataEvent("fileDeleted", datas[0]); + osparc.FlashMessenger.logAs(this.tr("Items successfully deleted"), "INFO"); } }); }, - __deleteFile: function(file) { - const fileId = file.getFileId(); - const locationId = file.getLocation(); + __deleteItem: function(itemId, locationId) { if (locationId !== 0 && locationId !== "0") { - osparc.FlashMessenger.getInstance().logAs(this.tr("Only files in simcore.s3 can be deleted")); + osparc.FlashMessenger.logAs(this.tr("Externally managed items cannot be deleted")); return null; } const dataStore = osparc.store.Data.getInstance(); - return dataStore.deleteFile(locationId, fileId); + return dataStore.deleteFile(locationId, itemId); }, } }); diff --git a/services/static-webserver/client/source/class/osparc/file/FilePicker.js b/services/static-webserver/client/source/class/osparc/file/FilePicker.js index fdce6e4aec9..65687d4f60a 100644 --- a/services/static-webserver/client/source/class/osparc/file/FilePicker.js +++ b/services/static-webserver/client/source/class/osparc/file/FilePicker.js @@ -39,28 +39,22 @@ qx.Class.define("osparc.file.FilePicker", { /** * @param node {osparc.data.model.Node} Node owning the widget */ - construct: function(node, pageContext = "workbench") { + construct: function(node, viewContext = "workbench") { this.base(arguments); this._setLayout(new qx.ui.layout.VBox(20)); this.set({ node, - pageContext }); - this.__buildLayout(); + this.__buildLayout(viewContext); }, properties: { node: { check: "osparc.data.model.Node" }, - - pageContext: { - check: ["workbench", "guided", "app"], - nullable: false - } }, events: { @@ -117,7 +111,7 @@ qx.Class.define("osparc.file.FilePicker", { return osparc.file.FilePicker.isOutputFromStore(outputs) || osparc.file.FilePicker.isOutputDownloadLink(outputs); }, - setOutputValue: function(node, outputValue) { + __setOutputValue: function(node, outputValue) { node.setOutputData({ "outFile": outputValue }); @@ -133,7 +127,7 @@ qx.Class.define("osparc.file.FilePicker", { setOutputValueFromStore: function(node, store, dataset, path, label) { if (store !== undefined && path) { - osparc.file.FilePicker.setOutputValue(node, { + this.__setOutputValue(node, { store, dataset, path, @@ -144,7 +138,7 @@ qx.Class.define("osparc.file.FilePicker", { setOutputValueFromLink: function(node, downloadLink, label) { if (downloadLink) { - osparc.file.FilePicker.setOutputValue(node, { + this.__setOutputValue(node, { downloadLink, label: label ? label : "" }); @@ -152,7 +146,7 @@ qx.Class.define("osparc.file.FilePicker", { }, resetOutputValue: function(node) { - osparc.file.FilePicker.setOutputValue(node, null); + this.__setOutputValue(node, null); }, getOutputFileMetadata: function(node) { @@ -161,17 +155,19 @@ qx.Class.define("osparc.file.FilePicker", { const params = { url: { locationId: outValue.store, - datasetId: outValue.dataset + path: outValue.path } }; - osparc.data.Resources.fetch("storageFiles", "getByLocationAndDataset", params) - .then(files => { - const fileMetadata = files.find(file => file.file_id === outValue.path); - if (fileMetadata) { - resolve(fileMetadata); - } else { - reject(); + osparc.data.Resources.fetch("storagePaths", "getPaths", params) + .then(pagResp => { + if (pagResp["items"]) { + const file = pagResp["items"].find(item => item.path === outValue.path); + if (file) { + resolve(file["file_meta_data"]); + return; + } } + reject(); }) .catch(() => reject()); }); @@ -280,19 +276,18 @@ qx.Class.define("osparc.file.FilePicker", { if (this.__filesTree) { this.__selectedFileFound = false; this.__filesTree.resetCache(); - this.__filesTree.populateTree(); + this.__filesTree.populateLocations(); } }, - __buildLayout: function() { + __buildLayout: function(viewContext) { this._removeAll(); const hasOutput = osparc.file.FilePicker.hasOutputAssigned(this.getNode().getOutputs()); if (hasOutput) { this.__buildInfoLayout(); } else { this.__addProgressBar(); - const isWorkbenchContext = this.getPageContext() === "workbench"; - if (isWorkbenchContext) { + if (viewContext === "workbench") { this.__buildWorkbenchLayout(); } else { this.setMargin(10); @@ -421,14 +416,25 @@ qx.Class.define("osparc.file.FilePicker", { if (files.length === 1) { const fileUploader = new osparc.file.FileUploader(this.getNode()); fileUploader.addListener("uploadAborted", () => this.__resetOutput()); - fileUploader.addListener("fileUploaded", () => { + fileUploader.addListener("fileUploaded", e => { + const fileMetadata = e.getData(); + if ( + "location" in fileMetadata && + "dataset" in fileMetadata && + "path" in fileMetadata && + "name" in fileMetadata + ) { + osparc.file.FilePicker.setOutputValueFromStore(this.getNode(), fileMetadata["location"], fileMetadata["dataset"], fileMetadata["path"], fileMetadata["name"]); + } else { + console.error("metadata info missing", fileMetadata); + } this.fireEvent("fileUploaded"); this.getNode().fireEvent("fileUploaded"); }, this); fileUploader.retrieveUrlAndUpload(files[0]); return true; } - osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); + osparc.FlashMessenger.logError(osparc.file.FileDrop.ONE_FILE_ONLY); } return false; }, @@ -545,8 +551,8 @@ qx.Class.define("osparc.file.FilePicker", { flex: 1 }); treeFolderLayout.add(treeLayout, 0); - const allowMultiselection = false; - const folderViewer = new osparc.file.FolderViewer(allowMultiselection); + const allowMultiSelection = false; + const folderViewer = new osparc.file.FolderViewer(allowMultiSelection); treeFolderLayout.add(folderViewer, 1); filesTree.addListener("selectionChanged", () => { @@ -581,9 +587,9 @@ qx.Class.define("osparc.file.FilePicker", { folderViewer.setFolder(parent); } }, this); - folderViewer.addListener("requestDatasetFiles", e => { + folderViewer.addListener("requestPathItems", e => { const data = e.getData(); - filesTree.requestDatasetFiles(data.locationId, data.datasetId); + filesTree.requestPathItems(data.locationId, data.path); }, this); const selectBtn = this.__selectButton = new qx.ui.form.Button(this.tr("Select")).set({ diff --git a/services/static-webserver/client/source/class/osparc/file/FileTreeItem.js b/services/static-webserver/client/source/class/osparc/file/FileTreeItem.js index 5e7a4a02236..7ef50a49ed4 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileTreeItem.js +++ b/services/static-webserver/client/source/class/osparc/file/FileTreeItem.js @@ -48,19 +48,21 @@ qx.Class.define("osparc.file.FileTreeItem", { this.set({ indent: 12, // defaults to 19, decorator: "rounded", + alignY: "middle", }); - - // create a date format like "Oct. 19, 2018 11:31 AM" - this._dateFormat = new qx.util.format.DateFormat( - qx.locale.Date.getDateFormat("medium") + " " + - qx.locale.Date.getTimeFormat("short") - ); }, properties: { + loaded: { + check: "Boolean", + event: "changeLoaded", + init: true, + nullable: false + }, + location: { check: "String", - event: "changePath", + event: "changeLocation", nullable: true }, @@ -70,6 +72,12 @@ qx.Class.define("osparc.file.FileTreeItem", { nullable: true }, + displayPath: { + check: "String", + event: "changeDisplayPath", + nullable: true + }, + pathLabel: { check: "Array", event: "changePathLabel", @@ -79,30 +87,16 @@ qx.Class.define("osparc.file.FileTreeItem", { itemId: { check: "String", event: "changeItemId", - apply: "_applyItemId", + apply: "__applyItemId", nullable: true }, - isDataset: { - check: "Boolean", - event: "changeIsDataset", - init: false, - nullable: false - }, - datasetId: { check: "String", event: "changeDatasetId", nullable: true }, - loaded: { - check: "Boolean", - event: "changeLoaded", - init: true, - nullable: false - }, - fileId: { check: "String", event: "changeFileId", @@ -119,12 +113,17 @@ qx.Class.define("osparc.file.FileTreeItem", { check: "String", event: "changeSize", nullable: true - } - }, + }, - members: { // eslint-disable-line qx-rules/no-refs-in-members - _dateFormat: null, + type: { + check: ["folder", "file", "loading"], + event: "changeType", + init: null, + nullable: false, + }, + }, + members: { // overridden _addWidgets: function() { // Here's our indentation and tree-lines @@ -144,70 +143,36 @@ qx.Class.define("osparc.file.FileTreeItem", { // Add lastModified const lastModifiedWidget = new qx.ui.basic.Label().set({ - width: 140, maxWidth: 140, - textAlign: "right" + textAlign: "right", + alignY: "middle", + paddingLeft: 10, }); - let that = this; this.bind("lastModified", lastModifiedWidget, "value", { - converter: function(value) { - if (value === null) { - return ""; - } - const date = new Date(value); - return that._dateFormat.format(date); // eslint-disable-line no-underscore-dangle - } + converter: value => value ? osparc.utils.Utils.formatDateAndTime(new Date(value)) : "" }); this.addWidget(lastModifiedWidget); // Add size const sizeWidget = new qx.ui.basic.Label().set({ - width: 70, - maxWidth: 70, - textAlign: "right" + maxWidth: 90, + textAlign: "right", + alignY: "middle", + paddingLeft: 10, }); this.bind("size", sizeWidget, "value", { - converter: function(value) { - if (value === null) { - return ""; - } - return osparc.utils.Utils.bytesToSize(value); - } + converter: value => value ? osparc.utils.Utils.bytesToSize(value) : "" }); this.addWidget(sizeWidget); - - - const permissions = osparc.data.Permissions.getInstance(); - // Add Path - const pathWidget = new qx.ui.basic.Label().set({ - width: 300, - maxWidth: 300, - textAlign: "right" - }); - this.bind("path", pathWidget, "value"); - this.addWidget(pathWidget); - permissions.bind("role", pathWidget, "visibility", { - converter: () => permissions.canDo("study.nodestree.uuid.read") ? "visible" : "excluded" - }); - - // Add NodeId - const fileIdWidget = new qx.ui.basic.Label().set({ - width: 300, - maxWidth: 300, - textAlign: "right" - }); - this.bind("fileId", fileIdWidget, "value"); - this.addWidget(fileIdWidget); - permissions.bind("role", fileIdWidget, "visibility", { - converter: () => permissions.canDo("study.nodestree.uuid.read") ? "visible" : "excluded" - }); }, - // override - _applyItemId: function(value, old) { - osparc.utils.Utils.setIdToWidget(this, "fileTreeItem_" + value); + __applyItemId: function(value, old) { + if (value) { + osparc.utils.Utils.setIdToWidget(this, "fileTreeItem_" + value); + } }, + // override _applyIcon: function(value, old) { this.base(arguments, value, old); const icon = this.getChildControl("icon", true); @@ -222,9 +187,4 @@ qx.Class.define("osparc.file.FileTreeItem", { } } }, - - destruct: function() { - this._dateFormat.dispose(); - this._dateFormat = null; - } }); diff --git a/services/static-webserver/client/source/class/osparc/file/FileUploader.js b/services/static-webserver/client/source/class/osparc/file/FileUploader.js index 415d8c3ecdb..a93e94b70ba 100644 --- a/services/static-webserver/client/source/class/osparc/file/FileUploader.js +++ b/services/static-webserver/client/source/class/osparc/file/FileUploader.js @@ -39,7 +39,7 @@ qx.Class.define("osparc.file.FileUploader", { events: { "uploadAborted": "qx.event.type.Event", - "fileUploaded": "qx.event.type.Event" + "fileUploaded": "qx.event.type.Data", }, statics: { @@ -60,6 +60,7 @@ qx.Class.define("osparc.file.FileUploader", { members: { __presignedLinkData: null, __uploadedParts: null, + __fileMetadata: null, // Request to the server an upload URL. retrieveUrlAndUpload: function(file) { @@ -80,6 +81,14 @@ qx.Class.define("osparc.file.FileUploader", { .then(presignedLinkData => { if (presignedLinkData.resp.urls) { this.__presignedLinkData = presignedLinkData; + + this.__fileMetadata = { + location: presignedLinkData.locationId, + dataset: studyId, + path: presignedLinkData.fileUuid, + name: file.name + }; + try { this.__uploadFile(file); } catch (error) { @@ -124,7 +133,7 @@ qx.Class.define("osparc.file.FileUploader", { const nProgress = Math.min(Math.max(100*progress-min, min), max); this.getNode().getStatus().setProgress(nProgress); if (this.__uploadedParts.every(uploadedPart => uploadedPart["e_tag"] !== null)) { - this.__checkCompleteUpload(file); + this.__checkCompleteUpload(); } } } catch (err) { @@ -153,7 +162,7 @@ qx.Class.define("osparc.file.FileUploader", { }, // Use XMLHttpRequest to complete the upload to S3 - __checkCompleteUpload: function(file) { + __checkCompleteUpload: function() { if (this.getNode()["fileUploadAbortRequested"]) { this.__abortUpload(); return; @@ -162,29 +171,21 @@ qx.Class.define("osparc.file.FileUploader", { const presignedLinkData = this.__presignedLinkData; this.getNode().getStatus().setProgress(this.self().PROGRESS_VALUES.COMPLETING); const completeUrl = presignedLinkData.resp.links.complete_upload; - const location = presignedLinkData.locationId; - const path = presignedLinkData.fileUuid; const xhr = new XMLHttpRequest(); xhr.onloadend = () => { - const fileMetadata = { - location, - dataset: this.getNode().getStudy().getUuid(), - path, - name: file.name - }; const resp = JSON.parse(xhr.responseText); if ("error" in resp && resp["error"]) { console.error(resp["error"]); this.__abortUpload(); } else if ("data" in resp) { if (xhr.status == 202) { - console.log("waiting for completion", file.name); + console.log("waiting for completion", this.__fileMetadata.name); // @odeimaiz: we need to poll the received new location in the response // we do have links.state -> poll that link until it says ok // right now this kind of work if files are small and this happens fast - this.__pollFileUploadState(resp["data"]["links"]["state"], fileMetadata); + this.__pollFileUploadState(resp["data"]["links"]["state"]); } else if (xhr.status == 200) { - this.__completeUpload(fileMetadata); + this.__completeUpload(); } } }; @@ -196,30 +197,27 @@ qx.Class.define("osparc.file.FileUploader", { xhr.send(JSON.stringify(body)); }, - __pollFileUploadState: function(stateLink, fileMetadata) { + __pollFileUploadState: function(stateLink) { const xhr = new XMLHttpRequest(); xhr.open("POST", stateLink, true); xhr.setRequestHeader("Content-Type", "application/json"); xhr.onloadend = () => { const resp = JSON.parse(xhr.responseText); if ("data" in resp && resp["data"] && resp["data"]["state"] === "ok") { - this.__completeUpload(fileMetadata); + this.__completeUpload(); } else { const interval = 2000; - qx.event.Timer.once(() => this.__pollFileUploadState(stateLink, fileMetadata), this, interval); + qx.event.Timer.once(() => this.__pollFileUploadState(stateLink), this, interval); } }; xhr.send(); }, - __completeUpload: function(fileMetadata) { + __completeUpload: function() { this.getNode()["fileUploadAbortRequested"] = false; - if ("location" in fileMetadata && "dataset" in fileMetadata && "path" in fileMetadata && "name" in fileMetadata) { - osparc.file.FilePicker.setOutputValueFromStore(this.getNode(), fileMetadata["location"], fileMetadata["dataset"], fileMetadata["path"], fileMetadata["name"]); - } this.__presignedLinkData = null; - this.fireEvent("fileUploaded"); + this.fireDataEvent("fileUploaded", this.__fileMetadata); }, __abortUpload: function() { diff --git a/services/static-webserver/client/source/class/osparc/file/FilesTree.js b/services/static-webserver/client/source/class/osparc/file/FilesTree.js index a682bbca1c2..9b8a13a8265 100644 --- a/services/static-webserver/client/source/class/osparc/file/FilesTree.js +++ b/services/static-webserver/client/source/class/osparc/file/FilesTree.js @@ -43,23 +43,14 @@ qx.Class.define("osparc.file.FilesTree", { this.base(arguments, null, "label", "children"); this.set({ - openMode: "none", + openMode: "dbltap", decorator: "no-border", font: "text-14", }); - this.resetChecks(); + this.__resetChecks(); this.addListener("tap", this.__selectionChanged, this); - - // Listen to "Enter" key - this.addListener("keypress", keyEvent => { - if (keyEvent.getKeyIdentifier() === "Enter") { - this.__itemSelected(); - } - }, this); - - this.__loadPaths = {}; }, properties: { @@ -75,149 +66,122 @@ qx.Class.define("osparc.file.FilesTree", { }, events: { - "selectionChanged": "qx.event.type.Event", // tap - "itemSelected": "qx.event.type.Event", // dbltap + "selectionChanged": "qx.event.type.Event", "fileCopied": "qx.event.type.Data", "filesAddedToTree": "qx.event.type.Event" }, statics: { isDir: function(item) { - let isDir = false; - if (item["get"+qx.lang.String.firstUp("path")]) { - if (item.getPath() !== null) { - isDir = true; - } - } - return isDir; + return item.getType() === "folder"; }, isFile: function(item) { - let isFile = false; - if (item["set"+qx.lang.String.firstUp("fileId")]) { - isFile = true; - } - return isFile; + return item.getType() === "file"; }, addLoadingChild: function(parent) { - const loadingModel = qx.data.marshal.Json.createModel({ - label: "Loading...", - location: null, - path: null, - children: [], - icon: "@FontAwesome5Solid/circle-notch/12" - }, true); + const loadingData = osparc.data.Converters.createLoadingEntry(); + const loadingModel = qx.data.marshal.Json.createModel(loadingData, true); parent.getChildren().append(loadingModel); }, removeLoadingChild: function(parent) { for (let i = parent.getChildren().length - 1; i >= 0; i--) { - if (parent.getChildren().toArray()[i].getLabel() === "Loading...") { - parent.getChildren().toArray() - .splice(i, 1); + if (parent.getChildren().toArray()[i].getType() === "loading") { + parent.getChildren().toArray().splice(i, 1); } } }, - - attachPathLabel: function(srcPathLabel, data) { - data["pathLabel"] = srcPathLabel.concat(data["label"]); - if ("children" in data) { - data.children.forEach(child => this.self().attachPathLabel(data["pathLabel"], child)); - } - } }, members: { __locations: null, - __datasets: null, + __pathModels: null, __loadPaths: null, - resetChecks: function() { - this.__locations = new Set(); - this.__datasets = new Set(); - }, - resetCache: function() { - this.resetChecks(); + this.__resetChecks(); const dataStore = osparc.store.Data.getInstance(); dataStore.resetCache(); }, + populateLocations: function() { + this.__resetChecks(); + + const treeName = "My Data"; + this.__resetTree(treeName); + const rootModel = this.getModel(); + rootModel.getChildren().removeAll(); + this.self().addLoadingChild(rootModel); + + this.set({ + hideRoot: true + }); + const dataStore = osparc.store.Data.getInstance(); + return dataStore.getLocations() + .then(locations => { + const datasetPromises = []; + if (this.__locations.size === 0) { + this.__resetChecks(); + this.__locationsToRoot(locations); + for (let i=0; i
" + this.tr("If the Update button is disabled, they might require manual intervention to be updated:"); - retiredText += "
- " + this.tr("Open the study"); - retiredText += "
- " + this.tr("Click on the retired service, download the data"); - retiredText += "
- " + this.tr("Upload the data to an updated version"); + const introText = new qx.ui.basic.Label().set({ + font: "text-14", + rich: true + }); + this._introText.add(introText); + let msg = ""; + const params = { + url: { + studyId: this._studyData["uuid"] } - const retiredLabel = new qx.ui.basic.Label(retiredText); - labels.push(retiredLabel); - } - const updatableServices = this.self().updatableNodeIds(this._studyData["workbench"]); - if (updatableServices.length === 0) { - const upToDateText = this.tr("All services are up to date to their latest compatible version."); - const upToDateLabel = new qx.ui.basic.Label(upToDateText); - labels.push(upToDateLabel); - } else if (canIWrite) { - const useUpdateButtonText = this.tr("Use the Update buttons to bring the services to their latest compatible version."); - const useUpdateButtonLabel = new qx.ui.basic.Label(useUpdateButtonText); - labels.push(useUpdateButtonLabel); - } else { - const notUpToDateText = this.tr("Some services are not up to date."); - const notUpToDateLabel = new qx.ui.basic.Label(notUpToDateText); - labels.push(notUpToDateLabel); - } + }; + osparc.data.Resources.fetch("studies", "getServices", params) + .then(resp => { + const services = resp["services"]; + if (osparc.study.Utils.getCantExecuteServices(services).length) { + msg += this.tr("Some services are inaccessible. Please contact the service owner:"); + msg += "
"; + } + if (osparc.study.Utils.anyServiceRetired(services)) { + msg += this.tr("Services marked in red are retired and can no longer be used."); + if (canIWrite) { + msg += "
" + this.tr("If the Update button is disabled, they might require manual intervention to be updated:"); + msg += "
- " + this.tr("Open the study"); + msg += "
- " + this.tr("Click on the retired service, download the data"); + msg += "
- " + this.tr("Upload the data to a newer version"); + } + msg += "
"; + } + if (osparc.study.Utils.anyServiceDeprecated(services)) { + msg += this.tr("Services marked in yellow are deprecated, they will be retired soon."); + if (canIWrite) { + msg += " " + this.tr("They can be updated by pressing the Update button."); + } + msg += "
"; + } + const anyServiceUpdatable = osparc.study.Utils.anyServiceUpdatable(services); + if (anyServiceUpdatable === false && msg === "") { + msg += this.tr("All services are up to date to their latest compatible version."); + msg += "
"; + } else if (canIWrite) { + msg += this.tr("Click Update to upgrade services to the latest compatible version."); + msg += "
"; + } else { + msg += this.tr("Some services are not up to date."); + } - labels.forEach(label => { - label.set({ - font: "text-14", - rich: true + introText.setValue(msg); }); - this._introText.add(label); - }); }, __updateService: async function(nodeId, key, version, button) { - const latestCompatible = osparc.service.Utils.getLatestCompatible(key, version); + const latestCompatible = osparc.store.Services.getLatestCompatible(key, version); const patchData = {}; if (key !== latestCompatible["key"]) { patchData["key"] = latestCompatible["key"]; @@ -201,6 +156,7 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", { const canIWrite = osparc.data.model.Study.canIWrite(this._studyData["accessRights"]); let i = 0; + const updatableServices = []; const workbench = this._studyData["workbench"]; for (const nodeId in workbench) { i++; @@ -218,7 +174,7 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", { const compatibleVersionLabel = new qx.ui.basic.Label().set({ font: "text-14" }); - const latestCompatible = osparc.service.Utils.getLatestCompatible(node["key"], node["version"]); + const latestCompatible = osparc.store.Services.getLatestCompatible(node["key"], node["version"]); if (latestCompatible) { // updatable osparc.store.Services.getService(latestCompatible["key"], latestCompatible["version"]) @@ -241,9 +197,9 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", { column: this.self().GRID_POS.COMPATIBLE_VERSION }); - const isUpdatable = osparc.service.Utils.isUpdatable(node); if (latestCompatible && canIWrite) { const updateButton = new osparc.ui.form.FetchButton(null, "@MaterialIcons/update/14"); + const isUpdatable = osparc.service.Utils.isUpdatable(metadata); updateButton.set({ enabled: isUpdatable }); @@ -257,6 +213,7 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", { label: this.tr("Update"), center: true }); + updatableServices.push(nodeId); } updateButton.addListener("execute", () => this.__updateService(nodeId, node["key"], node["version"], updateButton), this); this._servicesGrid.add(updateButton, { @@ -266,7 +223,6 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", { } } - const updatableServices = osparc.metadata.ServicesInStudyUpdate.updatableNodeIds(workbench); if (updatableServices.length && canIWrite) { const updateAllButton = this.__updateAllButton; updateAllButton.show(); diff --git a/services/static-webserver/client/source/class/osparc/navigation/BreadcrumbsSlideshow.js b/services/static-webserver/client/source/class/osparc/navigation/BreadcrumbsSlideshow.js index 21089318d18..e69e8b66ded 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/BreadcrumbsSlideshow.js +++ b/services/static-webserver/client/source/class/osparc/navigation/BreadcrumbsSlideshow.js @@ -46,7 +46,7 @@ qx.Class.define("osparc.navigation.BreadcrumbsSlideshow", { if (study.isPipelineEmpty()) { label.setValue(this.tr("Pipeline is empty")); } else { - label.setValue(this.tr("There are no visible nodes, enable some by editing the App Mode")); + label.setValue(this.tr("No visible nodes. Enable some by adjusting the app mode.")); } this._add(label); } diff --git a/services/static-webserver/client/source/class/osparc/navigation/LogoOnOff.js b/services/static-webserver/client/source/class/osparc/navigation/LogoOnOff.js index af2f54164ef..7399efe698d 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/LogoOnOff.js +++ b/services/static-webserver/client/source/class/osparc/navigation/LogoOnOff.js @@ -46,7 +46,7 @@ qx.Class.define("osparc.navigation.LogoOnOff", { check: "Boolean", init: false, nullable: false, - apply: "_applyOnline" + apply: "__applyOnline" } }, @@ -64,6 +64,7 @@ qx.Class.define("osparc.navigation.LogoOnOff", { } case "off-logo": { control = new qx.ui.basic.Image("osparc/offline.svg"); + osparc.utils.Utils.setAltToImage(control, "offline"); const container = this.getChildControl("off-logo-container"); container.add(control, { flex: 1 @@ -80,7 +81,7 @@ qx.Class.define("osparc.navigation.LogoOnOff", { return control || this.base(arguments, id); }, - _applyOnline: function(value) { + __applyOnline: function(value) { this.setSelection([this.getSelectables()[value ? 1 : 0]]); } } diff --git a/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js b/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js index e8d252a5f82..f9f046dd0b3 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js +++ b/services/static-webserver/client/source/class/osparc/navigation/NavigationBar.js @@ -50,6 +50,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { paddingLeft: 10, paddingRight: 10, height: this.self().HEIGHT, + backgroundColor: "background-main-1", }); osparc.utils.Utils.setIdToWidget(this, "navigationBar"); @@ -57,7 +58,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { events: { "backToDashboardPressed": "qx.event.type.Event", - "downloadStudyLogs": "qx.event.type.Event" + "openLogger": "qx.event.type.Event" }, properties: { @@ -80,12 +81,6 @@ qx.Class.define("osparc.navigation.NavigationBar", { minWidth: 30, minHeight: 30 }, - - PAGE_CONTEXT: { - 0: "dashboard", - 1: "workbench", - 2: "app" - } }, members: { @@ -97,11 +92,6 @@ qx.Class.define("osparc.navigation.NavigationBar", { }, __buildLayout: function() { - const colorStr = qx.theme.manager.Color.getInstance().resolve("background-main-1"); - const color = qx.util.ColorUtil.stringToRgb(colorStr); - this.getContentElement().setStyles({ - "background": `linear-gradient(0deg, rgba(1, 18, 26, 0.1) 0%, ${qx.util.ColorUtil.rgbToRgbString(color)} 4%)` - }); this.getChildControl("left-items"); this.getChildControl("center-items"); this.getChildControl("right-items"); @@ -127,6 +117,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { // right-items this.getChildControl("tasks-button"); + this.getChildControl("jobs-button"); this.getChildControl("notifications-button"); this.getChildControl("expiration-icon"); this.getChildControl("help"); @@ -206,7 +197,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { break; case "study-title-options": control = new osparc.navigation.StudyTitleWOptions(); - control.addListener("downloadStudyLogs", () => this.fireEvent("downloadStudyLogs")); + control.addListener("openLogger", () => this.fireEvent("openLogger")); this.getChildControl("left-items").add(control); break; case "read-only-info": { @@ -228,14 +219,14 @@ qx.Class.define("osparc.navigation.NavigationBar", { this.getChildControl("center-items").add(control); break; } - case "credits-button": - control = new osparc.desktop.credits.CreditsIndicatorButton(); - this.getChildControl("right-items").add(control); - break; case "tasks-button": control = new osparc.task.TasksButton(); this.getChildControl("right-items").add(control); break; + case "jobs-button": + control = new osparc.jobs.JobsButton(); + this.getChildControl("right-items").add(control); + break; case "notifications-button": control = new osparc.notification.NotificationsButton(); this.getChildControl("right-items").add(control); @@ -272,6 +263,10 @@ qx.Class.define("osparc.navigation.NavigationBar", { osparc.utils.Utils.setIdToWidget(control, "helpNavigationBtn"); this.getChildControl("right-items").add(control); break; + case "credits-button": + control = new osparc.desktop.credits.CreditsIndicatorButton(); + this.getChildControl("right-items").add(control); + break; case "log-in-button": { control = this.__createLoginBtn().set({ visibility: "excluded" @@ -302,7 +297,8 @@ qx.Class.define("osparc.navigation.NavigationBar", { __createHelpMenuBtn: function() { const menu = new qx.ui.menu.Menu().set({ - position: "top-right" + position: "top-right", + appearance: "menu-wider", }); const menuButton = new qx.ui.form.MenuButton(null, "@FontAwesome5Regular/question-circle/22", menu).set({ backgroundColor: "transparent" @@ -310,7 +306,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { osparc.utils.Utils.setIdToWidget(menu, "helpNavigationMenu"); - // menus + // quick starts and manuals osparc.store.Support.addQuickStartToMenu(menu); osparc.store.Support.addGuidedToursToMenu(menu); osparc.store.Support.addManualButtonsToMenu(menu, menuButton); @@ -318,8 +314,7 @@ qx.Class.define("osparc.navigation.NavigationBar", { // feedback osparc.store.Support.addSupportButtonsToMenu(menu, menuButton); - - osparc.utils.Utils.prettifyMenu(menu); + osparc.store.Support.addReleaseNotesToMenu(menu); return menuButton; }, @@ -330,6 +325,15 @@ qx.Class.define("osparc.navigation.NavigationBar", { return registerButton; }, + addDashboardTabButtons: function(tabButtons) { + this.__tabButtons = tabButtons; + this.getChildControl("center-items").add(tabButtons); + this.bind("study", this.__tabButtons, "visibility", { + converter: s => s ? "excluded" : "visible" + }); + this.__navBarResized(); + }, + __applyStudy: function(study) { const readOnlyInfo = this.getChildControl("read-only-info") if (study) { diff --git a/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js b/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js index 342f0de8cee..4c9c4cff32e 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js +++ b/services/static-webserver/client/source/class/osparc/navigation/StudyTitleWOptions.js @@ -31,7 +31,7 @@ qx.Class.define("osparc.navigation.StudyTitleWOptions", { }, events: { - "downloadStudyLogs": "qx.event.type.Event" + "openLogger": "qx.event.type.Event" }, properties: { @@ -68,22 +68,62 @@ qx.Class.define("osparc.navigation.StudyTitleWOptions", { }); }); break; - case "study-menu-download-logs": + case "study-menu-reload": control = new qx.ui.menu.Button().set({ - label: this.tr("Download logs"), + label: this.tr("Reload"), + icon: "@FontAwesome5Solid/redo-alt/12", + }); + control.addListener("execute", () => this.__reloadIFrame(), this); + break; + case "study-menu-convert-to-pipeline": + control = new qx.ui.menu.Button().set({ + label: this.tr("Convert to Pipeline"), + icon: null, + }); + control.addListener("execute", () => { + this.getStudy().getUi().setMode("workbench"); + }); + break; + case "study-menu-convert-to-standalone": + control = new qx.ui.menu.Button().set({ + label: this.tr("Convert to Standalone"), + icon: null, + }); + control.addListener("execute", () => { + this.getStudy().getUi().setMode("standalone"); + }); + break; + case "study-menu-restore": + control = new qx.ui.menu.Button().set({ + label: this.tr("Restore"), + icon: osparc.theme.common.Image.URLS["window-restore"] + "/20", + }); + control.addListener("execute", () => { + this.getStudy().getUi().setMode("workbench"); + }); + break; + case "study-menu-open-logger": + control = new qx.ui.menu.Button().set({ + label: this.tr("Platform Logs..."), icon: "@FontAwesome5Solid/download/14" }); - control.addListener("execute", () => this.fireEvent("downloadStudyLogs")); + control.addListener("execute", () => this.fireEvent("openLogger")); break; case "study-menu-button": { const optionsMenu = new qx.ui.menu.Menu(); + optionsMenu.setAppearance("menu-wider"); optionsMenu.add(this.getChildControl("study-menu-info")); - optionsMenu.add(this.getChildControl("study-menu-download-logs")); + optionsMenu.add(this.getChildControl("study-menu-reload")); + optionsMenu.add(this.getChildControl("study-menu-convert-to-pipeline")); + optionsMenu.add(this.getChildControl("study-menu-convert-to-standalone")); + optionsMenu.add(this.getChildControl("study-menu-restore")); + optionsMenu.add(this.getChildControl("study-menu-open-logger")); control = new qx.ui.form.MenuButton().set({ appearance: "fab-button", menu: optionsMenu, icon: "@FontAwesome5Solid/ellipsis-v/14", - allowGrowY: false + allowGrowY: false, + width: 24, }); this._add(control); break; @@ -104,9 +144,50 @@ qx.Class.define("osparc.navigation.StudyTitleWOptions", { return control || this.base(arguments, id); }, + __reloadIFrame: function() { + const nodes = this.getStudy().getWorkbench().getNodes(); + if (Object.keys(nodes).length === 1) { + Object.values(nodes)[0].getIframeHandler().restartIFrame(); + } + }, + __applyStudy: function(study) { if (study) { - study.bind("name", this.getChildControl("edit-title-label"), "value"); + const editTitle = this.getChildControl("edit-title-label"); + study.bind("name", editTitle, "value"); + + const reloadButton = this.getChildControl("study-menu-reload"); + study.getUi().bind("mode", reloadButton, "visibility", { + converter: mode => mode === "standalone" ? "visible" : "excluded" + }); + + const convertToPipelineButton = this.getChildControl("study-menu-convert-to-pipeline"); + const convertToStandaloneButton = this.getChildControl("study-menu-convert-to-standalone"); + if (osparc.product.Utils.hasConvertToPipelineEnabled()) { + study.getUi().bind("mode", convertToPipelineButton, "visibility", { + converter: mode => mode === "standalone" ? "visible" : "excluded" + }); + + const evaluateConvertToStandaloneButton = () => { + // exclude until we have the export to standalone backend functionality + convertToStandaloneButton.exclude(); + }; + study.getWorkbench().addListener("pipelineChanged", () => evaluateConvertToStandaloneButton()); + study.getUi().addListener("changeMode", () => evaluateConvertToStandaloneButton()); + } else { + convertToPipelineButton.exclude(); + convertToStandaloneButton.exclude(); + } + + const restoreButton = this.getChildControl("study-menu-restore"); + study.getUi().bind("mode", restoreButton, "visibility", { + converter: mode => mode === "standalone" ? "visible" : "excluded" + }); + + const loggerButton = this.getChildControl("study-menu-open-logger"); + study.getUi().bind("mode", loggerButton, "visibility", { + converter: mode => mode === "standalone" ? "visible" : "excluded" + }); } else { this.exclude(); } diff --git a/services/static-webserver/client/source/class/osparc/navigation/UserMenu.js b/services/static-webserver/client/source/class/osparc/navigation/UserMenu.js index b96841de7d9..dfe8898e0e5 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/UserMenu.js +++ b/services/static-webserver/client/source/class/osparc/navigation/UserMenu.js @@ -22,7 +22,7 @@ qx.Class.define("osparc.navigation.UserMenu", { this.base(arguments); this.set({ - font: "text-14" + appearance: "menu-wider", }); }, @@ -77,12 +77,6 @@ qx.Class.define("osparc.navigation.UserMenu", { }, this); this.add(control); break; - case "preferences": - control = new qx.ui.menu.Button(this.tr("Preferences")); - control.addListener("execute", () => osparc.navigation.UserMenuButton.openPreferences(), this); - osparc.utils.Utils.setIdToWidget(control, "userMenuPreferencesBtn"); - this.add(control); - break; case "organizations": control = new qx.ui.menu.Button(this.tr("Organizations")).set({ visibility: osparc.data.Permissions.getInstance().canDo("user.organizations.create") ? "visible" :"excluded" @@ -92,7 +86,7 @@ qx.Class.define("osparc.navigation.UserMenu", { this.add(control); break; case "market": - control = new qx.ui.menu.Button(this.tr("Market")); + control = new qx.ui.menu.Button(this.tr("The Shop")); control.addListener("execute", () => osparc.vipMarket.MarketWindow.openWindow()); this.add(control); break; @@ -162,7 +156,6 @@ qx.Class.define("osparc.navigation.UserMenu", { if (osparc.desktop.credits.Utils.areWalletsEnabled()) { this.getChildControl("billing-center"); } - this.getChildControl("preferences"); this.getChildControl("organizations"); } this.addSeparator(); @@ -187,8 +180,6 @@ qx.Class.define("osparc.navigation.UserMenu", { this.addSeparator(); this.getChildControl("log-out"); - - osparc.utils.Utils.prettifyMenu(this); }, __addAnnouncements: function() { @@ -217,21 +208,19 @@ qx.Class.define("osparc.navigation.UserMenu", { if (osparc.desktop.credits.Utils.areWalletsEnabled()) { this.getChildControl("billing-center"); } - this.getChildControl("preferences"); this.getChildControl("organizations"); } this.addSeparator(); - // quick starts + // quick starts and manuals osparc.store.Support.addQuickStartToMenu(this); osparc.store.Support.addGuidedToursToMenu(this); - - // manuals osparc.store.Support.addManualButtonsToMenu(this); this.addSeparator(); // feedbacks osparc.store.Support.addSupportButtonsToMenu(this); + osparc.store.Support.addReleaseNotesToMenu(this); this.addSeparator(); this.getChildControl("theme-switcher"); @@ -253,8 +242,6 @@ qx.Class.define("osparc.navigation.UserMenu", { } this.addSeparator(); this.getChildControl("log-out"); - - osparc.utils.Utils.prettifyMenu(this); } } }); diff --git a/services/static-webserver/client/source/class/osparc/navigation/UserMenuButton.js b/services/static-webserver/client/source/class/osparc/navigation/UserMenuButton.js index e1726533215..777834d2739 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/UserMenuButton.js +++ b/services/static-webserver/client/source/class/osparc/navigation/UserMenuButton.js @@ -70,13 +70,6 @@ qx.Class.define("osparc.navigation.UserMenuButton", { }); }, - statics: { - openPreferences: function() { - const preferencesWindow = osparc.desktop.preferences.PreferencesWindow.openWindow(); - return preferencesWindow; - } - }, - members: { __forceNullColor: null, diff --git a/services/static-webserver/client/source/class/osparc/node/LifeCycleView.js b/services/static-webserver/client/source/class/osparc/node/LifeCycleView.js index 5f810b18799..ea6ed0fbb28 100644 --- a/services/static-webserver/client/source/class/osparc/node/LifeCycleView.js +++ b/services/static-webserver/client/source/class/osparc/node/LifeCycleView.js @@ -110,7 +110,7 @@ qx.Class.define("osparc.node.LifeCycleView", { }); updateButton.addListener("execute", () => { updateButton.setFetching(true); - const latestCompatible = osparc.service.Utils.getLatestCompatible(node.getKey(), node.getVersion()); + const latestCompatible = osparc.store.Services.getLatestCompatible(node.getKey(), node.getVersion()); if (node.getKey() !== latestCompatible["key"]) { node.setKey(latestCompatible["key"]); } diff --git a/services/static-webserver/client/source/class/osparc/node/TierSelectionView.js b/services/static-webserver/client/source/class/osparc/node/TierSelectionView.js index f23b6077499..b4432cedecd 100644 --- a/services/static-webserver/client/source/class/osparc/node/TierSelectionView.js +++ b/services/static-webserver/client/source/class/osparc/node/TierSelectionView.js @@ -46,12 +46,11 @@ qx.Class.define("osparc.node.TierSelectionView", { tiersLayout.add(tierBox); const node = this.getNode(); - const pricingStore = osparc.store.Pricing.getInstance(); - pricingStore.fetchPricingPlansService(node.getKey(), node.getVersion()) + osparc.store.Pricing.getInstance().fetchPricingPlansService(node.getKey(), node.getVersion()) .then(pricingPlans => { if (pricingPlans && "pricingUnits" in pricingPlans && pricingPlans["pricingUnits"].length) { - const pricingUnits = pricingPlans["pricingUnits"].map(princingUnitData => { - const pricingUnit = new osparc.data.model.PricingUnit(princingUnitData); + const pricingUnits = pricingPlans["pricingUnits"].map(pricingUnitData => { + const pricingUnit = new osparc.data.model.PricingUnit(pricingUnitData); return pricingUnit; }); pricingUnits.forEach(pricingUnit => { diff --git a/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js b/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js index 3c75815c296..507eef686dd 100644 --- a/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js +++ b/services/static-webserver/client/source/class/osparc/node/UpdateResourceLimitsView.js @@ -154,13 +154,9 @@ qx.Class.define("osparc.node.UpdateResourceLimitsView", { }; osparc.data.Resources.fetch("nodesInStudyResources", "put", params) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Limits successfully updated")); - }) - .catch(err => { - console.error(err); - const msg = err.message || this.tr("Something went wrong updating the limits"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); + osparc.FlashMessenger.logAs(this.tr("Limits have been successfully updated")); }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while updating the limits"))) .finally(() => { this.__saveBtn.setFetching(false); this.__populateLayout(); diff --git a/services/static-webserver/client/source/class/osparc/node/slideshow/NodeView.js b/services/static-webserver/client/source/class/osparc/node/slideshow/NodeView.js index 4aeb8b0b4a6..1d9cd803c8b 100644 --- a/services/static-webserver/client/source/class/osparc/node/slideshow/NodeView.js +++ b/services/static-webserver/client/source/class/osparc/node/slideshow/NodeView.js @@ -38,13 +38,6 @@ qx.Class.define("osparc.node.slideshow.NodeView", { statics: { LOGGER_HEIGHT: 28, - - isPropsFormShowable: function(node) { - if (node && ("getPropsForm" in node) && node.getPropsForm()) { - return node.getPropsForm().hasVisibleInputs(); - } - return false; - } }, members: { @@ -135,8 +128,8 @@ qx.Class.define("osparc.node.slideshow.NodeView", { }, isSettingsGroupShowable: function() { - const node = this.getNode(); - return this.self().isPropsFormShowable(node); + // do not show Settings in App Mode + return false; }, __iFrameChanged: function() { diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js index f339a68bf06..5b0ecd6cdae 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationUI.js @@ -150,7 +150,7 @@ qx.Class.define("osparc.notification.NotificationUI", { "studyId": resourceId } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(study => { const studyAlias = osparc.product.Utils.getStudyAlias({ firstUpperCase: true @@ -194,7 +194,7 @@ qx.Class.define("osparc.notification.NotificationUI", { "studyId": resourceId } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(study => titleLabel.setValue(`Note added in '${study["name"]}'`)) .catch(() => this.setEnabled(false)); } @@ -273,7 +273,7 @@ qx.Class.define("osparc.notification.NotificationUI", { orgsWindow.openOrganizationDetails(orgId); } else { const msg = this.tr("You don't have access anymore"); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } }, @@ -283,7 +283,7 @@ qx.Class.define("osparc.notification.NotificationUI", { "studyId": studyId } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(studyData => { if (studyData) { const studyDataCopy = osparc.data.model.Study.deepCloneStudyObject(studyData); @@ -299,9 +299,9 @@ qx.Class.define("osparc.notification.NotificationUI", { } }) .catch(err => { - console.error(err); + console.warn(err); const msg = this.tr("You don't have access anymore"); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); }); }, @@ -326,7 +326,7 @@ qx.Class.define("osparc.notification.NotificationUI", { } } else { const msg = this.tr("You don't have access anymore"); - osparc.FlashMessenger.getInstance().logAs(msg, "WARNING"); + osparc.FlashMessenger.logAs(msg, "WARNING"); } } } diff --git a/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js b/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js index dd8d4b543d4..3c1cfd12152 100644 --- a/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js +++ b/services/static-webserver/client/source/class/osparc/notification/NotificationsButton.js @@ -53,8 +53,10 @@ qx.Class.define("osparc.notification.NotificationsButton", { case "icon": { control = new qx.ui.basic.Image(); const iconContainer = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ - alignY: "middle" - })); + alignY: "middle", + })).set({ + paddingLeft: 5, + }); iconContainer.add(control); this._add(iconContainer, { height: "100%" diff --git a/services/static-webserver/client/source/class/osparc/po/Invitations.js b/services/static-webserver/client/source/class/osparc/po/Invitations.js index 0b1493e35f7..0fb58f8aee3 100644 --- a/services/static-webserver/client/source/class/osparc/po/Invitations.js +++ b/services/static-webserver/client/source/class/osparc/po/Invitations.js @@ -116,10 +116,7 @@ qx.Class.define("osparc.po.Invitations", { .then(data => { this.__populateInvitationLayout(data); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => generateInvitationBtn.setFetching(false)); } }, this); diff --git a/services/static-webserver/client/source/class/osparc/po/MessageTemplates.js b/services/static-webserver/client/source/class/osparc/po/MessageTemplates.js index 8443a7fa788..6a6bdadd674 100644 --- a/services/static-webserver/client/source/class/osparc/po/MessageTemplates.js +++ b/services/static-webserver/client/source/class/osparc/po/MessageTemplates.js @@ -92,10 +92,7 @@ qx.Class.define("osparc.po.MessageTemplates", { }; osparc.data.Resources.fetch("productMetadata", "updateEmailTemplate", params) .then(() => osparc.FlashMessenger.logAs(this.tr("Template updated"), "INFO")) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this._buildLayout()); } } diff --git a/services/static-webserver/client/source/class/osparc/po/POCenter.js b/services/static-webserver/client/source/class/osparc/po/POCenter.js index 6141d5ffaf2..39a10d9afdb 100644 --- a/services/static-webserver/client/source/class/osparc/po/POCenter.js +++ b/services/static-webserver/client/source/class/osparc/po/POCenter.js @@ -24,7 +24,7 @@ qx.Class.define("osparc.po.POCenter", { const miniProfile = osparc.desktop.account.MyAccount.createMiniProfileView().set({ paddingRight: 10 }); - this.addWidgetOnTopOfTheTabs(miniProfile); + this.addWidgetToTabs(miniProfile); this.__addUsersPage(); this.__addPreRegistrationPage(); @@ -42,7 +42,7 @@ qx.Class.define("osparc.po.POCenter", { }, __addPreRegistrationPage: function() { - const title = this.tr("PreRegistration"); + const title = this.tr("Pre-Registration"); const iconSrc = "@FontAwesome5Solid/address-card/22"; const preRegistration = new osparc.po.PreRegistration(); this.addTab(title, iconSrc, preRegistration); diff --git a/services/static-webserver/client/source/class/osparc/po/PreRegistration.js b/services/static-webserver/client/source/class/osparc/po/PreRegistration.js index 8a1f0e767df..b1d6b80088a 100644 --- a/services/static-webserver/client/source/class/osparc/po/PreRegistration.js +++ b/services/static-webserver/client/source/class/osparc/po/PreRegistration.js @@ -79,7 +79,7 @@ qx.Class.define("osparc.po.PreRegistration", { if (form.validate()) { submitBtn.setFetching(true); - const flashErrorMsg = this.tr("Pre-Registration Failed. See details below"); + const flashErrorMsg = this.tr("Unsuccessful Pre-Registration. See details below"); const findingStatus = this.getChildControl("finding-status"); findingStatus.setValue(this.tr("Searching Pre-Registered users...")); @@ -89,12 +89,9 @@ qx.Class.define("osparc.po.PreRegistration", { data: JSON.parse(requestAccountData.getValue()) }; } catch (err) { - console.error(err); - const detailErrorMsg = `Error parsing Request Form JSON. ${err}`; findingStatus.setValue(detailErrorMsg); - - osparc.FlashMessenger.logAs(flashErrorMsg, "ERROR"); + osparc.FlashMessenger.logError(err, flashErrorMsg); submitBtn.setFetching(false); return } @@ -109,10 +106,9 @@ qx.Class.define("osparc.po.PreRegistration", { this.__populatePreRegistrationLayout(data); }) .catch(err => { - const detailErrorMsg = this.tr(`Error during Pre-Registeristration: ${err.message}`) + const detailErrorMsg = this.tr(`Error during Pre-Registration: ${err.message}`) findingStatus.setValue(detailErrorMsg); - console.error(err); - osparc.FlashMessenger.logAs(flashErrorMsg, "ERROR"); + osparc.FlashMessenger.logError(err, flashErrorMsg); }) .finally(() => submitBtn.setFetching(false)); } diff --git a/services/static-webserver/client/source/class/osparc/po/Users.js b/services/static-webserver/client/source/class/osparc/po/Users.js index eb011712b42..de8164957dd 100644 --- a/services/static-webserver/client/source/class/osparc/po/Users.js +++ b/services/static-webserver/client/source/class/osparc/po/Users.js @@ -90,8 +90,7 @@ qx.Class.define("osparc.po.Users", { }) .catch(err => { findingStatus.setValue(this.tr("Error searching users")); - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logError(err); }) .finally(() => searchBtn.setFetching(false)); } diff --git a/services/static-webserver/client/source/class/osparc/pricing/PlanDetails.js b/services/static-webserver/client/source/class/osparc/pricing/PlanDetails.js index 22c79cd9741..a745dbf1509 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/PlanDetails.js +++ b/services/static-webserver/client/source/class/osparc/pricing/PlanDetails.js @@ -33,7 +33,27 @@ qx.Class.define("osparc.pricing.PlanDetails", { "backToPricingPlans": "qx.event.type.Event" }, + statics: { + createTabPage: function(label, icon) { + const tabPage = new qx.ui.tabview.Page().set({ + layout: new qx.ui.layout.VBox() + }); + if (label) { + tabPage.setLabel(label); + } + if (icon) { + tabPage.setIcon(icon); + } + tabPage.getChildControl("button").set({ + font: "text-13" + }); + return tabPage; + } + }, + members: { + __servicesPage: null, + _createChildControlImpl: function(id) { let control; let layout; @@ -44,7 +64,7 @@ qx.Class.define("osparc.pricing.PlanDetails", { break; case "back-to-pp-button": control = new qx.ui.form.Button().set({ - toolTipText: this.tr("Back to Pricing Plans"), + toolTipText: this.tr("Return to Pricing Plans"), icon: "@FontAwesome5Solid/arrow-left/20", backgroundColor: "transparent" }); @@ -68,7 +88,7 @@ qx.Class.define("osparc.pricing.PlanDetails", { break; case "pricing-units": { control = new osparc.pricing.UnitsList(); - const tabPage = this.__createTabPage(this.tr("Pricing Units"), "@FontAwesome5Solid/paw/14"); + const tabPage = this.self().createTabPage(this.tr("Pricing Units"), "@FontAwesome5Solid/paw/14"); tabPage.add(control, { flex: 1 }); @@ -78,7 +98,7 @@ qx.Class.define("osparc.pricing.PlanDetails", { } case "service-list": { control = new osparc.pricing.ServicesList(); - const tabPage = this.__createTabPage(this.tr("Services"), "@FontAwesome5Solid/cogs/14"); + const tabPage = this.__servicesPage = this.self().createTabPage(this.tr("Services"), "@FontAwesome5Solid/cogs/14"); tabPage.add(control, { flex: 1 }); @@ -90,38 +110,27 @@ qx.Class.define("osparc.pricing.PlanDetails", { return control || this.base(arguments, id); }, - setCurrentPricingPlan: function(pricingPlanModel) { - if (pricingPlanModel === null) { + setCurrentPricingPlan: function(pricingPlan) { + if (pricingPlan === null) { return; } const pricingPlanListItem = this.getChildControl("pricing-plan-details"); - pricingPlanModel.bind("model", pricingPlanListItem, "model"); - pricingPlanModel.bind("ppId", pricingPlanListItem, "ppId"); - pricingPlanModel.bind("ppKey", pricingPlanListItem, "ppKey"); - pricingPlanModel.bind("title", pricingPlanListItem, "title"); - pricingPlanModel.bind("description", pricingPlanListItem, "description"); - pricingPlanModel.bind("isActive", pricingPlanListItem, "isActive"); + pricingPlan.bind("model", pricingPlanListItem, "model"); + pricingPlan.bind("ppId", pricingPlanListItem, "ppId"); + pricingPlan.bind("ppKey", pricingPlanListItem, "ppKey"); + pricingPlan.bind("title", pricingPlanListItem, "title"); + pricingPlan.bind("description", pricingPlanListItem, "description"); + pricingPlan.bind("isActive", pricingPlanListItem, "isActive"); // set PricingPlanId to the tab views - this.getChildControl("pricing-units").setPricingPlanId(pricingPlanModel.getModel()); - this.getChildControl("service-list").setPricingPlanId(pricingPlanModel.getModel()); - }, + this.getChildControl("pricing-units").setPricingPlanId(pricingPlan.getModel()); + this.getChildControl("service-list").setPricingPlanId(pricingPlan.getModel()); - __createTabPage: function(label, icon) { - const tabPage = new qx.ui.tabview.Page().set({ - layout: new qx.ui.layout.VBox() - }); - if (label) { - tabPage.setLabel(label); - } - if (icon) { - tabPage.setIcon(icon); - } - tabPage.getChildControl("button").set({ - font: "text-13" + // show services only if it's a TIER pricing plan + this.__servicesPage.getChildControl("button").set({ + visibility: pricingPlan.getClassification() === "TIER" ? "visible" : "excluded" }); - return tabPage; - } + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/pricing/PlanEditor.js b/services/static-webserver/client/source/class/osparc/pricing/PlanEditor.js index 50543ee77e5..bcaa379e69e 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/PlanEditor.js +++ b/services/static-webserver/client/source/class/osparc/pricing/PlanEditor.js @@ -217,13 +217,12 @@ qx.Class.define("osparc.pricing.PlanEditor", { }; osparc.store.Pricing.getInstance().postPricingPlan(newPricingPlanData) .then(() => { - osparc.FlashMessenger.getInstance().logAs(name + this.tr(" successfully created")); + osparc.FlashMessenger.logAs(name + this.tr(" successfully created")); this.fireEvent("done"); }) .catch(err => { - const errorMsg = err.message || this.tr("Something went wrong creating ") + name; - osparc.FlashMessenger.getInstance().logAs(errorMsg, "ERROR"); - console.error(err); + const errorMsg = this.tr("Something went wrong while creating ") + name; + osparc.FlashMessenger.logError(err, errorMsg); }) .finally(() => this.getChildControl("create").setFetching(false)); }, @@ -238,14 +237,10 @@ qx.Class.define("osparc.pricing.PlanEditor", { }; osparc.store.Pricing.getInstance().putPricingPlan(this.__pricingPlan["pricingPlanId"], updateData) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Successfully updated")); + osparc.FlashMessenger.logAs(this.tr("Successfully updated")); this.fireEvent("done"); }) - .catch(err => { - const errorMsg = err.message || this.tr("Something went wrong"); - osparc.FlashMessenger.getInstance().logAs(errorMsg, "ERROR"); - console.error(err); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.getChildControl("save").setFetching(false)); } } diff --git a/services/static-webserver/client/source/class/osparc/pricing/PlanListItem.js b/services/static-webserver/client/source/class/osparc/pricing/PlanListItem.js index a76527e35fd..9906b10ab3a 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/PlanListItem.js +++ b/services/static-webserver/client/source/class/osparc/pricing/PlanListItem.js @@ -120,7 +120,6 @@ qx.Class.define("osparc.pricing.PlanListItem", { control = new qx.ui.basic.Label().set({ font: "text-14", alignY: "middle", - width: 35, }); this._add(control, { row: 0, @@ -132,7 +131,6 @@ qx.Class.define("osparc.pricing.PlanListItem", { control = new qx.ui.basic.Label().set({ font: "text-14", alignY: "middle", - width: 80, }); this._add(control, { row: 0, @@ -164,7 +162,6 @@ qx.Class.define("osparc.pricing.PlanListItem", { control = new qx.ui.basic.Label().set({ font: "text-14", alignY: "middle", - width: 60, }); this.bind("classification", control, "value"); this._add(control, { diff --git a/services/static-webserver/client/source/class/osparc/pricing/ServicesList.js b/services/static-webserver/client/source/class/osparc/pricing/ServicesList.js index eb215da3c76..9c90f83b28e 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/ServicesList.js +++ b/services/static-webserver/client/source/class/osparc/pricing/ServicesList.js @@ -77,29 +77,42 @@ qx.Class.define("osparc.pricing.ServicesList", { .then(data => this.__populateList(data)); }, - __populateList: function(services) { - // before accessing the metadata in a sync way, we need to bring them to the cache - const metadataPromises = []; - services.forEach(service => { + __populateList: async function(services) { + const failedServices = []; + const servicePromises = services.map(async service => { const key = service["serviceKey"]; const version = service["serviceVersion"]; - metadataPromises.push(osparc.store.Services.getService(key, version)); - }); - Promise.all(metadataPromises) - .catch(err => console.error(err)) - .finally(() => { - const sList = []; - services.forEach(service => { - const key = service["serviceKey"]; - const version = service["serviceVersion"]; - const serviceMetadata = osparc.store.Services.getMetadata(key, version); - if (serviceMetadata) { - sList.push(new osparc.data.model.Service(serviceMetadata)); - } + try { + return await osparc.store.Services.getService(key, version); + } catch (err) { + console.error(err); + failedServices.push({ + key: service["serviceKey"], + version: service["serviceVersion"], }); - const servicesList = this.getChildControl("services-list"); - servicesList.setModel(new qx.data.Array(sList)); - }) + return null; // Return null to maintain array structure + } + }); + + const serviceModels = new qx.data.Array(); + // ensure that even if one request fails, the rest continue executing + const results = await Promise.allSettled(servicePromises); + results.forEach(result => { + if (result.status === "fulfilled" && result.value) { + const serviceMetadata = result.value; + serviceModels.push(new osparc.data.model.Service(serviceMetadata)); + } + }); + const servicesList = this.getChildControl("services-list"); + servicesList.setModel(serviceModels); + + if (failedServices.length) { + let msg = "Could not retrieve data from some services:
"; + failedServices.forEach(failedService => { + msg+= `- ${failedService.key}:${failedService.version}
`; + }); + osparc.FlashMessenger.logAs(msg, "WARNING"); + } }, __openAddServiceToPlan: function() { diff --git a/services/static-webserver/client/source/class/osparc/pricing/UnitEditor.js b/services/static-webserver/client/source/class/osparc/pricing/UnitEditor.js index 38f9022172e..671d4a53419 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/UnitEditor.js +++ b/services/static-webserver/client/source/class/osparc/pricing/UnitEditor.js @@ -18,45 +18,29 @@ qx.Class.define("osparc.pricing.UnitEditor", { extend: qx.ui.core.Widget, - construct: function(pricingUnit) { + construct: function(pricingPlanId, pricingUnit) { this.base(arguments); this._setLayout(new qx.ui.layout.VBox(10)); - const unitName = this.getChildControl("unit-name"); - const costPerUnit = this.getChildControl("cost-per-unit"); - this.getChildControl("comment"); - const specificInfo = this.getChildControl("specific-info"); - const unitExtraInfoCPU = this.getChildControl("unit-extra-info-cpu"); - const unitExtraInfoRAM = this.getChildControl("unit-extra-info-ram"); - const unitExtraInfoVRAM = this.getChildControl("unit-extra-info-vram"); - const unitExtraInfo = this.getChildControl("unit-extra-info"); - this.getChildControl("is-default"); - - const manager = this.__validator = new qx.ui.form.validation.Manager(); - unitName.setRequired(true); - costPerUnit.setRequired(true); - unitExtraInfoCPU.setRequired(true); - unitExtraInfoRAM.setRequired(true); - unitExtraInfoVRAM.setRequired(true); - unitExtraInfo.setRequired(true); - manager.add(unitName); - manager.add(costPerUnit); - manager.add(specificInfo); - manager.add(unitExtraInfo); + this.__validator = new qx.ui.form.validation.Manager(); + + this.set({ + pricingPlanId + }); if (pricingUnit) { this.set({ pricingUnitId: pricingUnit.getPricingUnitId(), unitName: pricingUnit.getName(), costPerUnit: pricingUnit.getCost(), + default: pricingUnit.getIsDefault(), }); + const extraInfo = osparc.utils.Utils.deepCloneObject(pricingUnit.getExtraInfo()); if (pricingUnit.getClassification() === "TIER") { this.set({ specificInfo: pricingUnit.getSpecificInfo() && pricingUnit.getSpecificInfo()["aws_ec2_instances"] ? pricingUnit.getSpecificInfo()["aws_ec2_instances"].toString() : "", - default: pricingUnit.getIsDefault(), }); - const extraInfo = osparc.utils.Utils.deepCloneObject(pricingUnit.getExtraInfo()); // extract the required fields from the unitExtraInfo this.set({ unitExtraInfoCPU: extraInfo["CPU"], @@ -69,6 +53,11 @@ qx.Class.define("osparc.pricing.UnitEditor", { this.set({ unitExtraInfo: extraInfo }); + } else if (pricingUnit.getClassification() === "LICENSE") { + // extract the required fields from the unitExtraInfo + this.set({ + unitExtraInfoNSeats: extraInfo["num_of_seats"], + }); } this.getChildControl("save"); } else { @@ -81,6 +70,7 @@ qx.Class.define("osparc.pricing.UnitEditor", { check: "Number", init: null, nullable: false, + apply: "__applyPricingPlanId" }, pricingUnitId: { @@ -146,6 +136,13 @@ qx.Class.define("osparc.pricing.UnitEditor", { event: "changeUnitExtraInfo" }, + unitExtraInfoNSeats: { + check: "Number", + init: 1, + nullable: false, + event: "changeUnitExtraInfoNSeats" + }, + default: { check: "Boolean", init: true, @@ -175,6 +172,8 @@ qx.Class.define("osparc.pricing.UnitEditor", { control = new qx.ui.form.TextField().set({ font: "text-14" }); + control.setRequired(true); + this.__validator.add(control); this.bind("unitName", control, "value"); control.bind("value", this, "unitName"); this.getChildControl("unit-form").add(control, this.tr("Unit Name")); @@ -184,6 +183,8 @@ qx.Class.define("osparc.pricing.UnitEditor", { minimum: 0, maximum: 10000 }); + control.setRequired(true); + this.__validator.add(control); this.bind("costPerUnit", control, "value"); control.bind("value", this, "costPerUnit"); this.getChildControl("unit-form").add(control, this.tr("Cost per unit")); @@ -200,6 +201,7 @@ qx.Class.define("osparc.pricing.UnitEditor", { control = new qx.ui.form.TextArea().set({ font: "text-14" }); + this.__validator.add(control); this.bind("specificInfo", control, "value"); control.bind("value", this, "specificInfo"); this.getChildControl("unit-form").add(control, this.tr("Specific info")); @@ -210,6 +212,7 @@ qx.Class.define("osparc.pricing.UnitEditor", { minimum: 0, maximum: 10000 }); + control.setRequired(true); this.bind("unitExtraInfoCPU", control, "value"); control.bind("value", this, "unitExtraInfoCPU"); this.getChildControl("unit-form").add(control, this.tr("CPU")); @@ -220,6 +223,7 @@ qx.Class.define("osparc.pricing.UnitEditor", { minimum: 0, maximum: 10000 }); + control.setRequired(true); this.bind("unitExtraInfoRAM", control, "value"); control.bind("value", this, "unitExtraInfoRAM"); this.getChildControl("unit-form").add(control, this.tr("RAM")); @@ -230,6 +234,7 @@ qx.Class.define("osparc.pricing.UnitEditor", { minimum: 0, maximum: 10000 }); + control.setRequired(true); this.bind("unitExtraInfoVRAM", control, "value"); control.bind("value", this, "unitExtraInfoVRAM"); this.getChildControl("unit-form").add(control, this.tr("VRAM")); @@ -239,6 +244,8 @@ qx.Class.define("osparc.pricing.UnitEditor", { control = new qx.ui.form.TextField().set({ font: "text-14" }); + control.setRequired(true); + this.__validator.add(control); this.bind("unitExtraInfo", control, "value", { converter: v => JSON.stringify(v) }); @@ -248,6 +255,18 @@ qx.Class.define("osparc.pricing.UnitEditor", { this.getChildControl("unit-form").add(control, this.tr("More Extra Info")); break; } + case "unit-extra-info-n-seats": { + control = new qx.ui.form.Spinner().set({ + minimum: 1, + maximum: 10000 + }); + control.setRequired(true); + this.__validator.add(control); + this.bind("unitExtraInfoNSeats", control, "value"); + control.bind("value", this, "unitExtraInfoNSeats"); + this.getChildControl("unit-form").add(control, this.tr("Number of Seats")); + break; + } case "is-default": { control = new qx.ui.form.CheckBox().set({ value: true @@ -302,87 +321,111 @@ qx.Class.define("osparc.pricing.UnitEditor", { return control || this.base(arguments, id); }, - __createPricingUnit: function() { - const unitName = this.getUnitName(); - const costPerUnit = this.getCostPerUnit(); - const comment = this.getComment(); - const awsEc2Instances = []; - const specificInfo = this.getSpecificInfo(); - if (specificInfo) { - awsEc2Instances.push(specificInfo); + __applyPricingPlanId: function(pricingPlanId) { + const pricingPlan = osparc.store.Pricing.getInstance().getPricingPlan(pricingPlanId); + if (pricingPlan) { + this.getChildControl("unit-name"); + this.getChildControl("cost-per-unit"); + this.getChildControl("comment"); + if (pricingPlan.getClassification() === "TIER") { + this.getChildControl("specific-info"); + this.getChildControl("unit-extra-info-cpu"); + this.getChildControl("unit-extra-info-ram"); + this.getChildControl("unit-extra-info-vram"); + this.getChildControl("unit-extra-info"); + } else if (pricingPlan.getClassification() === "LICENSE") { + this.getChildControl("unit-extra-info-n-seats"); + } + this.getChildControl("is-default"); } - const extraInfo = {}; - extraInfo["CPU"] = this.getUnitExtraInfoCPU(); - extraInfo["RAM"] = this.getUnitExtraInfoRAM(); - extraInfo["VRAM"] = this.getUnitExtraInfoVRAM(); - Object.assign(extraInfo, this.getUnitExtraInfo()); - const isDefault = this.getDefault(); - const params = { - url: { - "pricingPlanId": this.getPricingPlanId() - }, - data: { - "unitName": unitName, - "costPerUnit": costPerUnit, - "comment": comment, - "specificInfo": { + }, + + __createPricingUnit: function() { + const data = { + "unitName": this.getUnitName(), + "costPerUnit": this.getCostPerUnit(), + "comment": this.getComment(), + "default": this.getDefault(), + }; + + const pricingPlan = osparc.store.Pricing.getInstance().getPricingPlan(this.getPricingPlanId()); + if (pricingPlan) { + if (pricingPlan.getClassification() === "TIER") { + const awsEc2Instances = []; + const specificInfo = this.getSpecificInfo(); + if (specificInfo) { + awsEc2Instances.push(specificInfo); + } + data["specificInfo"] = { "aws_ec2_instances": awsEc2Instances - }, - "unitExtraInfo": extraInfo, - "default": isDefault + }; + const extraInfo = { + "CPU": this.getUnitExtraInfoCPU(), + "RAM": this.getUnitExtraInfoRAM(), + "VRAM": this.getUnitExtraInfoVRAM(), + }; + Object.assign(extraInfo, this.getUnitExtraInfo()); + data["unitExtraInfo"] = extraInfo; + } else if (pricingPlan.getClassification() === "LICENSE") { + data["specificInfo"] = { + "aws_ec2_instances": [], + }; + data["unitExtraInfo"] = { + "num_of_seats": this.getUnitExtraInfoNSeats(), + }; } - }; - osparc.data.Resources.fetch("pricingUnits", "post", params) + } + + osparc.store.Pricing.getInstance().createPricingUnit(this.getPricingPlanId(), data) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Successfully created")); + osparc.FlashMessenger.logAs(this.tr("Successfully created")); this.fireEvent("done"); }) - .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong"), "ERROR"); - console.error(err); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.getChildControl("create").setFetching(false)); }, __updatePricingUnit: function() { - const unitName = this.getUnitName(); - const costPerUnit = this.getCostPerUnit(); - const comment = this.getComment(); - const specificInfo = this.getSpecificInfo(); - const extraInfo = {}; - extraInfo["CPU"] = this.getUnitExtraInfoCPU(); - extraInfo["RAM"] = this.getUnitExtraInfoRAM(); - extraInfo["VRAM"] = this.getUnitExtraInfoVRAM(); - Object.assign(extraInfo, this.getUnitExtraInfo()); - const isDefault = this.getDefault(); - - const params = { - url: { - "pricingPlanId": this.getPricingPlanId(), - "pricingUnitId": this.getPricingUnitId() + const data = { + "unitName": this.getUnitName(), + "pricingUnitCostUpdate": { + "costPerUnit": this.getCostPerUnit(), + "comment": this.getComment(), }, - data: { - "unitName": unitName, - "pricingUnitCostUpdate": { - "cost_per_unit": costPerUnit, - "comment": comment - }, - "specificInfo": { + "default": this.getDefault(), + }; + + const pricingPlan = osparc.store.Pricing.getInstance().getPricingPlan(this.getPricingPlanId()); + if (pricingPlan) { + if (pricingPlan.getClassification() === "TIER") { + const specificInfo = this.getSpecificInfo(); + data["specificInfo"] = { + "aws_ec2_instances": [specificInfo] + }; + const extraInfo = { + "CPU": this.getUnitExtraInfoCPU(), + "RAM": this.getUnitExtraInfoRAM(), + "VRAM": this.getUnitExtraInfoVRAM(), + }; + Object.assign(extraInfo, this.getUnitExtraInfo()); + data["unitExtraInfo"] = extraInfo; + } else if (pricingPlan.getClassification() === "LICENSE") { + const specificInfo = this.getSpecificInfo(); + data["specificInfo"] = { "aws_ec2_instances": [specificInfo] - }, - "unitExtraInfo": extraInfo, - "default": isDefault + }; + data["unitExtraInfo"] = { + "num_of_seats": this.getUnitExtraInfoNSeats(), + }; } - }; - osparc.data.Resources.fetch("pricingUnits", "update", params) + } + + osparc.store.Pricing.getInstance().updatePricingUnit(this.getPricingPlanId(), this.getPricingUnitId(), data) .then(() => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Successfully updated")); + osparc.FlashMessenger.logAs(this.tr("Successfully updated")); this.fireEvent("done"); }) - .catch(err => { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong"), "ERROR"); - console.error(err); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.getChildControl("save").setFetching(false)); } } diff --git a/services/static-webserver/client/source/class/osparc/pricing/UnitsList.js b/services/static-webserver/client/source/class/osparc/pricing/UnitsList.js index 054d4e4f11b..b5563c3d75c 100644 --- a/services/static-webserver/client/source/class/osparc/pricing/UnitsList.js +++ b/services/static-webserver/client/source/class/osparc/pricing/UnitsList.js @@ -99,9 +99,7 @@ qx.Class.define("osparc.pricing.UnitsList", { }, __openCreatePricingUnit: function() { - const puCreator = new osparc.pricing.UnitEditor().set({ - pricingPlanId: this.getPricingPlanId() - }); + const puCreator = new osparc.pricing.UnitEditor(this.getPricingPlanId(), null); const title = this.tr("Pricing Unit Creator"); const win = osparc.ui.window.Window.popUpInWindow(puCreator, title, 400, 250); puCreator.addListener("done", () => { @@ -112,9 +110,7 @@ qx.Class.define("osparc.pricing.UnitsList", { }, __openUpdatePricingUnit: function(pricingUnit) { - const puEditor = new osparc.pricing.UnitEditor(pricingUnit).set({ - pricingPlanId: this.getPricingPlanId() - }); + const puEditor = new osparc.pricing.UnitEditor(this.getPricingPlanId(), pricingUnit); const title = this.tr("Pricing Unit Editor"); const win = osparc.ui.window.Window.popUpInWindow(puEditor, title, 400, 250); puEditor.addListener("done", () => { diff --git a/services/static-webserver/client/source/class/osparc/product/AboutProduct.js b/services/static-webserver/client/source/class/osparc/product/AboutProduct.js index 507c8c9ae7c..2736f96425a 100644 --- a/services/static-webserver/client/source/class/osparc/product/AboutProduct.js +++ b/services/static-webserver/client/source/class/osparc/product/AboutProduct.js @@ -49,10 +49,8 @@ qx.Class.define("osparc.product.AboutProduct", { __buildLayout: function() { switch (osparc.product.Utils.getProductName()) { case "s4l": - this.__buildS4LLayout(); - break; case "s4lacad": - this.__buildS4LAcademicLayout(); + this.__buildS4LLayout(); break; case "s4llite": this.__buildS4LLiteLayout(); @@ -62,7 +60,7 @@ qx.Class.define("osparc.product.AboutProduct", { this.__buildTIPLayout(); break; default: { - const noInfoText = this.tr("Information not available"); + const noInfoText = this.tr("Information is unavailable"); const noInfoLabel = osparc.product.quickStart.Utils.createLabel(noInfoText); this.add(noInfoLabel); break; @@ -73,11 +71,11 @@ qx.Class.define("osparc.product.AboutProduct", { __buildS4LLayout: function() { const licenseUrl = osparc.store.Support.getLicenseURL(); const text = this.tr(` - sim4life.io is a native implementation of the most advanced simulation platform, Sim4Life, in the cloud. \ + Sim4Life.web is a native implementation of the most advanced simulation platform, Sim4Life, in the cloud. \ The platform empowers users to simulate, analyze, and predict complex, multifaceted, and dynamic biological interactions within the full anatomical complexity of the human body. \ It provides the ability to set up and run complex simulations directly within any browser, utilizing cloud technology.
- sim4life.io makes use of technologies developed by our research partner for the o2S2PARC platform, the IT’IS Foundation, and co-funded by the U.S. National Institutes of Health’s SPARC initiative.\ + Sim4Life.web makes use of technologies developed by our research partner for the o2S2PARC platform, the IT’IS Foundation, and co-funded by the U.S. National Institutes of Health’s SPARC initiative.\
For more information about Sim4Life, please visit ${osparc.utils.Utils.createHTMLLink("sim4life.swiss", "https://sim4life.swiss/")}.
@@ -90,26 +88,6 @@ qx.Class.define("osparc.product.AboutProduct", { this.add(label); }, - __buildS4LAcademicLayout: function() { - const licenseUrl = osparc.store.Support.getLicenseURL(); - const text = this.tr(` - sim4life.science is a native implementation of the most advanced simulation platform, Sim4Life, in the cloud. \ - The platform empowers users to simulate, analyze, and predict complex, multifaceted, and dynamic biological interactions within the full anatomical complexity of the human body. \ - It provides the ability to set up and run complex simulations directly within any browser, utilizing cloud technology. -
- sim4life.science makes use of technologies developed by our research partner for the o2S2PARC platform, the IT’IS Foundation, and co-funded by the U.S. National Institutes of Health’s SPARC initiative.\ -
- For more information about Sim4Life, please visit ${osparc.utils.Utils.createHTMLLink("sim4life.swiss", "href='https://sim4life.swiss/")}. -
- To review license agreements, click ${osparc.utils.Utils.createHTMLLink("here", licenseUrl)}. -
- Send us an email ${this.__getMailTo()} - `); - - const label = osparc.product.quickStart.Utils.createLabel(text); - this.add(label); - }, - __buildS4LLiteLayout: function() { // https://zurichmedtech.github.io/s4l-lite-manual/#/docs/what_is_s4l_lite const introText = "Sim4Life.lite is a powerful web-based simulation platform that allows you to model and analyze real-world phenomena and to design complex technical devices in a validated environment. With its intuitive interface and advanced tools, Sim4Life.lite makes it easy to develop your simulation project, wherever you are."; diff --git a/services/static-webserver/client/source/class/osparc/product/Utils.js b/services/static-webserver/client/source/class/osparc/product/Utils.js index 4c77b84e0e1..656ce42654e 100644 --- a/services/static-webserver/client/source/class/osparc/product/Utils.js +++ b/services/static-webserver/client/source/class/osparc/product/Utils.js @@ -188,6 +188,16 @@ qx.Class.define("osparc.product.Utils", { return "REGISTER"; }, + hasConvertToPipelineEnabled: function() { + return false; + }, + + // oSPARC only + hasExportCMisEnabled: function() { + const product = this.getProductName(); + return product === "osparc"; + }, + // All products except oSPARC hasIdlingTrackerEnabled: function() { const product = this.getProductName(); @@ -256,7 +266,25 @@ qx.Class.define("osparc.product.Utils", { return this.isS4LProduct() && licensesEnabled; }, - getProductThumbUrl: function(asset = "Default.png") { + getIconUrl: function(asset = "Default.png") { + const base = "https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/icons" + let url; + switch (osparc.product.Utils.getProductName()) { + case "osparc": + url = `${base}/osparc/${asset}`; + break; + case "tis": + case "tiplite": + url = `${base}/tip/${asset}`; + break; + default: + url = `${base}/s4l/${asset}`; + break; + } + return url; + }, + + getThumbnailUrl: function(asset = "Default.png") { const base = "https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/full/project_thumbnails" let url; switch (osparc.product.Utils.getProductName()) { @@ -274,7 +302,7 @@ qx.Class.define("osparc.product.Utils", { return url; }, - getProductBackgroundUrl: function(asset = "Thumbnail-01.png") { + getBackgroundUrl: function(asset = "Thumbnail-01.png") { const base = "https://raw.githubusercontent.com/ZurichMedTech/s4l-assets/main/app/full/background-images" let url; switch (osparc.product.Utils.getProductName()) { @@ -293,14 +321,7 @@ qx.Class.define("osparc.product.Utils", { }, hasNewPlusButton: function() { - return [ - "osparc", - "s4l", - "s4lacad", - "s4llite", - // "tis", - // "tiplite", - ].includes(osparc.product.Utils.getProductName()); + return Boolean(osparc.store.Products.getInstance().getPlusButtonUiConfig()); }, } }); diff --git a/services/static-webserver/client/source/class/osparc/product/tours/Tours.js b/services/static-webserver/client/source/class/osparc/product/tours/Tours.js index 5216ec483b1..8f9e061817a 100644 --- a/services/static-webserver/client/source/class/osparc/product/tours/Tours.js +++ b/services/static-webserver/client/source/class/osparc/product/tours/Tours.js @@ -26,12 +26,18 @@ qx.Class.define("osparc.product.tours.Tours", { statics: { TOURS: { - "s4llite": { - fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/s4llite_tours.json") + "osparc": { + fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/osparc_tours.json") }, "s4l": { fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/s4l_tours.json") }, + "s4lacad": { + fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/s4l_tours.json") + }, + "s4llite": { + fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/s4llite_tours.json") + }, "tis": { fetchTours: () => osparc.product.tours.Tours.fetchTours("/resource/osparc/tours/tis_tours.json") }, diff --git a/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js b/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js index 215b17d935b..9850b098455 100644 --- a/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js +++ b/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js @@ -47,8 +47,7 @@ qx.Class.define("osparc.service.PricingUnitsList", { }, __fetchUnits: function() { - const pricingStore = osparc.store.Pricing.getInstance(); - pricingStore.fetchPricingPlansService(this.__serviceMetadata["key"], this.__serviceMetadata["version"]) + osparc.store.Pricing.getInstance().fetchPricingPlansService(this.__serviceMetadata["key"], this.__serviceMetadata["version"]) .then(data => this.__populateList(data["pricingUnits"])) .catch(err => { console.error(err); diff --git a/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js b/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js index ce84ade1dcb..6c007907944 100644 --- a/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js +++ b/services/static-webserver/client/source/class/osparc/service/ServiceListItem.js @@ -24,8 +24,7 @@ qx.Class.define("osparc.service.ServiceListItem", { this.set({ width: this.self().ITEM_WIDTH, height: this.self().ITEM_HEIGHT, - paddingTop: 0, - paddingBottom: 0, + padding: this.self().PADDING, allowGrowX: true, focusable: true, }); @@ -53,8 +52,9 @@ qx.Class.define("osparc.service.ServiceListItem", { statics: { LATEST: "latest", ITEM_WIDTH: 550, - ITEM_HEIGHT: 35, - SERVICE_ICON: osparc.product.Utils.getProductThumbUrl() + ITEM_HEIGHT: 32 + 2*4, // thumbnail + 2*PADDING + PADDING: 4, + SERVICE_THUMBNAIL: osparc.product.Utils.getThumbnailUrl() }, members: { @@ -66,7 +66,7 @@ qx.Class.define("osparc.service.ServiceListItem", { if (service.getThumbnail()) { this.getChildControl("icon").setSource(service.getThumbnail()); } else { - this.getChildControl("icon").setSource(this.self().SERVICE_ICON); + this.getChildControl("icon").setSource(this.self().SERVICE_THUMBNAIL); } service.bind("name", this.getChildControl("title"), "value"); diff --git a/services/static-webserver/client/source/class/osparc/service/StatusUI.js b/services/static-webserver/client/source/class/osparc/service/StatusUI.js index 7fab5f32b70..2203da004c4 100644 --- a/services/static-webserver/client/source/class/osparc/service/StatusUI.js +++ b/services/static-webserver/client/source/class/osparc/service/StatusUI.js @@ -124,6 +124,8 @@ qx.Class.define("osparc.service.StatusUI", { return qx.locale.Manager.tr("Idle"); case "WAITING_FOR_RESOURCES": return qx.locale.Manager.tr("Waiting for resources"); + case "FAILED": + return qx.locale.Manager.tr("Unsuccessful"); // dynamics case "idle": @@ -131,7 +133,7 @@ qx.Class.define("osparc.service.StatusUI", { case "ready": return qx.locale.Manager.tr("Ready"); case "failed": - return qx.locale.Manager.tr("Failed"); + return qx.locale.Manager.tr("Unsuccessful"); case "deprecated": return qx.locale.Manager.tr("Deprecated"); case "retired": @@ -235,7 +237,7 @@ qx.Class.define("osparc.service.StatusUI", { const chip = new osparc.ui.basic.Chip().set({ label: osparc.service.Utils.DEPRECATED_SERVICE_TEXT, icon: osparc.service.StatusUI.getIconSource("deprecated"), - textColor: "contrasted-text-dark", + textColor: "text-on-warning", backgroundColor: osparc.service.StatusUI.getColor("deprecated"), allowGrowX: false }); @@ -246,7 +248,7 @@ qx.Class.define("osparc.service.StatusUI", { const chip = new osparc.ui.basic.Chip().set({ label: osparc.service.Utils.RETIRED_SERVICE_TEXT, icon: osparc.service.StatusUI.getIconSource("retired"), - textColor: "contrasted-text-dark", + textColor: "text-on-warning", backgroundColor: osparc.service.StatusUI.getColor("retired"), allowGrowX: false }); diff --git a/services/static-webserver/client/source/class/osparc/service/Utils.js b/services/static-webserver/client/source/class/osparc/service/Utils.js index 48639568506..ab1ee841c74 100644 --- a/services/static-webserver/client/source/class/osparc/service/Utils.js +++ b/services/static-webserver/client/source/class/osparc/service/Utils.js @@ -24,7 +24,7 @@ * Here is a little example of how to use the widget. * *
- * let latestSrv = osparc.service.Utils.getLatest(key); + * let latestSrv = osparc.store.Services.getLatest(key); **/ @@ -130,86 +130,11 @@ qx.Class.define("osparc.service.Utils", { return services; }, - getVersions: function(key, filterDeprecated = true) { - const services = osparc.store.Services.servicesCached; - let versions = []; - if (key in services) { - const serviceVersions = services[key]; - versions = versions.concat(Object.keys(serviceVersions)); - if (filterDeprecated) { - versions = versions.filter(version => { - if (services[key][version]["retired"]) { - return false; - } - return true; - }); - } - versions.sort(osparc.utils.Utils.compareVersionNumbers); - } - return versions.reverse(); - }, - - getLatest: function(key) { - const services = osparc.store.Services.servicesCached; - if (key in services) { - const versions = this.getVersions(key, true); - if (versions.length) { - return services[key][versions[0]]; - } - } - return null; - }, - - getLatestCompatible: function(key, version) { - const services = osparc.store.Services.servicesCached; - if (key in services && version in services[key]) { - const serviceMD = services[key][version]; - if (serviceMD["compatibility"] && serviceMD["compatibility"]["canUpdateTo"]) { - const canUpdateTo = serviceMD["compatibility"]["canUpdateTo"]; - return { - key: "key" in canUpdateTo ? canUpdateTo["key"] : key, // key is optional - version: canUpdateTo["version"] - } - } - // the provided key/version itself is the latest compatible - return { - key, - version - } - } - return null; - }, - - getVersionDisplay: function(key, version) { - const services = osparc.store.Services.servicesCached; - if (key in services && version in services[key]) { - return this.extractVersionDisplay(services[key][version]); - } - return null; - }, - extractVersionDisplay: function(metadata) { - return metadata["versionDisplay"] ? metadata["versionDisplay"] : metadata["version"]; - }, - - getReleasedDate: function(key, version) { - const services = osparc.store.Services.servicesCached; - if ( - key in services && - version in services[key] && - "released" in services[key][version] - ) { - return services[key][version]["released"]; + if (metadata) { + return metadata["versionDisplay"] ? metadata["versionDisplay"] : metadata["version"]; } - return null; - }, - - versionToListItem: function(key, version) { - const versionDisplay = this.getVersionDisplay(key, version); - const listItem = new qx.ui.form.ListItem(versionDisplay); - osparc.utils.Utils.setIdToWidget(listItem, "serviceVersionItem_" + versionDisplay); - listItem.version = version; - return listItem; + return ""; }, canIWrite: function(serviceAccessRights) { @@ -228,14 +153,38 @@ qx.Class.define("osparc.service.Utils", { DEPRECATED_AUTOUPDATABLE_INSTRUCTIONS: qx.locale.Manager.tr("Please Stop the Service and then Update it"), RETIRED_AUTOUPDATABLE_INSTRUCTIONS: qx.locale.Manager.tr("Please Update the Service"), + extractVersionFromHistory: function(metadata) { + if (metadata["history"]) { + const found = metadata["history"].find(historyEntry => historyEntry["version"] === metadata["version"]); + return found; + } + return null; + }, + isUpdatable: function(metadata) { - const latestCompatible = this.getLatestCompatible(metadata["key"], metadata["version"]); - return latestCompatible && (metadata["key"] !== latestCompatible["key"] || metadata["version"] !== latestCompatible["version"]); + const historyEntry = this.extractVersionFromHistory(metadata); + if (historyEntry && historyEntry["compatibility"] && historyEntry["compatibility"]["canUpdateTo"]) { + const latestCompatible = historyEntry["compatibility"]["canUpdateTo"]; + return latestCompatible && (metadata["key"] !== latestCompatible["key"] || metadata["version"] !== latestCompatible["version"]); + } + return false; + }, + + __extractRetiredDate: function(metadata) { + if ("release" in metadata && metadata["release"]["retired"]) { + // this works for service latest + return new Date(metadata["release"]["retired"]); + } + const historyEntry = this.extractVersionFromHistory(metadata); + if (historyEntry && "retired" in historyEntry && historyEntry["retired"]) { + return new Date(historyEntry["retired"]); + } + return null; }, isDeprecated: function(metadata) { - if (metadata && "deprecated" in metadata && ![null, undefined].includes(metadata["deprecated"])) { - const deprecationTime = new Date(metadata["deprecated"]); + const deprecationTime = this.__extractRetiredDate(metadata); + if (deprecationTime) { const now = new Date(); return deprecationTime.getTime() > now.getTime(); } @@ -243,8 +192,8 @@ qx.Class.define("osparc.service.Utils", { }, isRetired: function(metadata) { - if (metadata && "deprecated" in metadata && ![null, undefined].includes(metadata["deprecated"])) { - const deprecationTime = new Date(metadata["deprecated"]); + const deprecationTime = this.__extractRetiredDate(metadata); + if (deprecationTime) { const now = new Date(); return deprecationTime.getTime() < now.getTime(); } @@ -252,34 +201,11 @@ qx.Class.define("osparc.service.Utils", { }, getDeprecationDateText: function(metadata) { - const deprecationTime = new Date(metadata["deprecated"]); - return qx.locale.Manager.tr("It will be Retired: ") + osparc.utils.Utils.formatDate(deprecationTime); - }, - - getFilePicker: function() { - return this.self().getLatest("simcore/services/frontend/file-picker"); - }, - - getParametersMetadata: function() { - const parametersMetadata = []; - const services = osparc.store.Services.servicesCached; - for (const key in services) { - if (key.includes("simcore/services/frontend/parameter/")) { - const latest = this.self().getLatest(key); - if (latest) { - parametersMetadata.push(latest); - } - } + if (this.isDeprecated(metadata) || this.isRetired(metadata)) { + const deprecationTime = this.__extractRetiredDate(metadata); + return qx.locale.Manager.tr("It will be Retired: ") + osparc.utils.Utils.formatDate(deprecationTime); } - return parametersMetadata; - }, - - getParameterMetadata: function(type) { - return this.self().getLatest("simcore/services/frontend/parameter/"+type); - }, - - getProbeMetadata: function(type) { - return this.self().getLatest("simcore/services/frontend/iterator-consumer/probe/"+type); + return ""; }, removeFileToKeyMap: function(service) { diff --git a/services/static-webserver/client/source/class/osparc/share/Collaborators.js b/services/static-webserver/client/source/class/osparc/share/Collaborators.js index 004666fcbab..ea819493cab 100644 --- a/services/static-webserver/client/source/class/osparc/share/Collaborators.js +++ b/services/static-webserver/client/source/class/osparc/share/Collaborators.js @@ -335,6 +335,21 @@ qx.Class.define("osparc.share.Collaborators", { }); item.addListener("removeMember", e => { const orgMember = e.getData(); + if ( + ["study", "template"].includes(this._resourceType) && + !osparc.share.CollaboratorsStudy.canCollaboratorBeRemoved(this._serializedDataCopy, orgMember["gid"]) + ) { + let msg = this.tr("Collaborator can't be removed:"); + msg += this._serializedDataCopy["name"] + this.tr(" needs at least one owner."); + if ( + Object.keys(this._serializedDataCopy["accessRights"]).length === 1 && + Object.values(this._serializedDataCopy["accessRights"])[0]["delete"] + ) { + msg += "
" + this.tr("You might want to delete it instead."); + } + osparc.FlashMessenger.logError(msg); + return; + } this._deleteMember(orgMember, item); }); } @@ -349,11 +364,8 @@ qx.Class.define("osparc.share.Collaborators", { __getLeaveStudyButton: function() { const myGid = osparc.auth.Data.getInstance().getGroupId(); if ( - (this._resourceType === "study") && - // check if I'm part of the access rights (not through an organization) - Object.keys(this._serializedDataCopy["accessRights"]).includes(myGid.toString()) && - // check also user is not "prjOwner". Backend will silently not let the frontend remove that user. - (this._serializedDataCopy["prjOwner"] !== osparc.auth.Data.getInstance().getEmail()) + ["study", "template"].includes(this._resourceType) && + osparc.share.CollaboratorsStudy.canCollaboratorBeRemoved(this._serializedDataCopy, myGid) ) { const leaveText = this.tr("Leave") + " " + osparc.product.Utils.getStudyAlias({ firstUpperCase: true @@ -363,29 +375,14 @@ qx.Class.define("osparc.share.Collaborators", { visibility: Object.keys(this._serializedDataCopy["accessRights"]).includes(myGid.toString()) ? "visible" : "excluded" }); leaveButton.addListener("execute", () => { - let msg = `"${this._serializedDataCopy["name"]}" ` + this.tr("will no longer be listed."); - if (!osparc.share.CollaboratorsStudy.checkRemoveCollaborator(this._serializedDataCopy, myGid)) { - msg += "
"; - msg += this.tr("If you remove yourself, there won't be any other Owners."); + const collaborator = { + gid: myGid, + name: osparc.store.Groups.getInstance().getGroupMe().getLabel(), } - const win = new osparc.ui.window.Confirmation(msg).set({ - caption: leaveText, - confirmText: this.tr("Leave"), - confirmAction: "delete" - }); - win.open(); - win.addListener("close", () => { - if (win.getConfirmed()) { - const collaborator = { - gid: myGid, - name: osparc.store.Groups.getInstance().getGroupMe().getLabel(), - } - this._deleteMember(collaborator) - .then(() => { - qx.event.message.Bus.dispatchByName("reloadStudies"); - }); - } - }, this); + this._deleteMember(collaborator) + .then(() => { + qx.event.message.Bus.dispatchByName("reloadStudies"); + }); }, this); return leaveButton; } @@ -431,7 +428,7 @@ qx.Class.define("osparc.share.Collaborators", { "description": collab.getDescription(), }; if (!("getUserId" in collab)) { - // orgnanization + // organization if (everyoneGIds.includes(parseInt(gid))) { collaborator["thumbnail"] = "@FontAwesome5Solid/globe/32"; } else if (!collaborator["thumbnail"]) { diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js index bd16086cb27..5ca0e1717d7 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsService.js @@ -76,13 +76,10 @@ qx.Class.define("osparc.share.CollaboratorsService", { .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); const text = this.tr("Service successfully shared"); - osparc.FlashMessenger.getInstance().logAs(text); + osparc.FlashMessenger.logAs(text); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Service"), "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while sharing the service"))); }, _deleteMember: function(collaborator, item) { @@ -92,7 +89,7 @@ qx.Class.define("osparc.share.CollaboratorsService", { const success = delete this._serializedDataCopy["accessRights"][collaborator["gid"]]; if (!success) { - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing Member"), "ERROR"); + osparc.FlashMessenger.logError(this.tr("Something went wrong while removing member")); if (item) { item.setEnabled(true); } @@ -102,13 +99,10 @@ qx.Class.define("osparc.share.CollaboratorsService", { osparc.store.Services.patchServiceData(this._serializedDataCopy, "accessRights", this._serializedDataCopy["accessRights"]) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); + osparc.FlashMessenger.logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while removing ") + collaborator["name"])) .finally(() => { if (item) { item.setEnabled(true); @@ -122,13 +116,10 @@ qx.Class.define("osparc.share.CollaboratorsService", { osparc.store.Services.patchServiceData(this._serializedDataCopy, "accessRights", this._serializedDataCopy["accessRights"]) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(successMsg); + osparc.FlashMessenger.logAs(successMsg); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(failureMsg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, failureMsg)) .finally(() => item.setEnabled(true)); }, @@ -137,13 +128,13 @@ qx.Class.define("osparc.share.CollaboratorsService", { collaborator["gid"], this.self().getOwnerAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.SERVICE[2].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.SERVICE[2].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.SERVICE[2].label}`), item ); }, _promoteToOwner: function(collaborator, item) { - osparc.FlashMessenger.getInstance().logAs(this.tr("Operation not available"), "WARNING"); + osparc.FlashMessenger.logAs(this.tr("Operation not available"), "WARNING"); }, _demoteToUser: function(collaborator, item) { @@ -151,13 +142,13 @@ qx.Class.define("osparc.share.CollaboratorsService", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.SERVICE[1].label}`), - this.tr(`Something went wrong demoting ${osparc.data.Roles.SERVICE[1].label}`), + this.tr(`Something went wrong while demoting ${osparc.data.Roles.SERVICE[1].label}`), item ); }, _demoteToEditor: function(collaborator, item) { - osparc.FlashMessenger.getInstance().logAs(this.tr("Operation not available"), "WARNING"); + osparc.FlashMessenger.logAs(this.tr("Operation not available"), "WARNING"); } } }); diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js index 771a4284331..a1fa2fb1ef8 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsStudy.js @@ -92,7 +92,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { }, // checks that if the user to remove is an owner, there will still be another owner - checkRemoveCollaborator: function(studyData, gid) { + canCollaboratorBeRemoved: function(studyData, gid) { const ownerGids = this.__getDeleters(studyData); if (ownerGids.includes(gid.toString())) { return ownerGids.length > 1; @@ -114,20 +114,17 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { gids.forEach(gid => { newCollaborators[gid] = this._resourceType === "study" ? this.self().getCollaboratorAccessRight() : this.self().getViewerAccessRight(); }); - osparc.info.StudyUtils.addCollaborators(this._serializedDataCopy, newCollaborators) + osparc.store.Study.addCollaborators(this._serializedDataCopy, newCollaborators) .then(() => { const text = resourceAlias + this.tr(" successfully shared"); - osparc.FlashMessenger.getInstance().logAs(text); + osparc.FlashMessenger.logAs(text); this.fireDataEvent("updateAccessRights", this._serializedDataCopy); this._reloadCollaboratorsList(); this.__pushNotifications(gids); this.__checkShareePermissions(gids); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the ") + resourceAlias, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while sharing the ") + resourceAlias)); }, _deleteMember: function(collaborator, item) { @@ -135,16 +132,13 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { item.setEnabled(false); } - return osparc.info.StudyUtils.removeCollaborator(this._serializedDataCopy, collaborator["gid"]) + return osparc.store.Study.removeCollaborator(this._serializedDataCopy, collaborator["gid"]) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); + osparc.FlashMessenger.logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while removing ") + collaborator["name"])) .finally(() => { if (item) { item.setEnabled(true); @@ -155,16 +149,13 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { __make: function(collaboratorGId, newAccessRights, successMsg, failureMsg, item) { item.setEnabled(false); - osparc.info.StudyUtils.updateCollaborator(this._serializedDataCopy, collaboratorGId, newAccessRights) + osparc.store.Study.updateCollaborator(this._serializedDataCopy, collaboratorGId, newAccessRights) .then(() => { this.fireDataEvent("updateAccessRights", this._serializedDataCopy); - osparc.FlashMessenger.getInstance().logAs(successMsg); + osparc.FlashMessenger.logAs(successMsg); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(failureMsg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, failureMsg)) .finally(() => { if (item) { item.setEnabled(true); @@ -177,7 +168,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.STUDY[2].label}`), item ); }, @@ -187,7 +178,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { collaborator["gid"], this.self().getOwnerAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[3].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[3].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.STUDY[3].label}`), item ); }, @@ -199,7 +190,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { gid, this.self().getViewerAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[1].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[1].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.STUDY[1].label}`), itm ); }; @@ -229,7 +220,7 @@ qx.Class.define("osparc.share.CollaboratorsStudy", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.STUDY[2].label}`), item ); }, diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsTag.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsTag.js index 0327bf29589..2f2dd4f5b32 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsTag.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsTag.js @@ -77,14 +77,11 @@ qx.Class.define("osparc.share.CollaboratorsTag", { osparc.store.Tags.getInstance().addCollaborators(this.__tag.getTagId(), newCollaborators) .then(() => { const text = this.tr("Tag successfully shared"); - osparc.FlashMessenger.getInstance().logAs(text); + osparc.FlashMessenger.logAs(text); this.fireDataEvent("updateAccessRights", this.__tag.serialize()); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Tag"), "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while sharing the tag"))); }, _deleteMember: function(collaborator, item) { @@ -95,13 +92,10 @@ qx.Class.define("osparc.share.CollaboratorsTag", { osparc.store.Tags.getInstance().removeCollaborator(this.__tag.getTagId(), collaborator["gid"]) .then(() => { this.fireDataEvent("updateAccessRights", this.__tag.serialize()); - osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); + osparc.FlashMessenger.logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while removing ") + collaborator["name"])) .finally(() => { if (item) { item.setEnabled(true); @@ -115,13 +109,10 @@ qx.Class.define("osparc.share.CollaboratorsTag", { osparc.store.Tags.getInstance().updateCollaborator(this.__tag.getTagId(), collaboratorGId, newAccessRights) .then(() => { this.fireDataEvent("updateAccessRights", this.__tag.serialize()); - osparc.FlashMessenger.getInstance().logAs(successMsg); + osparc.FlashMessenger.logAs(successMsg); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(failureMsg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, failureMsg)) .finally(() => { if (item) { item.setEnabled(true); @@ -134,7 +125,7 @@ qx.Class.define("osparc.share.CollaboratorsTag", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.STUDY[2].label}`), item ); }, @@ -144,7 +135,7 @@ qx.Class.define("osparc.share.CollaboratorsTag", { collaborator["gid"], this.self().getOwnerAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.STUDY[3].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.STUDY[3].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.STUDY[3].label}`), item ); }, @@ -154,7 +145,7 @@ qx.Class.define("osparc.share.CollaboratorsTag", { collaborator["gid"], this.self().getViewerAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[1].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[1].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.STUDY[1].label}`), item ); }, @@ -164,7 +155,7 @@ qx.Class.define("osparc.share.CollaboratorsTag", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.STUDY[2].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.STUDY[2].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.STUDY[2].label}`), item ); } diff --git a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js index 97dd58bc573..9accc310f84 100644 --- a/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js +++ b/services/static-webserver/client/source/class/osparc/share/CollaboratorsWorkspace.js @@ -73,14 +73,11 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { osparc.store.Workspaces.getInstance().addCollaborators(this.__workspace.getWorkspaceId(), newCollaborators) .then(() => { const text = this.tr("Workspace successfully shared"); - osparc.FlashMessenger.getInstance().logAs(text); + osparc.FlashMessenger.logAs(text); this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong sharing the Workspace"), "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while sharing the workspace"))); }, _deleteMember: function(collaborator, item) { @@ -91,13 +88,10 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { osparc.store.Workspaces.getInstance().removeCollaborator(this.__workspace.getWorkspaceId(), collaborator["gid"]) .then(() => { this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); - osparc.FlashMessenger.getInstance().logAs(collaborator["name"] + this.tr(" successfully removed")); + osparc.FlashMessenger.logAs(collaborator["name"] + this.tr(" successfully removed")); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Something went wrong removing ") + collaborator["name"], "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Something went wrong while removing ") + collaborator["name"])) .finally(() => { if (item) { item.setEnabled(true); @@ -111,13 +105,10 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { osparc.store.Workspaces.getInstance().updateCollaborator(this.__workspace.getWorkspaceId(), collaboratorGId, newAccessRights) .then(() => { this.fireDataEvent("updateAccessRights", this.__workspace.serialize()); - osparc.FlashMessenger.getInstance().logAs(successMsg); + osparc.FlashMessenger.logAs(successMsg); this._reloadCollaboratorsList(); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(failureMsg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, failureMsg)) .finally(() => { if (item) { item.setEnabled(true); @@ -130,7 +121,7 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.WORKSPACE[2].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.WORKSPACE[2].label}`), item ); }, @@ -140,7 +131,7 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { collaborator["gid"], this.self().getOwnerAccessRight(), this.tr(`Successfully promoted to ${osparc.data.Roles.WORKSPACE[3].label}`), - this.tr(`Something went wrong promoting to ${osparc.data.Roles.WORKSPACE[3].label}`), + this.tr(`Something went wrong while promoting to ${osparc.data.Roles.WORKSPACE[3].label}`), item ); }, @@ -152,7 +143,7 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { gid, this.self().getViewerAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.WORKSPACE[1].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.WORKSPACE[1].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.WORKSPACE[1].label}`), itm ); }; @@ -182,7 +173,7 @@ qx.Class.define("osparc.share.CollaboratorsWorkspace", { collaborator["gid"], this.self().getCollaboratorAccessRight(), this.tr(`Successfully demoted to ${osparc.data.Roles.WORKSPACE[2].label}`), - this.tr(`Something went wrong demoting to ${osparc.data.Roles.WORKSPACE[2].label}`), + this.tr(`Something went wrong while demoting to ${osparc.data.Roles.WORKSPACE[2].label}`), item ); } diff --git a/services/static-webserver/client/source/class/osparc/share/NewCollaboratorsManager.js b/services/static-webserver/client/source/class/osparc/share/NewCollaboratorsManager.js index 7f757b831a3..3008c818c99 100644 --- a/services/static-webserver/client/source/class/osparc/share/NewCollaboratorsManager.js +++ b/services/static-webserver/client/source/class/osparc/share/NewCollaboratorsManager.js @@ -124,10 +124,7 @@ qx.Class.define("osparc.share.NewCollaboratorsManager", { users.forEach(user => user["collabType"] = 2); this.__addPotentialCollaborators(users); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => this.__searchingCollaborators.exclude()); }, diff --git a/services/static-webserver/client/source/class/osparc/snapshots/IterationsView.js b/services/static-webserver/client/source/class/osparc/snapshots/IterationsView.js index c9b1557f111..fd2c8eaa384 100644 --- a/services/static-webserver/client/source/class/osparc/snapshots/IterationsView.js +++ b/services/static-webserver/client/source/class/osparc/snapshots/IterationsView.js @@ -76,7 +76,7 @@ qx.Class.define("osparc.snapshots.IterationsView", { "studyId": iteration["workcopy_project_id"] } }; - iterationPromises.push(osparc.data.Resources.getOne("studies", params)); + iterationPromises.push(osparc.data.Resources.fetch("studies", "getOne", params)); }); Promise.all(iterationPromises) .then(values => { @@ -204,7 +204,7 @@ qx.Class.define("osparc.snapshots.IterationsView", { "studyId": iterationId } }; - osparc.data.Resources.getOne("studies", params) + osparc.data.Resources.fetch("studies", "getOne", params) .then(data => { const studyData = this.__study.serialize(); studyData["workbench"] = data["workbench"]; @@ -275,7 +275,7 @@ qx.Class.define("osparc.snapshots.IterationsView", { .then(() => { this.__rebuildSnapshots(); }) - .catch(err => osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR")); + .catch(err => osparc.FlashMessenger.logError(err)); win.close(); }, this); editSnapshotView.addListener("cancel", () => { diff --git a/services/static-webserver/client/source/class/osparc/snapshots/SnapshotsView.js b/services/static-webserver/client/source/class/osparc/snapshots/SnapshotsView.js index 6cbef9261a0..25ebd5d6550 100644 --- a/services/static-webserver/client/source/class/osparc/snapshots/SnapshotsView.js +++ b/services/static-webserver/client/source/class/osparc/snapshots/SnapshotsView.js @@ -179,7 +179,7 @@ qx.Class.define("osparc.snapshots.SnapshotsView", { .then(() => { this.__rebuildSnapshots(); }) - .catch(err => osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR")); + .catch(err => osparc.FlashMessenger.logError(err)); win.close(); }, this); editSnapshotView.addListener("cancel", () => { diff --git a/services/static-webserver/client/source/class/osparc/store/Data.js b/services/static-webserver/client/source/class/osparc/store/Data.js index 322dc1313bd..202a3b4788d 100644 --- a/services/static-webserver/client/source/class/osparc/store/Data.js +++ b/services/static-webserver/client/source/class/osparc/store/Data.js @@ -35,17 +35,52 @@ qx.Class.define("osparc.store.Data", { "fileCopied": "qx.event.type.Data", }, + statics: { + getAllItems: async function(locationId, path, cursor, allItems = []) { + if (allItems.length >= 10000) { + const msg = qx.locale.Manager.tr("Oops... more than 10.000 items to be listed here. Maybe it's time to make a folder :)."); + osparc.FlashMessenger.logAs(msg, "WARNING"); + return allItems; + } + + const params = { + url: { + locationId, + path: path || null, + cursor: cursor || null, + } + }; + let pagResp = null; + if (path) { + pagResp = await osparc.data.Resources.fetch("storagePaths", cursor ? "getPathsPage" : "getPaths", params); + } else { + pagResp = await osparc.data.Resources.fetch("storagePaths", cursor ? "getDatasetsPage" : "getDatasets", params); + } + + let nextCursor = null; + if (pagResp) { + if (pagResp["items"]) { + allItems.push(...pagResp["items"]); + } + if (pagResp["next_page"]) { + nextCursor = pagResp["next_page"]; + } + } + + if (nextCursor) { + return this.getAllItems(locationId, path, nextCursor, allItems); + } + return allItems; + }, + }, + members: { __locationsCached: null, __datasetsByLocationCached: null, - __filesByLocationAndDatasetCached: null, resetCache: function() { this.__locationsCached = []; this.__datasetsByLocationCached = {}; - this.__filesByLocationAndDatasetCached = {}; - - osparc.store.Store.getInstance().reset("storageLocations"); }, getLocationsCached: function() { @@ -62,8 +97,7 @@ qx.Class.define("osparc.store.Data", { if (cachedData) { resolve(cachedData); } else { - // Get available storage locations - osparc.data.Resources.get("storageLocations") + osparc.data.Resources.fetch("storageLocations", "getLocations") .then(locations => { // Add them to cache this.__locationsCached = locations; @@ -82,132 +116,51 @@ qx.Class.define("osparc.store.Data", { if (locationId in cache && cache[locationId] && cache[locationId].length) { const data = { location: locationId, - datasets: cache[locationId] + items: cache[locationId] }; return data; } return null; }, - getDatasetsByLocation: function(locationId) { - const emptyData = { + getDatasetsByLocation: async function(locationId) { + const data = { location: locationId, - datasets: [] + items: [] }; - return new Promise((resolve, reject) => { - // Get list of datasets - if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) { - reject(emptyData); - } + if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) { + return data; + } - const cachedData = this.getDatasetsByLocationCached(locationId); - if (cachedData) { - resolve(cachedData); - } else { - const params = { - url: { - locationId - } - }; - osparc.data.Resources.fetch("storageDatasets", "getByLocation", params) - .then(datasets => { - const data = { - location: locationId, - datasets: [] - }; - if (datasets && datasets.length>0) { - data.datasets = datasets; - } - // Add it to cache - this.__datasetsByLocationCached[locationId] = data.datasets; - resolve(data); - }) - .catch(err => { - console.error(err); - reject(emptyData); - }); - } - }); - }, + const cachedData = this.getDatasetsByLocationCached(locationId); + if (cachedData) { + return cachedData; + } - getFilesByLocationAndDatasetCached: function(locationId, datasetId) { - const cache = this.__filesByLocationAndDatasetCached; - if (locationId in cache && datasetId in cache[locationId]) { - const data = { - location: locationId, - dataset: datasetId, - files: cache[locationId][datasetId] - }; + try { + const allItems = await this.self().getAllItems(locationId); + this.__datasetsByLocationCached[locationId] = allItems; + data["items"] = allItems; + return data; + } catch (err) { + console.error(err); return data; } - return null; }, - getFilesByLocationAndDataset: function(locationId, datasetId) { - const emptyData = { - location: locationId, - dataset: datasetId, - files: [] - }; - return new Promise((resolve, reject) => { - // Get list of file meta data - if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) { - reject(emptyData); - } - - const cachedData = this.getFilesByLocationAndDatasetCached(locationId, datasetId); - if (cachedData) { - resolve(cachedData); - } else { - const params = { - url: { - locationId, - datasetId - } - }; - osparc.data.Resources.fetch("storageFiles", "getByLocationAndDataset", params) - .then(files => { - const data = { - location: locationId, - dataset: datasetId, - files: files && files.length>0 ? files : [] - }; - // Add it to cache - if (!(locationId in this.__filesByLocationAndDatasetCached)) { - this.__filesByLocationAndDatasetCached[locationId] = {}; - } - this.__filesByLocationAndDatasetCached[locationId][datasetId] = data.files; - resolve(data); - }) - .catch(err => { - console.error(err); - reject(emptyData); - }); - } - }); - }, + getItemsByLocationAndPath: async function(locationId, path) { + // Get list of file meta data + if (locationId === 1 && !osparc.data.Permissions.getInstance().canDo("storage.datcore.read")) { + return []; + } - getNodeFiles: function(nodeId) { - return new Promise((resolve, reject) => { - const params = { - url: { - nodeId: encodeURIComponent(nodeId) - } - }; - osparc.data.Resources.fetch("storageFiles", "getByNode", params) - .then(files => { - console.log("Node Files", files); - if (files && files.length>0) { - resolve(files); - } else { - resolve([]); - } - }) - .catch(err => { - console.error(err); - reject([]); - }); - }); + try { + const allItems = await this.self().getAllItems(locationId, path); + return allItems; + } catch (err) { + console.error(err); + return []; + } }, getPresignedLink: function(download = true, locationId, fileUuid, fileSize) { @@ -264,7 +217,7 @@ qx.Class.define("osparc.store.Data", { fileUuid: encodeURIComponent(fileUuid) } }; - osparc.data.Resources.fetch("storageFiles", "put", params) + osparc.data.Resources.fetch("storageFiles", "copy", params) .then(files => { const data = { data: files, @@ -274,15 +227,14 @@ qx.Class.define("osparc.store.Data", { this.fireDataEvent("fileCopied", data); }) .catch(err => { - console.error(err); - console.error("Failed copying file", fileUuid, "to", pathId); - osparc.FlashMessenger.getInstance().logAs(this.tr("Failed copying file"), "ERROR"); + osparc.FlashMessenger.logError(err, this.tr("Unsuccessful file copy")); this.fireDataEvent("fileCopied", null); }); return true; }, + // if folder path is provided as fileUuid, it can also be deleted deleteFile: function(locationId, fileUuid) { if (!osparc.data.Permissions.getInstance().canDo("study.node.data.delete", true)) { return null; @@ -304,10 +256,7 @@ qx.Class.define("osparc.store.Data", { }; return data; }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.getInstance().logAs(this.tr("Failed deleting file"), "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Unsuccessful file deletion"))); } } }); diff --git a/services/static-webserver/client/source/class/osparc/store/Jobs.js b/services/static-webserver/client/source/class/osparc/store/Jobs.js new file mode 100644 index 00000000000..155a9759755 --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/store/Jobs.js @@ -0,0 +1,78 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2025 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +/** + * @asset(osparc/mock_jobs.json") + */ + +qx.Class.define("osparc.store.Jobs", { + extend: qx.core.Object, + type: "singleton", + + properties: { + jobs: { + check: "Array", + init: [], + nullable: true, + event: "changeJobs" + } + }, + + members: { + fetchJobs: function() { + return osparc.utils.Utils.fetchJSON("/resource/osparc/mock_jobs.json") + .then(jobsData => { + if ("jobs" in jobsData) { + jobsData["jobs"].forEach(jobData => { + this.addJob(jobData); + }); + } + return this.getJobs(); + }) + .catch(err => console.error(err)); + }, + + fetchJobInfo: function(jobId) { + return osparc.utils.Utils.fetchJSON("/resource/osparc/mock_jobs.json") + .then(jobsData => { + if ("jobs_info" in jobsData && jobId in jobsData["jobs_info"]) { + return jobsData["jobs_info"][jobId]; + } + return null; + }) + .catch(err => console.error(err)); + }, + + addJob: function(jobData) { + const jobs = this.getJobs(); + const index = jobs.findIndex(t => t.getJobId() === jobData["job_id"]); + if (index === -1) { + const job = new osparc.data.Job(jobData); + jobs.push(job); + this.fireEvent("changeJobs"); + return job; + } + return null; + }, + + removeJobs: function() { + const jobs = this.getJobs(); + jobs.forEach(job => job.dispose()); + this.fireEvent("changeJobs"); + }, + } +}); diff --git a/services/static-webserver/client/source/class/osparc/store/LicensedItems.js b/services/static-webserver/client/source/class/osparc/store/LicensedItems.js index 7e86d835d98..5fc436bfdb5 100644 --- a/services/static-webserver/client/source/class/osparc/store/LicensedItems.js +++ b/services/static-webserver/client/source/class/osparc/store/LicensedItems.js @@ -22,60 +22,53 @@ qx.Class.define("osparc.store.LicensedItems", { construct: function() { this.base(arguments); - this.__licensedItems = null; - this.__modelsCache = {}; + this.__licensedItems = {}; }, statics: { - VIP_MODELS: { - HUMAN_BODY: "https://itis.swiss/PD_DirectDownload/getDownloadableItems/HumanWholeBody", - HUMAN_BODY_REGION: "https://itis.swiss/PD_DirectDownload/getDownloadableItems/HumanBodyRegion", - ANIMAL: "https://itis.swiss/PD_DirectDownload/getDownloadableItems/AnimalWholeBody", - PHANTOM: "https://speag.swiss/PD_DirectDownload/getDownloadableItems/ComputationalPhantom", + getLowerLicensedItems: function(licensedItems, key, version) { + const lowerLicensedItems = []; + licensedItems.forEach(licensedItem => { + if (licensedItem["key"] === key && licensedItem["version"] < version) { + lowerLicensedItems.push(licensedItem); + } + }); + return lowerLicensedItems; }, - curateAnatomicalModels: function(anatomicalModelsRaw) { - const anatomicalModels = []; - const models = anatomicalModelsRaw["availableDownloads"]; - models.forEach(model => { - const curatedModel = {}; - Object.keys(model).forEach(key => { - if (key === "Features") { - let featuresRaw = model["Features"]; - featuresRaw = featuresRaw.substring(1, featuresRaw.length-1); // remove brackets - featuresRaw = featuresRaw.split(","); // split the string by commas - const features = {}; - featuresRaw.forEach(pair => { // each pair is "key: value" - const keyValue = pair.split(":"); - features[keyValue[0].trim()] = keyValue[1].trim() - }); - curatedModel["Features"] = features; - } else { - curatedModel[key] = model[key]; - } - }); - anatomicalModels.push(curatedModel); + seatsToNSeats: function(seats) { + let nSeats = 0; + seats.forEach(seat => { + if ("numOfSeats" in seat) { + nSeats += seat["numOfSeats"]; + } else if ("getNumOfSeats" in seat) { + nSeats += seat.getNumOfSeats(); + } }); - return anatomicalModels; + return nSeats; }, }, members: { __licensedItems: null, - __modelsCache: null, getLicensedItems: function() { - if (this.__licensedItems) { + if (Object.keys(this.__licensedItems).length) { return new Promise(resolve => resolve(this.__licensedItems)); } return osparc.data.Resources.getInstance().getAllPages("licensedItems") - .then(licensedItems => { - this.__licensedItems = licensedItems; + .then(licensedItemsData => { + licensedItemsData.forEach(licensedItemData => this.__addLicensedItemsToCache(licensedItemData)); return this.__licensedItems; }); }, + __addLicensedItemsToCache: function(licensedItemData) { + const licensedItem = new osparc.data.model.LicensedItem(licensedItemData); + this.__licensedItems[licensedItem.getLicensedItemId()] = licensedItem; + }, + getPurchasedLicensedItems: function(walletId, urlParams, options = {}) { let purchasesParams = { url: { @@ -90,7 +83,7 @@ qx.Class.define("osparc.store.LicensedItems", { return osparc.data.Resources.fetch("licensedItems", "purchases", purchasesParams, options); }, - purchaseLicensedItem: function(licensedItemId, walletId, pricingPlanId, pricingUnitId, numberOfSeats) { + purchaseLicensedItem: function(licensedItemId, walletId, pricingPlanId, pricingUnitId, numOfSeats) { const params = { url: { licensedItemId @@ -99,7 +92,7 @@ qx.Class.define("osparc.store.LicensedItems", { "wallet_id": walletId, "pricing_plan_id": pricingPlanId, "pricing_unit_id": pricingUnitId, - "num_of_seats": numberOfSeats, // this should go away + "num_of_seats": numOfSeats, // this should go away }, } return osparc.data.Resources.fetch("licensedItems", "purchase", params); @@ -118,48 +111,5 @@ qx.Class.define("osparc.store.LicensedItems", { } return osparc.data.Resources.fetch("licensedItems", "checkouts", purchasesParams, options); }, - - __fetchVipModels: async function(vipSubset) { - if (!(vipSubset in this.self().VIP_MODELS)) { - return []; - } - - if (vipSubset in this.__modelsCache) { - return this.__modelsCache[vipSubset]; - } - - return await fetch(this.self().VIP_MODELS[vipSubset], { - method:"POST" - }) - .then(resp => resp.json()) - .then(anatomicalModelsRaw => { - const allAnatomicalModels = this.self().curateAnatomicalModels(anatomicalModelsRaw); - const anatomicalModels = []; - allAnatomicalModels.forEach(model => { - const anatomicalModel = {}; - anatomicalModel["modelId"] = model["ID"]; - anatomicalModel["thumbnail"] = model["Thumbnail"]; - anatomicalModel["name"] = model["Features"]["name"] + " " + model["Features"]["version"]; - anatomicalModel["description"] = model["Description"]; - anatomicalModel["features"] = model["Features"]; - anatomicalModel["date"] = model["Features"]["date"]; - anatomicalModel["DOI"] = model["DOI"]; - anatomicalModels.push(anatomicalModel); - }); - this.__modelsCache[vipSubset] = anatomicalModels; - return anatomicalModels; - }); - }, - - getVipModels: async function(vipSubset) { - const vipModels = this.self().VIP_MODELS; - if (vipSubset && vipSubset in vipModels) { - return await this.__fetchVipModels(vipSubset); - } - const promises = []; - Object.keys(vipModels).forEach(sbs => promises.push(this.__fetchVipModels(sbs))); - return await Promise.all(promises) - .then(values => values.flat()); - }, } }); diff --git a/services/static-webserver/client/source/class/osparc/data/PollTasks.js b/services/static-webserver/client/source/class/osparc/store/PollTasks.js similarity index 61% rename from services/static-webserver/client/source/class/osparc/data/PollTasks.js rename to services/static-webserver/client/source/class/osparc/store/PollTasks.js index 33c06d0e3ba..625b1838451 100644 --- a/services/static-webserver/client/source/class/osparc/data/PollTasks.js +++ b/services/static-webserver/client/source/class/osparc/store/PollTasks.js @@ -15,14 +15,10 @@ ************************************************************************ */ -qx.Class.define("osparc.data.PollTasks", { +qx.Class.define("osparc.store.PollTasks", { extend: qx.core.Object, type: "singleton", - construct: function() { - this.initTasks(); - }, - properties: { tasks: { check: "Array", @@ -33,15 +29,15 @@ qx.Class.define("osparc.data.PollTasks", { }, members: { - addTask: function(taskData, interval) { - const tasks = this.getTasks(); - const index = tasks.findIndex(t => t.getTaskId() === taskData["task_id"]); - if (index === -1) { - const task = new osparc.data.PollTask(taskData, interval); - tasks.push(task); - return task; - } - return null; + fetchTasks: function() { + return osparc.data.Resources.get("tasks") + .then(tasksData => { + tasksData.forEach(taskData => { + const interval = 1000; + this.__addTask(taskData, interval); + }); + }) + .catch(err => console.error(err)); }, createPollingTask: function(fetchPromise, interval) { @@ -49,7 +45,7 @@ qx.Class.define("osparc.data.PollTasks", { fetchPromise .then(taskData => { if ("status_href" in taskData) { - const task = this.addTask(taskData, interval); + const task = this.__addTask(taskData, interval); resolve(task); } else { throw Error("Status missing"); @@ -59,9 +55,28 @@ qx.Class.define("osparc.data.PollTasks", { }); }, + __addTask: function(taskData, interval = 1000) { + const tasks = this.getTasks(); + const index = tasks.findIndex(t => t.getTaskId() === taskData["task_id"]); + if (index === -1) { + const task = new osparc.data.PollTask(taskData, interval); + tasks.push(task); + return task; + } + return null; + }, + + getDuplicateStudyTasks: function() { + return this.getTasks().filter(task => task.getTaskId().includes("from_study") && !task.getTaskId().includes("as_template")); + }, + + getPublishTemplateTasks: function() { + return this.getTasks().filter(task => task.getTaskId().includes("from_study") && task.getTaskId().includes("as_template")); + }, + removeTasks: function() { const tasks = this.getTasks(); tasks.forEach(task => task.dispose()); - } + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/store/Pricing.js b/services/static-webserver/client/source/class/osparc/store/Pricing.js index b43ec07f806..46f7629d8a8 100644 --- a/services/static-webserver/client/source/class/osparc/store/Pricing.js +++ b/services/static-webserver/client/source/class/osparc/store/Pricing.js @@ -82,6 +82,9 @@ qx.Class.define("osparc.store.Pricing", { }, fetchPricingUnits: function(pricingPlanId) { + if (this.getPricingPlan(pricingPlanId) && this.getPricingPlan(pricingPlanId).getPricingUnits().length !== 0) { + return new Promise(resolve => resolve(this.getPricingPlan(pricingPlanId).getPricingUnits())); + } const params = { url: { pricingPlanId, @@ -100,6 +103,38 @@ qx.Class.define("osparc.store.Pricing", { }); }, + createPricingUnit: function(pricingPlanId, pricingUnitData) { + const params = { + url: { + "pricingPlanId": pricingPlanId + }, + data: pricingUnitData + }; + return osparc.data.Resources.fetch("pricingUnits", "post", params) + .then(newPricingUnitData => { + const pricingPlan = this.getPricingPlan(pricingPlanId); + this.__addPricingUnitToCache(pricingPlan, newPricingUnitData); + return pricingPlan; + }) + }, + + updatePricingUnit: function(pricingPlanId, pricingUnitId, pricingUnitData) { + const params = { + url: { + "pricingPlanId": pricingPlanId, + "pricingUnitId": pricingUnitId, + }, + data: pricingUnitData + }; + return osparc.data.Resources.fetch("pricingUnits", "post", params) + .then(() => { + const pricingPlan = this.getPricingPlan(pricingPlanId); + // OM do not add but replace + this.__addPricingUnitToCache(pricingPlan, pricingUnitData); + return pricingPlan; + }) + }, + getPricingPlans: function() { return this.pricingPlansCached; }, diff --git a/services/static-webserver/client/source/class/osparc/store/Products.js b/services/static-webserver/client/source/class/osparc/store/Products.js index 4728bded609..38ef8ad2e6f 100644 --- a/services/static-webserver/client/source/class/osparc/store/Products.js +++ b/services/static-webserver/client/source/class/osparc/store/Products.js @@ -15,34 +15,82 @@ ************************************************************************ */ +/** + * @asset(osparc/ui_config.json") + * @asset(schemas/product-ui.json) + * @asset(object-path/object-path-0-11-4.min.js) + * @asset(ajv/ajv-6-11-0.min.js) + * @ignore(Ajv) + */ + qx.Class.define("osparc.store.Products", { extend: qx.core.Object, type: "singleton", members: { - __newStudyConfig: null, - - fetchNewStudyConfig: function() { - return osparc.utils.Utils.fetchJSON("/resource/osparc/new_studies.json") - .then(newStudiesData => { - const product = osparc.product.Utils.getProductName() - if (product in newStudiesData) { - this.__newStudyConfig = newStudiesData[product]; - return this.__newStudyConfig; - } - return {}; - }) - .catch(console.error); - }, + __uiConfig: null, - getNewStudyConfig: function() { + fetchUiConfig: function() { return new Promise(resolve => { - if (this.__newStudyConfig) { - resolve(this.__newStudyConfig); - } else { - resolve(this.fetchNewStudyConfig()) + if (osparc.auth.Data.getInstance().isGuest()) { + this.__uiConfig = {}; + resolve(this.__uiConfig); } + + Promise.all([ + osparc.data.Resources.fetch("productMetadata", "getUiConfig"), + osparc.utils.Utils.fetchJSON("/resource/osparc/ui_config.json"), + osparc.utils.Utils.fetchJSON("/resource/schemas/product-ui.json"), + ]) + .then(values => { + let uiConfig = {}; + const beUiConfig = values[0]; + const feUiConfig = values[1]; + const schema = values[2]; + if (beUiConfig && beUiConfig["ui"] && Object.keys(beUiConfig["ui"]).length) { + uiConfig = beUiConfig["ui"]; + } else { + const product = osparc.product.Utils.getProductName(); + if (feUiConfig && product in feUiConfig) { + uiConfig = feUiConfig[product]; + } + } + const ajvLoader = new qx.util.DynamicScriptLoader([ + "/resource/ajv/ajv-6-11-0.min.js", + "/resource/object-path/object-path-0-11-4.min.js" + ]); + ajvLoader.addListener("ready", () => { + const ajv = new Ajv({ + allErrors: true, + strictDefaults: true, + useDefaults: true, + strictTypes: true, + }); + const validate = ajv.compile(schema); + const valid = validate(uiConfig); + if (valid) { + this.__uiConfig = uiConfig; + resolve(this.__uiConfig); + } else { + osparc.FlashMessenger.logError("Wrong product.ui config"); + validate.errors.forEach(err => { + console.error(`Error at ${err.dataPath}: ${err.message}`); + }); + } + }); + ajvLoader.addListener("failed", console.error, this); + ajvLoader.start(); + }) + .catch(console.error); }); }, + + getPlusButtonUiConfig: function() { + return this.__uiConfig["plusButton"]; + }, + + getNewStudiesUiConfig: function() { + return this.__uiConfig["newStudies"]; + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/store/Services.js b/services/static-webserver/client/source/class/osparc/store/Services.js index c2abeed32ec..a01d4cf91a0 100644 --- a/services/static-webserver/client/source/class/osparc/store/Services.js +++ b/services/static-webserver/client/source/class/osparc/store/Services.js @@ -19,11 +19,11 @@ qx.Class.define("osparc.store.Services", { type: "static", statics: { - servicesCached: {}, + __servicesCached: {}, getServicesLatest: function(useCache = true) { return new Promise(resolve => { - if (useCache && Object.keys(this.servicesCached)) { + if (useCache && Object.keys(this.__servicesCached)) { // return latest only const latest = this.__getLatestCached(); resolve(latest); @@ -37,21 +37,123 @@ qx.Class.define("osparc.store.Services", { this.__addTSRInfos(servicesObj); this.__addExtraTypeInfos(servicesObj); - // use response to populate servicesCached Object.values(servicesObj).forEach(serviceKey => { - Object.values(serviceKey).forEach(srv => this.__addToCache(srv)); + Object.values(serviceKey).forEach(service => this.__addToCache(service)); }); resolve(servicesObj); }) + .catch(err => osparc.FlashMessenger.logError(err, qx.locale.Manager.tr("Unable to fetch Services"))); + }); + }, + + getLatest: function(key) { + const services = this.__servicesCached; + if (key in services) { + const latestMetadata = Object.values(services[key])[0]; + if (!osparc.service.Utils.isRetired(latestMetadata)) { + return latestMetadata; + } + } + return null; + }, + + getLatestCompatible: function(key, version) { + const services = this.__servicesCached; + if (key in services && version in services[key]) { + const historyEntry = osparc.service.Utils.extractVersionFromHistory(services[key][version]); + if (historyEntry["compatibility"] && historyEntry["compatibility"]["canUpdateTo"]) { + const canUpdateTo = historyEntry["compatibility"]["canUpdateTo"]; + return { + key: "key" in canUpdateTo ? canUpdateTo["key"] : key, // key is optional + version: canUpdateTo["version"] + }; + } + // the provided key/version itself is the latest compatible + return { + key, + version + }; + } + return null; + }, + + getVersionDisplay: function(key, version) { + const services = this.__servicesCached; + if (key in services && version in services[key]) { + return osparc.service.Utils.extractVersionDisplay(services[key][version]); + } + return null; + }, + + getReleasedDate: function(key, version) { + const services = this.__servicesCached; + if ( + key in services && + version in services[key] && + "released" in services[key][version] + ) { + return services[key][version]["released"]; + } + return null; + }, + + getService: function(key, version, useCache = true) { + return new Promise((resolve, reject) => { + if ( + useCache && + this.__isInCache(key, version) && + "history" in this.__servicesCached[key][version] + ) { + resolve(this.__servicesCached[key][version]); + return; + } + + const params = { + url: osparc.data.Resources.getServiceUrl(key, version) + }; + osparc.data.Resources.fetch("servicesV2", "getOne", params) + .then(service => { + this.__addHit(service); + this.__addTSRInfo(service); + this.__addExtraTypeInfo(service); + this.__addToCache(service) + resolve(service); + }) .catch(err => { - const msg = err.message || qx.locale.Manager.tr("Unable to fetch Services"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); console.error(err); + reject(); }); }); }, + __getAllVersions: function(key) { + const services = this.__servicesCached; + let versions = []; + if (key in services) { + const serviceVersions = services[key]; + versions = versions.concat(Object.keys(serviceVersions)); + versions.sort(osparc.utils.Utils.compareVersionNumbers); + } + return versions.reverse(); + }, + + populateVersionsSelectBox: function(key, selectBox) { + const versions = this.__getAllVersions(key); + return this.getService(key, versions[0]) + .then(latestMetadata => { + latestMetadata["history"].forEach(historyEntry => { + if (!historyEntry["retired"]) { + const versionDisplay = osparc.service.Utils.extractVersionDisplay(historyEntry); + const listItem = new qx.ui.form.ListItem(versionDisplay); + osparc.utils.Utils.setIdToWidget(listItem, "serviceVersionItem_" + versionDisplay); + listItem.version = historyEntry["version"]; + selectBox.add(listItem); + } + }); + }); + }, + getServicesLatestList: function(excludeFrontend = true, excludeDeprecated = true) { return new Promise(resolve => { const servicesList = []; @@ -65,19 +167,25 @@ qx.Class.define("osparc.store.Services", { // do not add frontend services continue; } - if (excludeDeprecated && serviceLatest["retired"]) { - // first check if a previous version of this service isn't retired - let versions = Object.keys(this.servicesCached[key]); - versions = versions.sort(osparc.utils.Utils.compareVersionNumbers).reverse(); - for (let j=0; j
"); + Object.values(workbench).forEach(node => { + const inaccessibleService = inaccessibleServices.find(srv => srv.key === node.key && srv.version === node.version); + if (inaccessibleService) { + const n = inaccessibleService.key.lastIndexOf("/"); + const friendlyKey = inaccessibleService.key.substring(n + 1); + msg += `- ${node.label} (${friendlyKey}:${inaccessibleService.version})
`; + } + }); + return msg; + }, + + getFilePicker: function() { + return this.getLatest("simcore/services/frontend/file-picker"); + }, + + getParametersMetadata: function() { + const parametersMetadata = []; + const services = this.__servicesCached; + for (const key in services) { + if (key.includes("simcore/services/frontend/parameter/")) { + const latest = this.getLatest(key); + if (latest) { + parametersMetadata.push(latest); + } + } + } + return parametersMetadata; + }, + + getParameterMetadata: function(type) { + return this.getLatest("simcore/services/frontend/parameter/"+type); + }, + + getProbeMetadata: function(type) { + return this.getLatest("simcore/services/frontend/iterator-consumer/probe/"+type); + }, + __addToCache: function(service) { const key = service.key; const version = service.version; - if (!(key in this.servicesCached)) { - this.servicesCached[key] = {}; - } - this.servicesCached[key][version] = service; - this.servicesCached[key][version]["cached"] = true; - - if ("history" in service) { - service["history"].forEach(historyEntry => { - const hVersion = historyEntry.version; - if (!(hVersion in this.servicesCached[key])) { - this.servicesCached[key][hVersion] = {}; - this.servicesCached[key][hVersion]["cached"] = false; - } - // merge history data into current metadata - this.servicesCached[key][hVersion] = { - ...this.servicesCached[key][hVersion], - ...historyEntry - }; - }); + if (!(key in this.__servicesCached)) { + this.__servicesCached[key] = {}; } + this.__servicesCached[key][version] = service; }, __isInCache: function(key, version) { return ( - key in this.servicesCached && - version in this.servicesCached[key] && - this.servicesCached[key][version]["cached"] + key in this.__servicesCached && + version in this.__servicesCached[key] ); }, __getLatestCached: function() { const latestServices = {}; - for (const key in this.servicesCached) { - let versions = Object.keys(this.servicesCached[key]); + for (const key in this.__servicesCached) { + let versions = Object.keys(this.__servicesCached[key]); versions = versions.sort(osparc.utils.Utils.compareVersionNumbers).reverse(); - const latest = this.servicesCached[key][versions[0]]; + const latest = this.__servicesCached[key][versions[0]]; latestServices[key] = osparc.utils.Utils.deepCloneObject(latest); } return latestServices; diff --git a/services/static-webserver/client/source/class/osparc/store/StaticInfo.js b/services/static-webserver/client/source/class/osparc/store/StaticInfo.js index 201966cb7fa..341827779eb 100644 --- a/services/static-webserver/client/source/class/osparc/store/StaticInfo.js +++ b/services/static-webserver/client/source/class/osparc/store/StaticInfo.js @@ -70,9 +70,8 @@ qx.Class.define("osparc.store.StaticInfo", { }, areLicensesEnabled: function() { - const staticKey = "webserverLicenses"; - const licensesEnabled = this.getValue(staticKey); - return Boolean(licensesEnabled); + const isDisabled = osparc.utils.DisabledPlugins.isLicensesDisabled(); + return !isDisabled; }, getTrashRetentionDays: function() { diff --git a/services/static-webserver/client/source/class/osparc/store/Study.js b/services/static-webserver/client/source/class/osparc/store/Study.js new file mode 100644 index 00000000000..9f57f5281ac --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/store/Study.js @@ -0,0 +1,115 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2025 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.store.Study", { + type: "static", + + statics: { + patchStudyData: function(studyData, fieldKey, value) { + if (osparc.data.model.Study.OwnPatch.includes(fieldKey)) { + console.error(fieldKey, "has it's own PATCH path"); + return null; + } + + const patchData = {}; + patchData[fieldKey] = value; + const params = { + url: { + "studyId": studyData["uuid"] + }, + data: patchData + }; + return osparc.data.Resources.fetch("studies", "patch", params) + .then(() => { + studyData[fieldKey] = value; + // A bit hacky, but it's not sent back to the backend + studyData["lastChangeDate"] = new Date().toISOString(); + }); + }, + + patchNodeData: function(studyData, nodeId, patchData) { + const params = { + url: { + "studyId": studyData["uuid"], + "nodeId": nodeId + }, + data: patchData + }; + return osparc.data.Resources.fetch("studies", "patchNode", params) + .then(() => { + Object.keys(patchData).forEach(key => { + studyData["workbench"][nodeId][key] = patchData[key]; + }); + // A bit hacky, but it's not sent back to the backend + studyData["lastChangeDate"] = new Date().toISOString(); + }); + }, + + addCollaborators: function(studyData, newCollaborators) { + const promises = []; + Object.keys(newCollaborators).forEach(gid => { + const params = { + url: { + "studyId": studyData["uuid"], + "gId": gid + }, + data: newCollaborators[gid] + }; + promises.push(osparc.data.Resources.fetch("studies", "postAccessRights", params)); + }); + return Promise.all(promises) + .then(() => { + Object.keys(newCollaborators).forEach(gid => { + studyData["accessRights"][gid] = newCollaborators[gid]; + }); + studyData["lastChangeDate"] = new Date().toISOString(); + }) + .catch(err => osparc.FlashMessenger.logError(err)); + }, + + removeCollaborator: function(studyData, gid) { + const params = { + url: { + "studyId": studyData["uuid"], + "gId": gid + } + }; + return osparc.data.Resources.fetch("studies", "deleteAccessRights", params) + .then(() => { + delete studyData["accessRights"][gid]; + studyData["lastChangeDate"] = new Date().toISOString(); + }) + .catch(err => osparc.FlashMessenger.logError(err)); + }, + + updateCollaborator: function(studyData, gid, newPermissions) { + const params = { + url: { + "studyId": studyData["uuid"], + "gId": gid + }, + data: newPermissions + }; + return osparc.data.Resources.fetch("studies", "putAccessRights", params) + .then(() => { + studyData["accessRights"][gid] = newPermissions; + studyData["lastChangeDate"] = new Date().toISOString(); + }) + .catch(err => osparc.FlashMessenger.logError(err)); + }, + } +}); diff --git a/services/static-webserver/client/source/class/osparc/store/Support.js b/services/static-webserver/client/source/class/osparc/store/Support.js index 65e5174929d..281127d06ff 100644 --- a/services/static-webserver/client/source/class/osparc/store/Support.js +++ b/services/static-webserver/client/source/class/osparc/store/Support.js @@ -132,6 +132,14 @@ qx.Class.define("osparc.store.Support", { }); }, + addReleaseNotesToMenu: function(menu) { + const releaseTag = osparc.utils.Utils.getReleaseTag(); + const releaseLink = osparc.utils.Utils.getReleaseLink(); + const releaseBtn = new qx.ui.menu.Button(qx.locale.Manager.tr("Release Notes") + " " + releaseTag, "@FontAwesome5Solid/book/14"); + releaseBtn.addListener("execute", () => window.open(releaseLink), this); + menu.add(releaseBtn); + }, + mailToLink: function(email, subject, centered = true) { const color = qx.theme.manager.Color.getInstance().resolve("text"); let textLink = `${email}`; @@ -181,6 +189,6 @@ qx.Class.define("osparc.store.Support", { } createAccountWindow.center(); createAccountWindow.open(); - } + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/store/Tags.js b/services/static-webserver/client/source/class/osparc/store/Tags.js index c2830262cd9..d672722f516 100644 --- a/services/static-webserver/client/source/class/osparc/store/Tags.js +++ b/services/static-webserver/client/source/class/osparc/store/Tags.js @@ -45,7 +45,6 @@ qx.Class.define("osparc.store.Tags", { tagsData.forEach(tagData => { const tag = this.__addToCache(tagData); tags.push(tag); - this.fetchAccessRights(tag); }); return tags; }); @@ -106,7 +105,11 @@ qx.Class.define("osparc.store.Tags", { .catch(console.error); }, - fetchAccessRights: function(tag) { + fetchAccessRights: function(tag, reload = false) { + if (reload === false && tag.getAccessRights() !== null) { + return new Promise(resolve => resolve()); + } + const params = { url: { "tagId": tag.getTagId() diff --git a/services/static-webserver/client/source/class/osparc/study/BillingSettings.js b/services/static-webserver/client/source/class/osparc/study/BillingSettings.js index 623fc9ae0f3..85a4b83cdcb 100644 --- a/services/static-webserver/client/source/class/osparc/study/BillingSettings.js +++ b/services/static-webserver/client/source/class/osparc/study/BillingSettings.js @@ -192,7 +192,7 @@ qx.Class.define("osparc.study.BillingSettings", { if (myWallets.find(wllt => wllt === wallet)) { // It's my wallet this._createChildControlImpl("debt-explanation").set({ - value: this.tr("Top up the Credit Account:
Purchase additional credits to bring the Credit Account balance back to a positive value.") + value: this.tr("Top up the Credit Account:
Purchase additional credits to restore a positive balance.") }); const buyCreditsButton = this._createChildControlImpl("buy-credits-button"); buyCreditsButton.addListener("execute", () => this.__openBuyCreditsWindow(), this); @@ -258,10 +258,7 @@ qx.Class.define("osparc.study.BillingSettings", { // also switch the study's wallet to this one this.__switchWallet(wallet.getWalletId()); }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err)); }, __debtPayed: function() { @@ -287,12 +284,9 @@ qx.Class.define("osparc.study.BillingSettings", { .then(() => { this.__studyWalletId = walletId; const msg = this.tr("Credit Account saved"); - osparc.FlashMessenger.getInstance().logAs(msg, "INFO"); - }) - .catch(err => { - console.error(err); - osparc.FlashMessenger.logAs(err.message, "ERROR"); + osparc.FlashMessenger.logAs(msg, "INFO"); }) + .catch(err => osparc.FlashMessenger.logError(err)) .finally(() => { creditAccountBox.setEnabled(true); }); diff --git a/services/static-webserver/client/source/class/osparc/study/Import.js b/services/static-webserver/client/source/class/osparc/study/Import.js index 87b2e58bbfb..fd63aaaef6c 100644 --- a/services/static-webserver/client/source/class/osparc/study/Import.js +++ b/services/static-webserver/client/source/class/osparc/study/Import.js @@ -49,7 +49,7 @@ qx.Class.define("osparc.study.Import", { const size = file.size; const maxSize = 10 * 1000 * 1000 * 1000; // 10 GB if (size > maxSize) { - osparc.FlashMessenger.logAs(`The file is too big. Maximum size is ${maxSize}MB. Please provide with a smaller file or a repository URL.`, "ERROR"); + osparc.FlashMessenger.logError(`The file is too big. Maximum size is ${maxSize}MB. Please provide with a smaller file or a repository URL.`); return; } this.fireDataEvent("fileReady", file); diff --git a/services/static-webserver/client/source/class/osparc/study/NodePricingUnits.js b/services/static-webserver/client/source/class/osparc/study/NodePricingUnits.js index f6bc409fb39..dae4d7370c4 100644 --- a/services/static-webserver/client/source/class/osparc/study/NodePricingUnits.js +++ b/services/static-webserver/client/source/class/osparc/study/NodePricingUnits.js @@ -100,8 +100,7 @@ qx.Class.define("osparc.study.NodePricingUnits", { const studyId = this.getStudyId(); const nodeId = this.getNodeId(); - const pricingStore = osparc.store.Pricing.getInstance(); - pricingStore.fetchPricingPlansService(nodeKey, nodeVersion) + osparc.store.Pricing.getInstance().fetchPricingPlansService(nodeKey, nodeVersion) .then(pricingPlanData => { if (pricingPlanData) { const unitParams = { @@ -132,10 +131,7 @@ qx.Class.define("osparc.study.NodePricingUnits", { const pricingPlanId = this.getPricingPlanId(); this.self().patchPricingUnitSelection(studyId, nodeId, pricingPlanId, selectedPricingUnitId) .then(() => pricingUnitTiers.setSelectedUnitId(selectedPricingUnitId)) - .catch(err => { - const msg = err.message || this.tr("Cannot change Tier"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - }) + .catch(err => osparc.FlashMessenger.logError(err, this.tr("Cannot change Tier"))) .finally(() => pricingUnitTiers.setEnabled(true)); } }); diff --git a/services/static-webserver/client/source/class/osparc/study/PricingUnitLicense.js b/services/static-webserver/client/source/class/osparc/study/PricingUnitLicense.js index 47335fef924..0563af22af4 100644 --- a/services/static-webserver/client/source/class/osparc/study/PricingUnitLicense.js +++ b/services/static-webserver/client/source/class/osparc/study/PricingUnitLicense.js @@ -74,7 +74,7 @@ qx.Class.define("osparc.study.PricingUnitLicense", { // add price info const price = this.getChildControl("price"); pricingUnit.bind("cost", price, "value", { - converter: v => this.tr("Credits") + ": " + v + converter: v => this.tr("Credits") + ": " + osparc.utils.Utils.addWhiteSpaces(v) }); // add edit button @@ -85,12 +85,17 @@ qx.Class.define("osparc.study.PricingUnitLicense", { }, __rentUnit: function() { + const nSeats = parseInt(this.getUnitData().getExtraInfo()["num_of_seats"]); + const nCredits = this.getUnitData().getCost(); const expirationDate = osparc.study.PricingUnitLicense.getExpirationDate(); - const msg = this.getUnitData().getName() + this.tr(" will be available until ") + osparc.utils.Utils.formatDate(expirationDate); + let msg = nSeats + " seat" + (nSeats > 1 ? "s " : " ") + this.tr("will be available until ") + osparc.utils.Utils.formatDate(expirationDate); + msg += `
The rental will cost ${nCredits} credits`; + msg += `
I hereby accept the Terms and Conditions`; const confirmationWin = new osparc.ui.window.Confirmation(msg).set({ caption: this.tr("Rent"), confirmText: this.tr("Rent"), }); + confirmationWin.open(); confirmationWin.addListener("close", () => { if (confirmationWin.getConfirmed()) { diff --git a/services/static-webserver/client/source/class/osparc/study/StudyOptions.js b/services/static-webserver/client/source/class/osparc/study/StudyOptions.js index 1ed9a4cd961..37cd905c33c 100644 --- a/services/static-webserver/client/source/class/osparc/study/StudyOptions.js +++ b/services/static-webserver/client/source/class/osparc/study/StudyOptions.js @@ -91,12 +91,8 @@ qx.Class.define("osparc.study.StudyOptions", { }, updateName: function(studyData, name) { - return osparc.info.StudyUtils.patchStudyData(studyData, "name", name) - .catch(err => { - console.error(err); - const msg = err.message || qx.locale.Manager.tr("Something went wrong Renaming"); - osparc.FlashMessenger.logAs(msg, "ERROR"); - }); + return osparc.store.Study.patchStudyData(studyData, "name", name) + .catch(err => osparc.FlashMessenger.logError(err, qx.locale.Manager.tr("Something went wrong while renaming"))); }, updateWallet: function(studyId, walletId) { @@ -107,11 +103,7 @@ qx.Class.define("osparc.study.StudyOptions", { } }; return osparc.data.Resources.fetch("studies", "selectWallet", params) - .catch(err => { - console.error(err); - const msg = err.message || qx.locale.Manager.tr("Error selecting Credit Account"); - osparc.FlashMessenger.getInstance().logAs(msg, "ERROR"); - }); + .catch(err => osparc.FlashMessenger.logError(err, qx.locale.Manager.tr("An issue occurred while selecting Credit Account"))); }, }, @@ -245,7 +237,7 @@ qx.Class.define("osparc.study.StudyOptions", { } }; Promise.all([ - osparc.data.Resources.getOne("studies", params), + osparc.data.Resources.fetch("studies", "getOne", params), osparc.data.Resources.fetch("studies", "getWallet", params) ]) .then(values => { diff --git a/services/static-webserver/client/source/class/osparc/study/StudyPreview.js b/services/static-webserver/client/source/class/osparc/study/StudyPreview.js index 62d5d01470f..c1d6af633be 100644 --- a/services/static-webserver/client/source/class/osparc/study/StudyPreview.js +++ b/services/static-webserver/client/source/class/osparc/study/StudyPreview.js @@ -37,7 +37,7 @@ qx.Class.define("osparc.study.StudyPreview", { __buildPreview: function() { const study = this.__study; const uiMode = study.getUi().getMode(); - if (uiMode !== "app" && !study.isPipelineEmpty()) { + if (uiMode === "workbench" && !study.isPipelineEmpty()) { const workbenchUIPreview = new osparc.workbench.WorkbenchUIPreview(); workbenchUIPreview.setStudy(study); workbenchUIPreview.loadModel(study.getWorkbench()); diff --git a/services/static-webserver/client/source/class/osparc/study/StudyPricingUnits.js b/services/static-webserver/client/source/class/osparc/study/StudyPricingUnits.js index e3e8514fbaf..ccc9a673ccb 100644 --- a/services/static-webserver/client/source/class/osparc/study/StudyPricingUnits.js +++ b/services/static-webserver/client/source/class/osparc/study/StudyPricingUnits.js @@ -38,8 +38,8 @@ qx.Class.define("osparc.study.StudyPricingUnits", { }, statics: { - includeInList: function(node) { - return !osparc.data.model.Node.isFrontend(node); + includeInList: function(nodeData) { + return !osparc.data.model.Node.isFrontend(nodeData); }, }, @@ -61,9 +61,9 @@ qx.Class.define("osparc.study.StudyPricingUnits", { if ("workbench" in this.__studyData) { const workbench = this.__studyData["workbench"]; Object.keys(workbench).forEach(nodeId => { - const node = workbench[nodeId]; - if (this.self().includeInList(node)) { - const nodePricingUnits = new osparc.study.NodePricingUnits(this.__studyData["uuid"], nodeId, node); + const nodeData = workbench[nodeId]; + if (this.self().includeInList(nodeData)) { + const nodePricingUnits = new osparc.study.NodePricingUnits(this.__studyData["uuid"], nodeId, nodeData); this.__nodePricingUnits.push(nodePricingUnits); this._add(nodePricingUnits); promises.push(nodePricingUnits.showPricingUnits()); diff --git a/services/static-webserver/client/source/class/osparc/study/Utils.js b/services/static-webserver/client/source/class/osparc/study/Utils.js index 445b2f82c31..f548d08b85a 100644 --- a/services/static-webserver/client/source/class/osparc/study/Utils.js +++ b/services/static-webserver/client/source/class/osparc/study/Utils.js @@ -23,88 +23,79 @@ qx.Class.define("osparc.study.Utils", { type: "static", statics: { - extractServices: function(workbench) { - const services = []; + __isAnyLinkedNodeMissing: function(studyData) { + const existingNodeIds = Object.keys(studyData["workbench"]); + const linkedNodeIds = osparc.data.model.Workbench.getLinkedNodeIds(studyData["workbench"]); + const allExist = linkedNodeIds.every(linkedNodeId => existingNodeIds.includes(linkedNodeId)); + return !allExist; + }, + + isCorrupt: function(studyData) { + return this.__isAnyLinkedNodeMissing(studyData); + }, + + extractUniqueServices: function(workbench) { + const services = new Set([]); Object.values(workbench).forEach(srv => { - services.push({ + services.add({ key: srv.key, version: srv.version }); }); - return services; + return Array.from(services); }, - getInaccessibleServices: function(workbench) { - const allServices = osparc.store.Services.servicesCached; - const unaccessibleServices = []; - const wbServices = new Set(this.extractServices(workbench)); - wbServices.forEach(srv => { - if (srv.key in allServices && srv.version in allServices[srv.key]) { - return; - } - const idx = unaccessibleServices.findIndex(unSrv => unSrv.key === srv.key && unSrv.version === srv.version); - if (idx === -1) { - unaccessibleServices.push(srv); - } - }); - return unaccessibleServices; + getCantExecuteServices: function(studyServices = []) { + return studyServices.filter(studyService => studyService["myAccessRights"]["execute"] === false); }, - getInaccessibleServicesMsg: function(inaccessibleServices, workbench) { - let msg = qx.locale.Manager.tr("Service(s) not accessible:
"); - Object.values(workbench).forEach(node => { - const inaccessibleService = inaccessibleServices.find(srv => srv.key === node.key && srv.version === node.version); - if (inaccessibleService) { - const n = inaccessibleService.key.lastIndexOf("/"); - const friendlyKey = inaccessibleService.key.substring(n + 1); - msg += `- ${node.label} (${friendlyKey}:${inaccessibleService.version})
`; + anyServiceRetired: function(studyServices) { + const isRetired = studyServices.some(studyService => { + if (studyService["release"] && studyService["release"]["retired"]) { + const retirementDate = new Date(studyService["release"]["retired"]); + const currentDate = new Date(); + return retirementDate < currentDate; } + return false; }); - return msg; - }, - - isWorkbenchUpdatable: function(workbench) { - const services = new Set(this.extractServices(workbench)); - const isUpdatable = Array.from(services).some(srv => osparc.service.Utils.isUpdatable(srv)); - return isUpdatable; + return isRetired; }, - isWorkbenchRetired: function(workbench) { - const allServices = osparc.store.Services.servicesCached; - const services = new Set(this.extractServices(workbench)); - const isRetired = Array.from(services).some(srv => { - if (srv.key in allServices && srv.version in allServices[srv.key]) { - const serviceMD = allServices[srv.key][srv.version]; - if (serviceMD["retired"]) { - const retirementDate = new Date(serviceMD["retired"]); - const currentDate = new Date(); - return retirementDate < currentDate; - } - return false; + anyServiceDeprecated: function(studyServices) { + const isDeprecated = studyServices.some(studyService => { + if (studyService["release"] && studyService["release"]["retired"]) { + const retirementDate = new Date(studyService["release"]["retired"]); + const currentDate = new Date(); + return retirementDate > currentDate; } return false; }); - return isRetired; + return isDeprecated; }, - isWorkbenchDeprecated: function(workbench) { - const allServices = osparc.store.Services.servicesCached; - const services = new Set(this.extractServices(workbench)); - const isRetired = Array.from(services).some(srv => { - if (srv.key in allServices && srv.version in allServices[srv.key]) { - const serviceMD = allServices[srv.key][srv.version]; - if ("retired" in serviceMD && serviceMD["retired"]) { - const retirementDate = new Date(serviceMD["retired"]); - const currentDate = new Date(); - return retirementDate > currentDate; - } - return false; + anyServiceUpdatable: function(studyServices) { + const isUpdatable = studyServices.some(studyService => { + if (studyService["release"] && studyService["release"]["compatibility"]) { + return Boolean(studyService["release"]["compatibility"]); } return false; }); - return isRetired; + return isUpdatable; + }, + + updatableNodeIds: function(workbench, studyServices) { + const nodeIds = []; + for (const nodeId in workbench) { + const node = workbench[nodeId]; + const studyServiceFound = studyServices.find(studyService => studyService["key"] === node["key"] && studyService["release"]["version"] === node["version"]); + if (studyServiceFound && studyServiceFound["release"] && studyServiceFound["release"]["compatibility"]) { + nodeIds.push(nodeId); + } + } + return nodeIds; }, + createStudyFromService: function(key, version, existingStudies, newStudyLabel, contextProps = {}) { return new Promise((resolve, reject) => { osparc.store.Services.getService(key, version) @@ -142,9 +133,13 @@ qx.Class.define("osparc.study.Utils", { "y": 100 } }; - const inaccessibleServices = this.getInaccessibleServices(minStudyData["workbench"]) + // maybe check it's dynamic + if (!("mode" in minStudyData["ui"])) { + minStudyData["ui"]["mode"] = "standalone"; + } + const inaccessibleServices = osparc.store.Services.getInaccessibleServices(minStudyData["workbench"]) if (inaccessibleServices.length) { - const msg = this.getInaccessibleServicesMsg(inaccessibleServices, minStudyData["workbench"]); + const msg = osparc.store.Services.getInaccessibleServicesMsg(inaccessibleServices, minStudyData["workbench"]); reject({ message: msg }); @@ -157,10 +152,7 @@ qx.Class.define("osparc.study.Utils", { .then(studyData => resolve(studyData["uuid"])) .catch(err => reject(err)); }) - .catch(err => { - osparc.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - console.error(err); - }); + .catch(err => osparc.FlashMessenger.logError(err)); }); }, @@ -170,7 +162,7 @@ qx.Class.define("osparc.study.Utils", { pollTask: true }; const fetchPromise = osparc.data.Resources.fetch("studies", "postNewStudy", params, options); - const pollTasks = osparc.data.PollTasks.getInstance(); + const pollTasks = osparc.store.PollTasks.getInstance(); const interval = 1000; pollTasks.createPollingTask(fetchPromise, interval) .then(task => { @@ -189,80 +181,83 @@ qx.Class.define("osparc.study.Utils", { createStudyFromTemplate: function(templateData, loadingPage, contextProps = {}) { return new Promise((resolve, reject) => { - const inaccessibleServices = this.getInaccessibleServices(templateData["workbench"]); - if (inaccessibleServices.length) { - const msg = this.getInaccessibleServicesMsg(inaccessibleServices, templateData["workbench"]); - reject({ - message: msg + osparc.store.Services.getStudyServicesMetadata(templateData) + .finally(() => { + const inaccessibleServices = osparc.store.Services.getInaccessibleServices(templateData["workbench"]); + if (inaccessibleServices.length) { + const msg = osparc.store.Services.getInaccessibleServicesMsg(inaccessibleServices, templateData["workbench"]); + reject({ + message: msg + }); + return; + } + // context props, otherwise Study will be created in the root folder of my personal workspace + const minStudyData = Object.assign(osparc.data.model.Study.createMinStudyObject(), contextProps); + minStudyData["name"] = templateData["name"]; + minStudyData["description"] = templateData["description"]; + minStudyData["thumbnail"] = templateData["thumbnail"]; + const params = { + url: { + templateId: templateData["uuid"] + }, + data: minStudyData + }; + const options = { + pollTask: true + }; + const fetchPromise = osparc.data.Resources.fetch("studies", "postNewStudyFromTemplate", params, options); + const pollTasks = osparc.store.PollTasks.getInstance(); + const interval = 1000; + pollTasks.createPollingTask(fetchPromise, interval) + .then(task => { + const title = qx.locale.Manager.tr("CREATING ") + osparc.product.Utils.getStudyAlias({allUpperCase: true}) + " ..."; + const progressSequence = new osparc.widget.ProgressSequence(title).set({ + minHeight: 180 // four tasks + }); + progressSequence.addOverallProgressBar(); + loadingPage.clearMessages(); + loadingPage.addWidgetToMessages(progressSequence); + task.addListener("updateReceived", e => { + const updateData = e.getData(); + if ("task_progress" in updateData && loadingPage) { + const progress = updateData["task_progress"]; + const message = progress["message"]; + const percent = progress["percent"] ? parseFloat(progress["percent"].toFixed(3)) : progress["percent"]; + progressSequence.setOverallProgress(percent); + const existingTask = progressSequence.getTask(message); + if (existingTask) { + // update task + osparc.widget.ProgressSequence.updateTaskProgress(existingTask, { + value: percent, + progressLabel: parseFloat((percent*100).toFixed(2)) + "%" + }); + } else { + // new task + // all the previous steps to 100% + progressSequence.getTasks().forEach(tsk => osparc.widget.ProgressSequence.updateTaskProgress(tsk, { + value: 1, + progressLabel: "100%" + })); + // and move to the next new task + const subTask = progressSequence.addNewTask(message); + osparc.widget.ProgressSequence.updateTaskProgress(subTask, { + value: percent, + progressLabel: "0%" + }); + } + } + }, this); + task.addListener("resultReceived", e => { + const studyData = e.getData(); + resolve(studyData); + }, this); + task.addListener("pollingError", e => { + const err = e.getData(); + reject(err); + }, this); + }) + .catch(err => reject(err)); }); - return; - } - // context props, otherwise Study will be created in the root folder of my personal workspace - const minStudyData = Object.assign(osparc.data.model.Study.createMinStudyObject(), contextProps); - minStudyData["name"] = templateData["name"]; - minStudyData["description"] = templateData["description"]; - minStudyData["thumbnail"] = templateData["thumbnail"]; - const params = { - url: { - templateId: templateData["uuid"] - }, - data: minStudyData - }; - const options = { - pollTask: true - }; - const fetchPromise = osparc.data.Resources.fetch("studies", "postNewStudyFromTemplate", params, options); - const pollTasks = osparc.data.PollTasks.getInstance(); - const interval = 1000; - pollTasks.createPollingTask(fetchPromise, interval) - .then(task => { - const title = qx.locale.Manager.tr("CREATING ") + osparc.product.Utils.getStudyAlias({allUpperCase: true}) + " ..."; - const progressSequence = new osparc.widget.ProgressSequence(title).set({ - minHeight: 180 // four tasks - }); - progressSequence.addOverallProgressBar(); - loadingPage.clearMessages(); - loadingPage.addWidgetToMessages(progressSequence); - task.addListener("updateReceived", e => { - const updateData = e.getData(); - if ("task_progress" in updateData && loadingPage) { - const progress = updateData["task_progress"]; - const message = progress["message"]; - const percent = progress["percent"] ? parseFloat(progress["percent"].toFixed(3)) : progress["percent"]; - progressSequence.setOverallProgress(percent); - const existingTask = progressSequence.getTask(message); - if (existingTask) { - // update task - osparc.widget.ProgressSequence.updateTaskProgress(existingTask, { - value: percent, - progressLabel: parseFloat((percent*100).toFixed(2)) + "%" - }); - } else { - // new task - // all the previous steps to 100% - progressSequence.getTasks().forEach(tsk => osparc.widget.ProgressSequence.updateTaskProgress(tsk, { - value: 1, - progressLabel: "100%" - })); - // and move to the next new task - const subTask = progressSequence.addNewTask(message); - osparc.widget.ProgressSequence.updateTaskProgress(subTask, { - value: percent, - progressLabel: "0%" - }); - } - } - }, this); - task.addListener("resultReceived", e => { - const studyData = e.getData(); - resolve(studyData); - }, this); - task.addListener("pollingError", e => { - const err = e.getData(); - reject(err); - }, this); - }) - .catch(err => reject(err)); }); }, @@ -270,9 +265,16 @@ qx.Class.define("osparc.study.Utils", { return Boolean("debt" in studyData && studyData["debt"] < 0); }, + getUiMode: function(studyData) { + if ("ui" in studyData && "mode" in studyData["ui"]) { + return studyData["ui"]["mode"]; + } + return null; + }, + __getBlockedState: function(studyData) { - if (studyData["workbench"]) { - const unaccessibleServices = osparc.study.Utils.getInaccessibleServices(studyData["workbench"]) + if (studyData["services"]) { + const unaccessibleServices = osparc.study.Utils.getCantExecuteServices(studyData["services"]) if (unaccessibleServices.length) { return "UNKNOWN_SERVICES"; } @@ -308,7 +310,7 @@ qx.Class.define("osparc.study.Utils", { canShowStudyData: function(studyData) { const blocked = this.__getBlockedState(studyData); - return [false].includes(blocked); + return ["UNKNOWN_SERVICES", false].includes(blocked); }, canShowPreview: function(studyData) { @@ -335,5 +337,69 @@ qx.Class.define("osparc.study.Utils", { const blocked = this.__getBlockedState(studyData); return ["UNKNOWN_SERVICES", false].includes(blocked); }, + + getNonFrontendNodes: function(studyData) { + return Object.values(studyData["workbench"]).filter(nodeData => !osparc.data.model.Node.isFrontend(nodeData)); + }, + + guessIcon: async function(studyData) { + if (osparc.product.Utils.isProduct("tis") || osparc.product.Utils.isProduct("tiplite")) { + return this.__guessTIPIcon(studyData); + } + const icon = await this.__guessIcon(studyData); + return icon; + }, + + __guessIcon: function(studyData) { + const defaultIcon = osparc.dashboard.CardBase.PRODUCT_ICON; + return new Promise(resolve => { + // the was to guess the TI type is to check the boot mode of the ti-postpro in the pipeline + const wbServices = this.self().getNonFrontendNodes(studyData); + if (wbServices.length === 1) { + const wbService = wbServices[0]; + osparc.store.Services.getService(wbService.key, wbService.version) + .then(serviceMetadata => { + if (serviceMetadata["icon"]) { + resolve(serviceMetadata["icon"]); + } + resolve(defaultIcon); + }); + } else if (wbServices.length > 1) { + resolve("osparc/icons/diagram.png"); + } else { + resolve(defaultIcon); + } + }); + }, + + __guessTIPIcon: function(studyData) { + // the was to guess the TI type is to check the boot mode of the ti-postpro in the pipeline + const tiPostpro = Object.values(studyData["workbench"]).find(srv => srv.key.includes("ti-postpro")); + if (tiPostpro && tiPostpro["bootOptions"]) { + switch (tiPostpro["bootOptions"]["boot_mode"]) { + case "0": + // classic TI + return "osparc/icons/TI.png"; + case "1": + // multichannel + return "osparc/icons/MC.png"; + case "2": + // phase-modulation + return "osparc/icons/PM.png"; + case "3": + // personalized TI + return "osparc/icons/pTI.png"; + case "4": + // personalized multichannel + return "osparc/icons/pMC.png"; + case "5": + // personalized phase-modulation + return "osparc/icons/pPM.png"; + default: + return "osparc/icons/TI.png"; + } + } + return "osparc/icons/TI.png"; + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/task/TaskUI.js b/services/static-webserver/client/source/class/osparc/task/TaskUI.js index 513bf34ec91..3e79c75137b 100644 --- a/services/static-webserver/client/source/class/osparc/task/TaskUI.js +++ b/services/static-webserver/client/source/class/osparc/task/TaskUI.js @@ -133,16 +133,6 @@ qx.Class.define("osparc.task.TaskUI", { }, this); }, - start: function() { - const tasks = osparc.task.Tasks.getInstance(); - tasks.addTask(this); - }, - - stop: function() { - const tasks = osparc.task.Tasks.getInstance(); - tasks.removeTask(this); - }, - setIcon: function(source) { this.getChildControl("icon").getContentElement().removeClass("rotate"); this.getChildControl("icon").setSource(source); diff --git a/services/static-webserver/client/source/class/osparc/task/TasksButton.js b/services/static-webserver/client/source/class/osparc/task/TasksButton.js index cbf113d4eb3..0d6f85e24d4 100644 --- a/services/static-webserver/client/source/class/osparc/task/TasksButton.js +++ b/services/static-webserver/client/source/class/osparc/task/TasksButton.js @@ -27,10 +27,11 @@ qx.Class.define("osparc.task.TasksButton", { width: 30, alignX: "center", cursor: "pointer", - visibility: "excluded" + visibility: "excluded", + toolTipText: this.tr("Tasks"), }); - const tasks = osparc.task.Tasks.getInstance(); + const tasks = osparc.task.TasksContainer.getInstance(); tasks.getTasks().addListener("change", e => this.__updateTasksButton(), this); this.addListener("tap", () => this.__showTasks(), this); }, @@ -40,8 +41,8 @@ qx.Class.define("osparc.task.TasksButton", { let control; switch (id) { case "icon": { - control = new qx.ui.basic.Image("@FontAwesome5Solid/cog/24"); - osparc.utils.Utils.addClass(control.getContentElement(), "rotate"); + control = new qx.ui.basic.Image("@FontAwesome5Solid/cog/22"); + osparc.utils.Utils.addClass(control.getContentElement(), "rotateSlow"); const logoContainer = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ alignY: "middle" @@ -74,7 +75,7 @@ qx.Class.define("osparc.task.TasksButton", { this._createChildControlImpl("icon"); const number = this.getChildControl("number"); - const tasks = osparc.task.Tasks.getInstance(); + const tasks = osparc.task.TasksContainer.getInstance(); const nTasks = tasks.getTasks().length; number.setValue(nTasks.toString()); nTasks ? this.show() : this.exclude(); @@ -83,7 +84,7 @@ qx.Class.define("osparc.task.TasksButton", { __showTasks: function() { const that = this; const tapListener = event => { - const tasks = osparc.task.Tasks.getInstance(); + const tasks = osparc.task.TasksContainer.getInstance(); const tasksContainer = tasks.getTasksContainer(); if (osparc.utils.Utils.isMouseOnElement(tasksContainer, event)) { return; @@ -103,14 +104,14 @@ qx.Class.define("osparc.task.TasksButton", { bounds.top = parseInt(rect.y); } } - const tasks = osparc.task.Tasks.getInstance(); + const tasks = osparc.task.TasksContainer.getInstance(); tasks.setTasksContainerPosition(bounds.left+bounds.width, osparc.navigation.NavigationBar.HEIGHT+3); tasks.getTasksContainer().show(); document.addEventListener("mousedown", tapListener); }, __hideTasks: function() { - const tasks = osparc.task.Tasks.getInstance(); + const tasks = osparc.task.TasksContainer.getInstance(); tasks.getTasksContainer().exclude(); } } diff --git a/services/static-webserver/client/source/class/osparc/task/Tasks.js b/services/static-webserver/client/source/class/osparc/task/TasksContainer.js similarity index 72% rename from services/static-webserver/client/source/class/osparc/task/Tasks.js rename to services/static-webserver/client/source/class/osparc/task/TasksContainer.js index 8494db9754b..6e22ee89c96 100644 --- a/services/static-webserver/client/source/class/osparc/task/Tasks.js +++ b/services/static-webserver/client/source/class/osparc/task/TasksContainer.js @@ -15,7 +15,7 @@ ************************************************************************ */ -qx.Class.define("osparc.task.Tasks", { +qx.Class.define("osparc.task.TasksContainer", { extend: qx.core.Object, type: "singleton", @@ -39,17 +39,21 @@ qx.Class.define("osparc.task.Tasks", { __tasks: null, __tasksContainer: null, - addTask: function(task) { - this.__tasks.push(task); - this.__tasksContainer.addAt(task, 0); + addTaskUI: function(taskUI) { + const alreadyExists = this.__tasks.filter(task => task.getTask().getTaskId() === taskUI.getTask().getTaskId()).length; + if (alreadyExists) { + return; + } + this.__tasks.push(taskUI); + this.__tasksContainer.addAt(taskUI, 0); }, - removeTask: function(task) { - if (this.__tasks.indexOf(task) > -1) { - this.__tasks.remove(task); + removeTaskUI: function(taskUI) { + if (this.__tasks.indexOf(taskUI) > -1) { + this.__tasks.remove(taskUI); } - if (this.__tasksContainer.indexOf(task) > -1) { - this.__tasksContainer.remove(task); + if (this.__tasksContainer.indexOf(taskUI) > -1) { + this.__tasksContainer.remove(taskUI); } }, diff --git a/services/static-webserver/client/source/class/osparc/tester/TesterCenter.js b/services/static-webserver/client/source/class/osparc/tester/TesterCenter.js index 023fafb4dc5..547bd4ae3d2 100644 --- a/services/static-webserver/client/source/class/osparc/tester/TesterCenter.js +++ b/services/static-webserver/client/source/class/osparc/tester/TesterCenter.js @@ -24,7 +24,7 @@ qx.Class.define("osparc.tester.TesterCenter", { const miniProfile = osparc.desktop.account.MyAccount.createMiniProfileView().set({ paddingRight: 10 }); - this.addWidgetOnTopOfTheTabs(miniProfile); + this.addWidgetToTabs(miniProfile); this.__addSocketMessagesPage(); this.__addConsoleErrorsPage(); diff --git a/services/static-webserver/client/source/class/osparc/theme/Appearance.js b/services/static-webserver/client/source/class/osparc/theme/Appearance.js index 8af7db982cb..a1bc3cd837b 100644 --- a/services/static-webserver/client/source/class/osparc/theme/Appearance.js +++ b/services/static-webserver/client/source/class/osparc/theme/Appearance.js @@ -22,7 +22,7 @@ qx.Theme.define("osparc.theme.Appearance", { "strong-ui": { style: () => { return { - textColor: "default-button-text", + textColor: "white", backgroundColor: "product-color", }; } @@ -719,6 +719,17 @@ qx.Theme.define("osparc.theme.Appearance", { } }, + "menu-wider": { + include: "menu", + style: () => { + return { + decorator: "border-simple", + font: "text-14", + padding: 4, + }; + } + }, + "menu-button": { alias: "atom", @@ -727,7 +738,7 @@ qx.Theme.define("osparc.theme.Appearance", { decorator: "rounded", cursor: states.disabled ? "not-allowed" : "pointer", backgroundColor: states.selected || states.hovered ? "pb-new" : undefined, - textColor: states.selected ? "default-button-text" : "text", + textColor: "text", padding: [4, 8] } } @@ -971,7 +982,7 @@ qx.Theme.define("osparc.theme.Appearance", { style: state => ({ decorator: state.hovered || state.focused ? "form-button-warning-hover" : "form-button-warning", backgroundColor: state.hovered || state.focused ? "default-button-hover-background" : "warning", - textColor: state.hovered || state.focused ? "default-button-text" : "black", + textColor: "black", }) }, @@ -980,7 +991,7 @@ qx.Theme.define("osparc.theme.Appearance", { style: state => ({ decorator: state.hovered || state.focused ? "form-button-danger-hover" : "form-button-danger", backgroundColor: state.hovered || state.focused ? "default-button-hover-background" : "error", - textColor: state.hovered || state.focused ? "default-button-text" : "default-button-text" // dark theme's text color + textColor: "black", }) }, @@ -997,7 +1008,7 @@ qx.Theme.define("osparc.theme.Appearance", { decorator: "tab-button", cursor: "pointer", padding: 5, - textColor: "default-button-text", + textColor: "white", backgroundColor: "default-button-background" }; if (states.hovered) { diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js index 64a3784c276..47b42050a36 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js @@ -122,28 +122,6 @@ qx.Theme.define("osparc.theme.ColorDark", { "tooltip": "flash_message_bg", "tooltip-text": "text", - // table - "table-header": "background-main", - "table-header-foreground": "c09", - "table-header-border": "c07", - "table-focus-indicator": "background-main-5", - - // used in table code - "table-header-cell": "background-main", - "table-row-background-even": "background-main", - "table-row-background-odd": "background-main", - "table-row-background-focused": "background-main-1", - "table-row-background-focused-selected": "background-main-2", - "table-row-background-selected": "background-main-2", - - // foreground - "table-row-selected": "c12", - "table-row": "c09", - - // table grid color - "table-row-line": "background-main", - "table-column-line": "background-main", - // used in progressive code "progressive-table-header": "c08", "progressive-table-row-background-even": "background-main", diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js index 6b798200e18..629e75ccef5 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js @@ -124,28 +124,6 @@ qx.Theme.define("osparc.theme.ColorLight", { "tooltip-text": "text", - // table - "table-header": "background-main", - "table-header-foreground": "c09", - "table-header-border": "c07", - "table-focus-indicator": "background-main-5", - - // used in table code - "table-header-cell": "background-main", - "table-row-background-even": "background-main", - "table-row-background-odd": "background-main", - "table-row-background-focused": "background-main-1", - "table-row-background-focused-selected": "background-main-2", - "table-row-background-selected": "background-main-2", - - // foreground - "table-row-selected": "c12", - "table-row": "c09", - - // table grid color - "table-row-line": "background-main", - "table-column-line": "background-main", - // used in progressive code "progressive-table-header": "c08", "progressive-table-row-background-even": "background-main", diff --git a/services/static-webserver/client/source/class/osparc/theme/Decoration.js b/services/static-webserver/client/source/class/osparc/theme/Decoration.js index a1381421494..2a32cae2559 100644 --- a/services/static-webserver/client/source/class/osparc/theme/Decoration.js +++ b/services/static-webserver/client/source/class/osparc/theme/Decoration.js @@ -101,6 +101,10 @@ qx.Theme.define("osparc.theme.Decoration", { } }, + "form-input-focused-invalid": { + include: "form-input-invalid" + }, + "form-array-container": { style: { radius: 2, diff --git a/services/static-webserver/client/source/class/osparc/theme/mixin/Color.js b/services/static-webserver/client/source/class/osparc/theme/mixin/Color.js index 95883e8284f..aaa58363b21 100644 --- a/services/static-webserver/client/source/class/osparc/theme/mixin/Color.js +++ b/services/static-webserver/client/source/class/osparc/theme/mixin/Color.js @@ -29,7 +29,6 @@ qx.Theme.define("osparc.theme.mixin.Color", { "pb-locked": "rgba(113, 157, 181, 0.4)", // button - "default-button-text": "rgba(255, 255, 255, 1)", "default-button-text-outline": "contrasted-text-light", "default-button-text-action": "contrasted-text-dark", "default-button": "product-color", @@ -44,8 +43,6 @@ qx.Theme.define("osparc.theme.mixin.Color", { "default-button-disabled-background": "rgba(113, 157, 181, 0.25)", "default-button-focus-blur": "rgba(254, 233, 86, 1)", - "status_icon": "rgba(255, 255, 255, 1)", - // separator "border-separator": "product-color", @@ -54,6 +51,29 @@ qx.Theme.define("osparc.theme.mixin.Color", { "logger-warning-message": "warning-yellow", "logger-error-message": "failed-red", - "workbench-edge-selected": "busy-orange" + "workbench-edge-selected": "busy-orange", + + + // table + "table-header": "transparent", + "table-header-foreground": "text", // text color + "table-header-border": "text", // header underline + "table-header-cell": "transparent", + + // used in table code + "table-focus-indicator": "transparent", + "table-row-background-even": "transparent", + "table-row-background-odd": "transparent", + "table-row-background-focused": "transparent", + "table-row-background-focused-selected": "background-main-2", + "table-row-background-selected": "background-main-2", + + // foreground + "table-row-selected": "text", + "table-row": "text", + + // table grid color + "table-row-line": "transparent", + "table-column-line": "transparent", } }); diff --git a/services/static-webserver/client/source/class/osparc/tours/Manager.js b/services/static-webserver/client/source/class/osparc/tours/Manager.js index cf2618c3d4e..2d4445dcf7c 100644 --- a/services/static-webserver/client/source/class/osparc/tours/Manager.js +++ b/services/static-webserver/client/source/class/osparc/tours/Manager.js @@ -66,7 +66,7 @@ qx.Class.define("osparc.tours.Manager", { switch (id) { case "intro-text": control = new qx.ui.basic.Label().set({ - value: this.tr("This collection of Guided Tours will show you how to use the framework:"), + value: this.tr("This collection of Guided Tours will show you how to use the platform:"), rich: true, wrap: true, font: "text-14" diff --git a/services/static-webserver/client/source/class/osparc/ui/basic/DateAndBy.js b/services/static-webserver/client/source/class/osparc/ui/basic/DateAndBy.js index 320a5562f71..42ec2db9500 100644 --- a/services/static-webserver/client/source/class/osparc/ui/basic/DateAndBy.js +++ b/services/static-webserver/client/source/class/osparc/ui/basic/DateAndBy.js @@ -96,9 +96,12 @@ qx.Class.define("osparc.ui.basic.DateAndBy", { const atom = this.getChildControl("last-touching"); const myGroupId = osparc.auth.Data.getInstance().getGroupId(); if (groupId === myGroupId) { - atom.setLabel("by me"); + atom.set({ + label: "by me", + icon: null, + }) } else { - atom.setLabel("by"); + atom.setLabel("by "); osparc.dashboard.CardBase.addHintFromGids(atom, [groupId]); } } diff --git a/services/static-webserver/client/source/class/osparc/ui/basic/Logo.js b/services/static-webserver/client/source/class/osparc/ui/basic/Logo.js index e35d7659bda..c0d82042f21 100644 --- a/services/static-webserver/client/source/class/osparc/ui/basic/Logo.js +++ b/services/static-webserver/client/source/class/osparc/ui/basic/Logo.js @@ -32,6 +32,8 @@ qx.Class.define("osparc.ui.basic.Logo", { this.__resetSourcePath(); + osparc.utils.Utils.setAltToImage(this, "product-logo"); + const themeManager = qx.theme.manager.Meta.getInstance(); themeManager.addListener("changeTheme", () => this.__resetSourcePath(), this); diff --git a/services/static-webserver/client/source/class/osparc/ui/basic/NodeStatusUI.js b/services/static-webserver/client/source/class/osparc/ui/basic/NodeStatusUI.js index d89c747fbea..25d7917c230 100644 --- a/services/static-webserver/client/source/class/osparc/ui/basic/NodeStatusUI.js +++ b/services/static-webserver/client/source/class/osparc/ui/basic/NodeStatusUI.js @@ -145,7 +145,7 @@ qx.Class.define("osparc.ui.basic.NodeStatusUI", { this.getNode().getStatus().addListener("changeProgress", e => { const progress = e.getData(); if (progress > 0 && progress < 100) { - this.getChildControl("label").setValue(this.tr("Uploading")); + this.getChildControl("label").setValue(this.tr("Uploading...")); } }); } diff --git a/services/static-webserver/client/source/class/osparc/ui/basic/SVGImage.js b/services/static-webserver/client/source/class/osparc/ui/basic/SVGImage.js index 83ef2e94646..878413af5e9 100644 --- a/services/static-webserver/client/source/class/osparc/ui/basic/SVGImage.js +++ b/services/static-webserver/client/source/class/osparc/ui/basic/SVGImage.js @@ -127,6 +127,22 @@ qx.Class.define("osparc.ui.basic.SVGImage", { const brightnessValue = l3 => l3; const contrastValue = l4 => l4 > 50 ? 50 : l4; return `invert(${invertValue(l)}%) sepia(${sepiaValue(s)}%) saturate(${saturateValue(s)}%) hue-rotate(${h}deg) brightness(${brightnessValue(l)}%) contrast(${contrastValue(l)}%)`; + }, + + setColorToImage: function(image, keywordOrRgb) { + if (keywordOrRgb === null) { + keywordOrRgb = "text"; + } + let filterValue = this.self().keywordToCSSFilter(keywordOrRgb); + if (filterValue === null) { + const hexColor = qx.theme.manager.Color.getInstance().resolve(keywordOrRgb); + const rgbColor = qx.util.ColorUtil.hexStringToRgb(hexColor); + filterValue = this.self().rgbToCSSFilter(rgbColor); + } + const myStyle = { + "filter": filterValue + }; + image.getContentElement().setStyles(myStyle); } }, @@ -160,19 +176,7 @@ qx.Class.define("osparc.ui.basic.SVGImage", { * @param keywordOrRgb {string} predefined keyword or rgb in the following format "0,255,0" */ __applyImageColor: function(keywordOrRgb) { - if (keywordOrRgb === null) { - keywordOrRgb = "text"; - } - let filterValue = this.self().keywordToCSSFilter(keywordOrRgb); - if (filterValue === null) { - const hexColor = qx.theme.manager.Color.getInstance().resolve(keywordOrRgb); - const rgbColor = qx.util.ColorUtil.hexStringToRgb(hexColor); - filterValue = this.self().rgbToCSSFilter(rgbColor); - } - const myStyle = { - "filter": filterValue - }; - this.getChildControl("image").getContentElement().setStyles(myStyle); - } + this.self().setColorToImage(this.getChildControl("image"), keywordOrRgb); + }, } }); diff --git a/services/static-webserver/client/source/class/osparc/ui/basic/Thumbnail.js b/services/static-webserver/client/source/class/osparc/ui/basic/Thumbnail.js index 53e9d77285c..ff7c6f5d98d 100644 --- a/services/static-webserver/client/source/class/osparc/ui/basic/Thumbnail.js +++ b/services/static-webserver/client/source/class/osparc/ui/basic/Thumbnail.js @@ -72,7 +72,7 @@ qx.Class.define("osparc.ui.basic.Thumbnail", { allowStretchX: true, allowStretchY: true, alignX: "center", - alignY: "middle" + alignY: "middle", }); this.addCenteredWidget(control); break; @@ -83,7 +83,11 @@ qx.Class.define("osparc.ui.basic.Thumbnail", { __applySource: function(val) { const image = this.getChildControl("image"); if (val) { - image.setSource(val); + if (osparc.utils.Utils.isValidHttpUrl(val)) { + osparc.utils.Utils.setUrlSourceToImage(image, val); + } else { + image.setSource(val); + } } }, @@ -96,10 +100,16 @@ qx.Class.define("osparc.ui.basic.Thumbnail", { if (srcWidth && srcHeight) { const aspectRatio = srcWidth/srcHeight; if (this.getBounds() && this.getBounds().width < image.getMaxWidth()) { - image.setMaxWidth(this.getBounds().width); + image.set({ + minWidth: parseInt(this.getBounds().width), + maxWidth: parseInt(this.getBounds().width), + }); } if (this.getBounds() && this.getBounds().height < image.getMaxHeight()) { - image.setMaxHeight(this.getBounds().height); + image.set({ + minHeight: parseInt(this.getBounds().height), + maxHeight: parseInt(this.getBounds().height), + }); } const maxWidth = image.getMaxWidth(); const maxHeight = image.getMaxHeight(); @@ -107,12 +117,18 @@ qx.Class.define("osparc.ui.basic.Thumbnail", { if (maxWidth && maxHeight) { const newMaxHeight = maxWidth/aspectRatio; if (newMaxHeight < maxHeight) { - image.setMaxHeight(parseInt(newMaxHeight)); + image.set({ + minHeight: parseInt(newMaxHeight), + maxHeight: parseInt(newMaxHeight), + }); return; } const newMaxWidth = maxHeight*aspectRatio; if (newMaxWidth < maxWidth) { - image.setMaxWidth(parseInt(newMaxWidth)); + image.set({ + minWidth: parseInt(newMaxWidth), + maxWidth: parseInt(newMaxWidth), + }); return; } return; @@ -120,13 +136,19 @@ qx.Class.define("osparc.ui.basic.Thumbnail", { if (maxWidth) { const newMaxHeight = maxWidth/aspectRatio; - image.setMaxHeight(parseInt(newMaxHeight)); + image.set({ + minHeight: parseInt(newMaxHeight), + maxHeight: parseInt(newMaxHeight), + }); return; } if (maxHeight) { const newMaxWidth = maxHeight*aspectRatio; - image.setMaxWidth(parseInt(newMaxWidth)); + image.set({ + minWidth: parseInt(newMaxWidth), + maxWidth: parseInt(newMaxWidth), + }); return; } } diff --git a/services/static-webserver/client/source/class/osparc/ui/form/renderer/SingleWithIcon.js b/services/static-webserver/client/source/class/osparc/ui/form/renderer/SingleWithIcon.js new file mode 100644 index 00000000000..a8252d6040c --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/ui/form/renderer/SingleWithIcon.js @@ -0,0 +1,62 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2025 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.ui.form.renderer.SingleWithIcon", { + extend: qx.ui.form.renderer.Single, + + construct: function(form, icons) { + if (icons) { + this.__icons = icons; + } else { + this.__icons = {}; + } + + this.base(arguments, form); + }, + + members: { + __icons: null, + + setIcons: function(icons) { + this.__icons = icons; + + this._onFormChange(); + }, + + // overridden + addItems: function(items, names, title, itemOptions, headerOptions) { + this.base(arguments, items, names, title, itemOptions, headerOptions); + + // header + let row = title === null ? 0 : 1; + + for (let i = 0; i < items.length; i++) { + if (i in this.__icons) { + const image = new qx.ui.basic.Image(this.__icons[i]).set({ + alignY: "middle", + }); + this._add(image, { + row, + column: 2, + }); + } + + row++; + } + }, + } +}); diff --git a/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js b/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js index c51147869fb..5b76a46045a 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/CollaboratorListItem.js @@ -138,7 +138,7 @@ qx.Class.define("osparc.ui.list.CollaboratorListItem", { // highlight me const email = osparc.auth.Data.getInstance().getEmail(); - if (email === value) { + if (value && value.includes(email)) { this.addState("selected"); } }, diff --git a/services/static-webserver/client/source/class/osparc/ui/list/MemberListItem.js b/services/static-webserver/client/source/class/osparc/ui/list/MemberListItem.js index 1019b421875..db088d1f691 100644 --- a/services/static-webserver/client/source/class/osparc/ui/list/MemberListItem.js +++ b/services/static-webserver/client/source/class/osparc/ui/list/MemberListItem.js @@ -36,7 +36,7 @@ qx.Class.define("osparc.ui.list.MemberListItem", { // highlight me const email = osparc.auth.Data.getInstance().getEmail(); - if (email === value) { + if (value && value.includes(email)) { this.addState("selected"); } }, diff --git a/services/static-webserver/client/source/class/osparc/ui/markdown/MarkdownWindow.js b/services/static-webserver/client/source/class/osparc/ui/markdown/MarkdownWindow.js new file mode 100644 index 00000000000..d77a15ad4a0 --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/ui/markdown/MarkdownWindow.js @@ -0,0 +1,39 @@ +/* + * oSPARC - The SIMCORE frontend - https://osparc.io + * Copyright: 2025 IT'IS Foundation - https://itis.swiss + * License: MIT - https://opensource.org/licenses/MIT + * Authors: Odei Maiz (odeimaiz) + */ + +qx.Class.define("osparc.ui.markdown.MarkdownWindow", { + extend: osparc.ui.window.Window, + + construct: function(markdownUrl) { + this.base(arguments); + + this.set({ + layout: new qx.ui.layout.VBox(), + resizable: true, + showMaximize: false, + showMinimize: false, + centerOnAppear: true, + clickAwayClose: true, + modal: true + }); + + const markdown = new osparc.ui.markdown.Markdown().set({ + padding: 15, + }); + const scrollContainer = new qx.ui.container.Scroll(); + scrollContainer.add(markdown); + this._add(scrollContainer, { + flex: 1, + }); + + if (markdownUrl) { + fetch(markdownUrl) + .then(res => res.text()) + .then(text => markdown.setValue(text)); + } + }, +}); diff --git a/services/static-webserver/client/source/class/osparc/ui/message/FlashMessage.js b/services/static-webserver/client/source/class/osparc/ui/message/FlashMessage.js index d111984ad1e..c7a8f811dfe 100644 --- a/services/static-webserver/client/source/class/osparc/ui/message/FlashMessage.js +++ b/services/static-webserver/client/source/class/osparc/ui/message/FlashMessage.js @@ -31,7 +31,7 @@ qx.Class.define("osparc.ui.message.FlashMessage", { */ construct: function(message, level, duration) { this.base(arguments); - this._setLayout(new qx.ui.layout.VBox(15)); + this._setLayout(new qx.ui.layout.VBox(10)); this.set({ padding: 18, @@ -48,14 +48,13 @@ qx.Class.define("osparc.ui.message.FlashMessage", { textColor: this.self().LOG_LEVEL_COLOR_MAP[level].color }); - if (message) { - this.setMessage(message); - } + this.setMessage(message); - // also support duration 0: the message won't be automatically removed - if (duration != null) { - this.setDuration(duration); + if ([null, undefined].includes(duration)) { + const wordCount = message.split(" ").length; + duration = Math.max(5500, wordCount*500); // An average reader takes 300ms to read a word } + this.setDuration(duration); this.getChildControl("closebutton"); }, @@ -68,13 +67,14 @@ qx.Class.define("osparc.ui.message.FlashMessage", { message: { check: "String", - nullable: true, - apply: "__applyMessage" + nullable: false, + apply: "__applyMessage", }, duration: { check: "Number", - nullable: true + init: null, + nullable: true, } }, @@ -142,10 +142,7 @@ qx.Class.define("osparc.ui.message.FlashMessage", { }, __applyMessage: function(value) { - const label = this.getChildControl("message"); - if (label) { - label.setValue(value); - } + this.getChildControl("message").setValue(value); }, addWidget: function(widget) { diff --git a/services/static-webserver/client/source/class/osparc/ui/message/Loading.js b/services/static-webserver/client/source/class/osparc/ui/message/Loading.js index bfd238cd9e2..e222ead4cbf 100644 --- a/services/static-webserver/client/source/class/osparc/ui/message/Loading.js +++ b/services/static-webserver/client/source/class/osparc/ui/message/Loading.js @@ -19,6 +19,7 @@ * The loading page * * ----------------------- + * | [] | * | | * | oSparc/service logo | * | spinner + header | @@ -31,19 +32,20 @@ qx.Class.define("osparc.ui.message.Loading", { extend: qx.ui.core.Widget, - /** - * Constructor for the Loading widget. - * - * @param {Boolean} showMaximizeButton - */ - construct: function(showMaximizeButton = false) { + construct: function() { this.base(arguments); - this._setLayout(new qx.ui.layout.HBox()); - this.set({ - alignX: "center" - }); - this.__buildLayout(showMaximizeButton); + const layout = new qx.ui.layout.Grid(20, 20); + layout.setRowFlex(this.self().GRID_POS.SPACER_TOP, 1); + layout.setRowFlex(this.self().GRID_POS.SPACER_BOTTOM, 1); + layout.setColumnFlex(0, 1); + layout.setColumnMaxWidth(1, 400); + layout.setColumnAlign(1, "center", "middle"); + layout.setColumnFlex(2, 1); + layout.setColumnAlign(2, "right", "middle"); + this._setLayout(layout); + + this.__buildLayout(); }, properties: { @@ -64,15 +66,21 @@ qx.Class.define("osparc.ui.message.Loading", { check: "Array", nullable: true, apply: "__applyMessages" + }, + + /** + * Show Restart-Maximize Toolbar + */ + showToolbar: { + check: "Boolean", + init: false, + event: "changeShowToolbar", } }, - // from osparc.widget.PersistentIframe events: { - /** Fired if the iframe is restored from a minimized or maximized state */ "restore" : "qx.event.type.Event", - /** Fired if the iframe is maximized */ - "maximize" : "qx.event.type.Event" + "maximize" : "qx.event.type.Event", }, statics: { @@ -82,42 +90,38 @@ qx.Class.define("osparc.ui.message.Loading", { STATUS_ICON_SIZE: 20, GRID_POS: { - LOGO: 1, - WAITING: 2, - MESSAGES: 3, - EXTRA_WIDGETS: 4 + TOOLBAR: 0, + SPACER_TOP: 1, + LOGO: 2, + WAITING: 3, + MESSAGES: 4, + EXTRA_WIDGETS: 5, + SPACER_BOTTOM: 6, } }, members: { - __mainLayout: null, __thumbnail: null, __header: null, __messagesContainer: null, __extraWidgets: null, __maxButton: null, - __buildLayout: function(showMaximizeButton) { - this.__createMainLayout(); - this.__createMaximizeButton(showMaximizeButton); - }, - - __createMainLayout: function() { - const layout = new qx.ui.layout.Grid(20, 20); - layout.setColumnFlex(0, 1); - const mainLayout = this.__mainLayout = new qx.ui.container.Composite(new qx.ui.layout.VBox(20).set({ - alignX: "center", - alignY: "middle" - })).set({ - width: 400, - padding: 0 - }); + __buildLayout: function() { this._add(new qx.ui.core.Widget(), { - flex: 1 + column: 0, + row: 0 }); - this._add(mainLayout); - this._add(new qx.ui.core.Widget(), { - flex: 1 + + const maxLayout = this.__createMaximizeToolbar(); + this._add(maxLayout, { + column: 2, + row: this.self().GRID_POS.TOOLBAR + }); + + this._add(new qx.ui.core.Spacer(), { + column: 1, + row: this.self().GRID_POS.SPACER_TOP }); const productLogoPath = osparc.product.Utils.getLogoPath(); @@ -137,8 +141,8 @@ qx.Class.define("osparc.ui.message.Loading", { height: logoHeight }); } - mainLayout.addAt(thumbnail, { - column: 0, + this._add(thumbnail, { + column: 1, row: this.self().GRID_POS.LOGO }); @@ -150,53 +154,76 @@ qx.Class.define("osparc.ui.message.Loading", { gap: 15, allowGrowX: false }); + const icon = waitingHeader.getChildControl("icon"); + osparc.service.StatusUI.updateCircleAnimation(icon); const label = waitingHeader.getChildControl("label"); label.set({ rich: true, - wrap: true + wrap: true, + alignX: "center", }); - const icon = waitingHeader.getChildControl("icon"); - osparc.service.StatusUI.updateCircleAnimation(icon); - mainLayout.addAt(waitingHeader, { - column: 0, + this._add(waitingHeader, { + column: 1, row: this.self().GRID_POS.WAITING }); const messages = this.__messagesContainer = new qx.ui.container.Composite(new qx.ui.layout.VBox(10).set({ alignX: "center" })); - mainLayout.addAt(messages, { - column: 0, + this._add(messages, { + column: 1, row: this.self().GRID_POS.MESSAGES }); const extraWidgets = this.__extraWidgets = new qx.ui.container.Composite(new qx.ui.layout.VBox(10).set({ alignX: "center" })); - mainLayout.addAt(extraWidgets, { - column: 0, + this._add(extraWidgets, { + column: 1, row: this.self().GRID_POS.EXTRA_WIDGETS }); + + this._add(new qx.ui.core.Spacer(), { + column: 1, + row: this.self().GRID_POS.SPACER_BOTTOM + }); + }, + + maximizeIFrame: function(maximize) { + if (maximize) { + this.fireEvent("maximize"); + this.addState("maximized"); + } else { + this.fireEvent("restore"); + this.removeState("maximized"); + } + const maxButton = this.__maxButton; + maxButton.set({ + label: osparc.widget.PersistentIframe.getZoomLabel(maximize), + icon: osparc.widget.PersistentIframe.getZoomIcon(maximize) + }); + osparc.utils.Utils.setIdToWidget(maxButton, osparc.widget.PersistentIframe.getMaximizeWidgetId(maximize)); + qx.event.message.Bus.getInstance().dispatchByName("maximizeIframe", this.hasState("maximized")); }, - __createMaximizeButton: function(showMaximizeButton) { + __createMaximizeToolbar: function() { const maximize = false; const maxButton = this.__maxButton = osparc.widget.PersistentIframe.createToolbarButton(maximize).set({ + maxHeight: 25, label: osparc.widget.PersistentIframe.getZoomLabel(maximize), icon: osparc.widget.PersistentIframe.getZoomIcon(maximize), - visibility: showMaximizeButton ? "visible" : "excluded" }); osparc.utils.Utils.setIdToWidget(maxButton, osparc.widget.PersistentIframe.getMaximizeWidgetId(maximize)); maxButton.addListener("execute", () => this.maximizeIFrame(!this.hasState("maximized")), this); - const maximizeLayout = new qx.ui.container.Composite(new qx.ui.layout.VBox()).set({ - maxWidth: 100 - }); - maximizeLayout.add(maxButton); - maximizeLayout.add(new qx.ui.core.Widget(), { - flex: 1 + const toolbarLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ + alignX: "right", + })); + this.bind("showToolbar", toolbarLayout, "visibility", { + converter: showToolbar => showToolbar ? "visible" : "hidden" }); - this._add(maximizeLayout); + toolbarLayout.add(maxButton); + return toolbarLayout; }, __applyLogo: function(newLogo) { @@ -264,23 +291,5 @@ qx.Class.define("osparc.ui.message.Loading", { addExtraWidget: function(widget) { this.__extraWidgets.add(widget); }, - - // from osparc.widget.PersistentIframe - maximizeIFrame: function(maximize) { - if (maximize) { - this.fireEvent("maximize"); - this.addState("maximized"); - } else { - this.fireEvent("restore"); - this.removeState("maximized"); - } - const maxButton = this.__maxButton; - maxButton.set({ - label: osparc.widget.PersistentIframe.getZoomLabel(maximize), - icon: osparc.widget.PersistentIframe.getZoomIcon(maximize) - }); - osparc.utils.Utils.setIdToWidget(maxButton, osparc.widget.PersistentIframe.getMaximizeWidgetId(maximize)); - qx.event.message.Bus.getInstance().dispatchByName("maximizeIframe", this.hasState("maximized")); - } } }); diff --git a/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ButtonRenderer.js b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ButtonRenderer.js new file mode 100644 index 00000000000..445840c570d --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ButtonRenderer.js @@ -0,0 +1,58 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2035 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.ui.table.cellrenderer.ButtonRenderer", { + extend: osparc.ui.table.cellrenderer.Html, + + construct: function(clickAction) { + this.base(arguments); + + this.setClickAction(clickAction); + }, + + properties: { + clickAction: { + check: "String", + nullable: false, + init: "clickAction", + }, + + buttonContent: { + check: "String", + nullable: false, + init: "", + } + }, + + members: { + // Override + _getContentHtml: function(cellInfo) { + const clickAction = this.getClickAction(); + const buttonContent = this.getButtonContent(); + + // Return the button with the image + return ` + + `; + }, + } +}); diff --git a/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/Html.js b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/Html.js index bc715a54051..11e1c5b976e 100644 --- a/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/Html.js +++ b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/Html.js @@ -20,9 +20,7 @@ */ qx.Class.define("osparc.ui.table.cellrenderer.Html", { extend: qx.ui.table.cellrenderer.Html, - construct: function() { - this.base(arguments); - }, + members: { // Override _getCellStyle: function(cellInfo) { diff --git a/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ImageButtonRenderer.js b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ImageButtonRenderer.js new file mode 100644 index 00000000000..8b9fd7896bd --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/ui/table/cellrenderer/ImageButtonRenderer.js @@ -0,0 +1,44 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2035 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.ui.table.cellrenderer.ImageButtonRenderer", { + extend: osparc.ui.table.cellrenderer.ButtonRenderer, + + construct: function(clickAction, iconPath) { + this.base(arguments, clickAction); + + this.setIconPath(iconPath); + }, + + properties: { + iconPath: { + check: "String", + init: null, + nullable: false, + apply: "__applyIconPath", + }, + }, + + members: { + __applyIconPath: function(iconPath) { + const resMgr = qx.util.ResourceManager.getInstance(); + const iconUrl = resMgr.toUri(iconPath); // Resolves to the correct URL of the asset + + this.setButtonContent(`
Here you can check the logs of the progress:"); + const text = this.tr("To proceed, we need to prepare some inputs. You can check the progress logs here:"); const title = new qx.ui.basic.Label(text).set({ font: "text-14", rich: true diff --git a/services/static-webserver/client/source/class/osparc/widget/ProgressSequence.js b/services/static-webserver/client/source/class/osparc/widget/ProgressSequence.js index d9c3e257d81..09cadba98af 100644 --- a/services/static-webserver/client/source/class/osparc/widget/ProgressSequence.js +++ b/services/static-webserver/client/source/class/osparc/widget/ProgressSequence.js @@ -25,7 +25,8 @@ qx.Class.define("osparc.widget.ProgressSequence", { this.set({ backgroundColor: "window-popup-background", - paddingBottom: 8 + paddingBottom: 8, + minWidth: 400, }); this.__initLayout(title); diff --git a/services/static-webserver/client/source/class/osparc/widget/NodeDataManager.js b/services/static-webserver/client/source/class/osparc/widget/StudyDataManager.js similarity index 77% rename from services/static-webserver/client/source/class/osparc/widget/NodeDataManager.js rename to services/static-webserver/client/source/class/osparc/widget/StudyDataManager.js index 397477d1d94..092d04be44c 100644 --- a/services/static-webserver/client/source/class/osparc/widget/NodeDataManager.js +++ b/services/static-webserver/client/source/class/osparc/widget/StudyDataManager.js @@ -25,12 +25,12 @@ * Here is a little example of how to use the widget. * *
- * let dataManager = new osparc.widget.NodeDataManager(null, nodeId); + * let dataManager = new osparc.widget.StudyDataManager(null, nodeId); * this.getRoot().add(dataManager); **/ -qx.Class.define("osparc.widget.NodeDataManager", { +qx.Class.define("osparc.widget.StudyDataManager", { extend: qx.ui.core.Widget, /** @@ -42,22 +42,26 @@ qx.Class.define("osparc.widget.NodeDataManager", { this._setLayout(new qx.ui.layout.VBox(10)); - if (studyId) { - this.set({ - studyId - }); - } + this.setStudyId(studyId); if (nodeId) { - this.set({ - nodeId - }); + this.setNodeId(nodeId); } this.__buildLayout(); this.__reloadTree(); }, + statics: { + popUpInWindow: function(studyId, nodeId, title) { + const studyDataManager = new osparc.widget.StudyDataManager(studyId, nodeId); + if (!title) { + title = osparc.product.Utils.getStudyAlias({firstUpperCase: true}) + qx.locale.Manager.tr(" Files"); + } + return osparc.ui.window.Window.popUpInWindow(studyDataManager, title, osparc.dashboard.ResourceDetails.WIDTH, osparc.dashboard.ResourceDetails.HEIGHT); + }, + }, + properties: { studyId: { check: "String", @@ -91,9 +95,6 @@ qx.Class.define("osparc.widget.NodeDataManager", { const treeFolderView = this.getChildControl("tree-folder-view"); treeFolderView.getChildControl("folder-tree").setBackgroundColor("window-popup-background"); - const reloadButton = treeFolderView.getChildControl("reload-button"); - reloadButton.addListener("execute", () => this.__reloadTree(), this); - const selectedFileLayout = treeFolderView.getChildControl("folder-viewer").getChildControl("selected-file-layout"); selectedFileLayout.addListener("fileDeleted", e => this.__fileDeleted(e.getData()), this); }, @@ -103,11 +104,12 @@ qx.Class.define("osparc.widget.NodeDataManager", { const foldersTree = treeFolderView.getChildControl("folder-tree"); foldersTree.resetCache(); - if (this.getStudyId()) { - foldersTree.populateStudyTree(this.getStudyId()); - } if (this.getNodeId()) { - foldersTree.populateNodeTree(this.getNodeId()); + foldersTree.populateNodeTree(this.getStudyId(), this.getNodeId()); + treeFolderView.requestSize(this.getStudyId(), this.getNodeId()); + } else if (this.getStudyId()) { + foldersTree.populateStudyTree(this.getStudyId()); + treeFolderView.requestSize(this.getStudyId()); } const folderViewer = treeFolderView.getChildControl("folder-viewer"); @@ -134,13 +136,12 @@ qx.Class.define("osparc.widget.NodeDataManager", { treeFolderView.openPath(path); }; - if (this.getStudyId()) { - foldersTree.populateStudyTree(this.getStudyId()) + if (this.getNodeId()) { + foldersTree.populateNodeTree(this.getStudyId(), this.getNodeId()) .then(() => openSameFolder()) .catch(err => console.error(err)); - } - if (this.getNodeId()) { - foldersTree.populateNodeTree(this.getNodeId()) + } else if (this.getStudyId()) { + foldersTree.populateStudyTree(this.getStudyId()) .then(() => openSameFolder()) .catch(err => console.error(err)); } diff --git a/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js b/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js index c9a1795bf4a..1658268f563 100644 --- a/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js +++ b/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js @@ -126,6 +126,7 @@ qx.Class.define("osparc.widget.logger.LoggerView", { switch (id) { case "toolbar": control = new qx.ui.toolbar.ToolBar(); + control.setBackgroundColor("transparent"); this._add(control); break; case "pin-node": { @@ -143,7 +144,9 @@ qx.Class.define("osparc.widget.logger.LoggerView", { control = new qx.ui.form.TextField().set({ appearance: "toolbar-textfield", liveUpdate: true, - placeholder: this.tr("Filter") + placeholder: this.tr("Filter"), + marginLeft: 5, + marginRight: 5, }); osparc.utils.Utils.setIdToWidget(control, "logsFilterField"); toolbar.add(control, { @@ -221,7 +224,7 @@ qx.Class.define("osparc.widget.logger.LoggerView", { toolTipText: this.tr("Download logs"), appearance: "toolbar-button" }); - osparc.utils.Utils.setIdToWidget(control, "downloadLogsButton"); + osparc.utils.Utils.setIdToWidget(control, "__downloadLogsButton"); toolbar.add(control); break; } @@ -256,7 +259,7 @@ qx.Class.define("osparc.widget.logger.LoggerView", { toolbar.add(copySelectedToClipboardButton); const downloadButton = this.getChildControl("download-logs-button"); - downloadButton.addListener("execute", () => this.downloadLogs(), this); + downloadButton.addListener("execute", () => this.__downloadLogs(), this); toolbar.add(downloadButton); return toolbar; @@ -345,7 +348,7 @@ qx.Class.define("osparc.widget.logger.LoggerView", { } }, - downloadLogs: function() { + __downloadLogs: function() { const logs = this.__getLogsString(); const blob = new Blob([logs], {type: "text/plain"}); osparc.utils.Utils.downloadBlobContent(blob, "logs.log"); diff --git a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js index c0a3a8e835d..0ae4c40e3ef 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js +++ b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js @@ -134,14 +134,14 @@ qx.Class.define("osparc.workbench.DiskUsageController", { case "CRITICAL": if (shouldDisplayMessage(prevDiskUsageState, warningLevel)) { message = qx.locale.Manager.tr(`Out of Disk Space on "Service Filesystem" for ${nodeName}
The volume Service Filesystem has only ${freeSpace} disk space remaining. You can free up disk space by removing unused files in your service. Alternatively, you can run your service with a pricing plan that supports your storage requirements.`); - osparc.FlashMessenger.getInstance().logAs(message, "ERROR"); + osparc.FlashMessenger.logError(message); this.__prevDiskUsageStateList[objIndex].state = warningLevel; } break; case "WARNING": if (shouldDisplayMessage(prevDiskUsageState, warningLevel)) { message = qx.locale.Manager.tr(`Low Disk Space on "Service Filesystem" for ${nodeName}
The volume Service Filesystem has only ${freeSpace} disk space remaining. You can free up disk space by removing unused files in your service. Alternatively, you can run your service with a pricing plan that supports your storage requirements.`); - osparc.FlashMessenger.getInstance().logAs(message, "WARNING"); + osparc.FlashMessenger.logAs(message, "WARNING"); this.__prevDiskUsageStateList[objIndex].state = warningLevel; } break; diff --git a/services/static-webserver/client/source/class/osparc/workbench/NodeUI.js b/services/static-webserver/client/source/class/osparc/workbench/NodeUI.js index cc1af9a52e3..d1535459f94 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/NodeUI.js +++ b/services/static-webserver/client/source/class/osparc/workbench/NodeUI.js @@ -150,7 +150,7 @@ qx.Class.define("osparc.workbench.NodeUI", { const statusLabel = nodeStatus.getChildControl("label"); const requestOpenLogger = () => this.fireEvent("requestOpenLogger"); const evaluateLabel = () => { - const failed = statusLabel.getValue() === "Failed"; + const failed = statusLabel.getValue() === "Unsuccessful"; statusLabel.setCursor(failed ? "pointer" : "auto"); if (nodeStatus.hasListener("tap")) { nodeStatus.removeListener("tap", requestOpenLogger); diff --git a/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js b/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js index d14212aa20f..1679b23bcd6 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js +++ b/services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js @@ -225,35 +225,33 @@ qx.Class.define("osparc.workbench.ServiceCatalog", { }); osparc.service.Utils.sortObjectsBasedOn(filteredServices, this.__sortBy); - const filteredServicesObj = this.__filteredServicesObj = osparc.service.Utils.convertArrayToObject(filteredServices); - - const groupedServicesList = []; - for (const key in filteredServicesObj) { - const serviceMetadata = osparc.service.Utils.getLatest(key); - if (serviceMetadata) { - const service = new osparc.data.model.Service(serviceMetadata); - groupedServicesList.push(service); - } - } + this.__filteredServicesObj = osparc.service.Utils.convertArrayToObject(filteredServices); + + const servicesModels = []; + filteredServices.forEach(filteredService => { + const service = new osparc.data.model.Service(filteredService); + servicesModels.push(service); + }); - this.__serviceList.setModel(new qx.data.Array(groupedServicesList)); + this.__serviceList.setModel(new qx.data.Array(servicesModels)); }, __changedSelection: function(key) { if (this.__versionsBox) { - let selectBox = this.__versionsBox; + const selectBox = this.__versionsBox; selectBox.removeAll(); if (key in this.__filteredServicesObj) { const latest = new qx.ui.form.ListItem(this.self().LATEST); latest.version = this.self().LATEST; selectBox.add(latest); - const versions = osparc.service.Utils.getVersions(key); - versions.forEach(version => { - const listItem = osparc.service.Utils.versionToListItem(key, version); - selectBox.add(listItem); - }); - osparc.utils.Utils.growSelectBox(selectBox, 200); - selectBox.setSelection([latest]); + osparc.store.Services.populateVersionsSelectBox(key, selectBox) + .then(() => { + osparc.utils.Utils.growSelectBox(selectBox, 200); + const idx = selectBox.getSelectables().indexOf(latest); + if (idx > -1) { + selectBox.setSelection([latest]); + } + }); } } if (this.__addBtn) { diff --git a/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js b/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js index 19a123cf547..ec3f146fef3 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js +++ b/services/static-webserver/client/source/class/osparc/workbench/WorkbenchUI.js @@ -202,7 +202,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { }, __addStartHint: function() { - this.__startHint = new qx.ui.basic.Label(this.tr("Double click to start adding a node")).set({ + this.__startHint = new qx.ui.basic.Label(this.tr("Double-click to add a node")).set({ font: "workbench-start-hint", textColor: "workbench-start-hint", visibility: "excluded" @@ -299,7 +299,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { return null; } if (this.getStudy().isPipelineRunning()) { - osparc.FlashMessenger.getInstance().logAs(osparc.data.model.Workbench.CANT_ADD_NODE, "ERROR"); + osparc.FlashMessenger.logError(osparc.data.model.Workbench.CANT_ADD_NODE); return null; } const srvCat = new osparc.workbench.ServiceCatalog(); @@ -1919,7 +1919,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { }; if (type === "rect") { if ([null, undefined].includes(annotation)) { - osparc.FlashMessenger.getInstance().logAs(this.tr("Draw a rectangle first"), "WARNING"); + osparc.FlashMessenger.logAs(this.tr("Draw a rectangle first"), "WARNING"); return false; } serializeData.attributes = osparc.wrapper.Svg.getRectAttributes(annotation); @@ -2005,7 +2005,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { x: e.offsetX, y: e.offsetY }; - const service = qx.data.marshal.Json.createModel(osparc.service.Utils.getFilePicker()); + const service = qx.data.marshal.Json.createModel(osparc.store.Services.getFilePicker()); const nodeUI = await this.__addNode(service, pos); if (nodeUI) { const filePicker = new osparc.file.FilePicker(nodeUI.getNode(), "workbench"); @@ -2013,10 +2013,10 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { filePicker.addListener("fileUploaded", () => this.fireDataEvent("nodeSelected", nodeUI.getNodeId()), this); } } else { - osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR"); + osparc.FlashMessenger.logError(osparc.file.FileDrop.ONE_FILE_ONLY); } } else { - osparc.FlashMessenger.getInstance().logAs(this.tr("Folders are not accepted. You might want to upload a zip file."), "ERROR"); + osparc.FlashMessenger.logError(this.tr("Folders are not accepted. Please upload a zip file instead.")); } } }, @@ -2028,7 +2028,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", { const data = this.__isDraggingLink["dragData"]; this.__isDraggingLink = null; const pos = this.__pointerEventToWorkbenchPos(e, false); - const service = qx.data.marshal.Json.createModel(osparc.service.Utils.getFilePicker()); + const service = qx.data.marshal.Json.createModel(osparc.store.Services.getFilePicker()); const nodeUI = await this.__addNode(service, pos); if (nodeUI) { const node = nodeUI.getNode(); diff --git a/services/static-webserver/client/source/resource/common/common.css b/services/static-webserver/client/source/resource/common/common.css index 034546a9155..d032c109151 100644 --- a/services/static-webserver/client/source/resource/common/common.css +++ b/services/static-webserver/client/source/resource/common/common.css @@ -15,7 +15,11 @@ } .rotate { - animation: rotation 1.5s infinite linear; + animation: rotation 2s infinite linear; +} + +.rotateSlow { + animation: rotation 4s infinite linear; } .verticalText { diff --git a/services/static-webserver/client/source/resource/osparc/circle-info-text.svg b/services/static-webserver/client/source/resource/osparc/circle-info-text.svg new file mode 100644 index 00000000000..b753c59738b --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/circle-info-text.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/static-webserver/client/source/resource/osparc/circle-stop-text.svg b/services/static-webserver/client/source/resource/osparc/circle-stop-text.svg new file mode 100644 index 00000000000..1470fd0f295 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/circle-stop-text.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/static-webserver/client/source/resource/osparc/icons/MC.png b/services/static-webserver/client/source/resource/osparc/icons/MC.png new file mode 100644 index 00000000000..e0b1b0e22dd Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/MC.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/PM.png b/services/static-webserver/client/source/resource/osparc/icons/PM.png new file mode 100644 index 00000000000..b37a969a9bb Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/PM.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/TI.png b/services/static-webserver/client/source/resource/osparc/icons/TI.png new file mode 100644 index 00000000000..81355a82873 Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/TI.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/diagram.png b/services/static-webserver/client/source/resource/osparc/icons/diagram.png new file mode 100644 index 00000000000..7ed509475bd Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/diagram.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/pMC.png b/services/static-webserver/client/source/resource/osparc/icons/pMC.png new file mode 100644 index 00000000000..0af588f4b51 Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/pMC.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/pPM.png b/services/static-webserver/client/source/resource/osparc/icons/pPM.png new file mode 100644 index 00000000000..64eb011b33d Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/pPM.png differ diff --git a/services/static-webserver/client/source/resource/osparc/icons/pTI.png b/services/static-webserver/client/source/resource/osparc/icons/pTI.png new file mode 100644 index 00000000000..911dad5a02b Binary files /dev/null and b/services/static-webserver/client/source/resource/osparc/icons/pTI.png differ diff --git a/services/static-webserver/client/source/resource/osparc/logs-text.svg b/services/static-webserver/client/source/resource/osparc/logs-text.svg new file mode 100644 index 00000000000..20dd82103ce --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/logs-text.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/static-webserver/client/source/resource/osparc/market/AnimalWholeBody.svg b/services/static-webserver/client/source/resource/osparc/market/AnimalWholeBody.svg new file mode 100644 index 00000000000..cbfbc598fc1 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/market/AnimalWholeBody.svg @@ -0,0 +1,68 @@ + + diff --git a/services/static-webserver/client/source/resource/osparc/market/ComputationalPhantom.svg b/services/static-webserver/client/source/resource/osparc/market/ComputationalPhantom.svg new file mode 100644 index 00000000000..806af72544a --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/market/ComputationalPhantom.svg @@ -0,0 +1,48 @@ + + diff --git a/services/static-webserver/client/source/resource/osparc/market/HumanBodyRegion.svg b/services/static-webserver/client/source/resource/osparc/market/HumanBodyRegion.svg new file mode 100644 index 00000000000..f3fd1e47848 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/market/HumanBodyRegion.svg @@ -0,0 +1,60 @@ + + diff --git a/services/static-webserver/client/source/resource/osparc/market/HumanWholeBody.svg b/services/static-webserver/client/source/resource/osparc/market/HumanWholeBody.svg new file mode 100644 index 00000000000..2321ac47670 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/market/HumanWholeBody.svg @@ -0,0 +1,51 @@ + + diff --git a/services/static-webserver/client/source/resource/osparc/market/RentedModels.svg b/services/static-webserver/client/source/resource/osparc/market/RentedModels.svg new file mode 100644 index 00000000000..12fa9d1d5c1 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/market/RentedModels.svg @@ -0,0 +1,53 @@ + + diff --git a/services/static-webserver/client/source/resource/osparc/mock_jobs.json b/services/static-webserver/client/source/resource/osparc/mock_jobs.json new file mode 100644 index 00000000000..3aa775971e7 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/mock_jobs.json @@ -0,0 +1,85 @@ +{ + "jobs": [{ + "job_id": "0cde4607-07de-4bb5-b3e6-487f22387a70", + "solver": "isolve:2.4.12", + "status": "PUBLISHED", + "progress": 0, + "submitted_at": "2024-11-27 14:03:17.357523", + "started_at": null, + "instance": "p5.8xlarge" + }, { + "job_id": "5685a699-4927-479e-9b34-e5ab1616303a", + "solver": "fenics:1.0.4", + "status": "WAITING FOR RESOURCES", + "progress": 2, + "submitted_at": "2024-11-27 14:03:17.413844", + "started_at": "2024-11-27 16:03:17.413844", + "instance": "p5.8xlarge" + }, { + "job_id": "a8ca6a02-8816-48a6-8c6b-b94b3e431c8c", + "solver": "isolve:2.4.12", + "status": "WAITING FOR CLUSTER", + "progress": 0, + "submitted_at": "2024-11-27 14:03:17.357523", + "started_at": null, + "instance": "p5.8xlarge" + }, { + "job_id": "a8ca6a02-8816-48a6-8c6b-b94b3e431c8d", + "solver": "sleeper:2.1.2", + "status": "STARTED", + "progress": 50, + "submitted_at": "2024-11-27 14:03:17.357523", + "started_at": "2025-03-19 14:03:17.357523", + "instance": "p5.8xlarge" + }], + "jobs_info": { + "a8ca6a02-8816-48a6-8c6b-b94b3e431c8d": { + "parent_project_id" : "522f544a-1e05-461e-82ed-e26806e4640d", + "parent_project": "My Project", + "job_id": "a8ca6a02-8816-48a6-8c6b-b94b3e431c8d", + "job_name": "EM", + "log_state": 3, + "solver_key": "simcore/services/comp/sleeper", + "solver_version": "2.1.2", + "version_display" : "The latest", + "input_file_path": "/home/user/blah_blah_a8ca6a02-8816-48a6-8c6b-b94b3e431c8d_Input.h5", + "internal_job_id": "19b883a4-8407-4b81-9c5b-abe5a7f42bc8" + }, + "5685a699-4927-479e-9b34-e5ab1616303a": { + "parent_project_id" : "522f544a-1e05-461e-82ed-e26806e4640d", + "parent_project": "My Project", + "job_id": "5685a699-4927-479e-9b34-e5ab1616303a", + "job_name": "EM", + "log_state": 3, + "solver_key": "simcore/services/comp/sleeper", + "solver_version": "2.1.2", + "version_display" : "The latest", + "input_file_path": "/home/user/blah_blah_5685a699-4927-479e-9b34-e5ab1616303a_Input.h5", + "internal_job_id": "19b883a4-8407-4b81-9c5b-abe5a7f42bc8" + }, + "a8ca6a02-8816-48a6-8c6b-b94b3e431c8c": { + "parent_project_id" : "522f544a-1e05-461e-82ed-e26806e4640d", + "parent_project": "My Project", + "job_id": "a8ca6a02-8816-48a6-8c6b-b94b3e431c8c", + "job_name": "EM", + "log_state": 3, + "solver_key": "simcore/services/comp/sleeper", + "solver_version": "2.1.2", + "version_display" : "The latest", + "input_file_path": "/home/user/blah_blah_a8ca6a02-8816-48a6-8c6b-b94b3e431c8c_Input.h5", + "internal_job_id": "19b883a4-8407-4b81-9c5b-abe5a7f42bc8" + }, + "0cde4607-07de-4bb5-b3e6-487f22387a70": { + "parent_project_id" : "522f544a-1e05-461e-82ed-e26806e4640d", + "parent_project": "My Project", + "job_id": "0cde4607-07de-4bb5-b3e6-487f22387a70", + "job_name": "EM", + "log_state": 3, + "solver_key": "simcore/services/comp/sleeper", + "solver_version": "2.1.2", + "version_display" : "The latest", + "input_file_path": "/home/user/blah_blah_0cde4607-07de-4bb5-b3e6-487f22387a70_Input.h5", + "internal_job_id": "19b883a4-8407-4b81-9c5b-abe5a7f42bc8" + } + } +} diff --git a/services/static-webserver/client/source/resource/osparc/new_studies.json b/services/static-webserver/client/source/resource/osparc/new_studies.json deleted file mode 100644 index ed29e1145d5..00000000000 --- a/services/static-webserver/client/source/resource/osparc/new_studies.json +++ /dev/null @@ -1,210 +0,0 @@ -{ - "tis": { - "categories": [{ - "id": "precomputed", - "title": "Precomputed" - }, { - "id": "personalized", - "title": "Personalized", - "description": "In the process, TIP will launch simulations on AWS.
The associated resource costs will be deduced from your Credits." - }], - "resources": [{ - "resourceType": "template", - "expectedTemplateLabel": "TI Planning Tool", - "title": "Classic TI", - "newStudyLabel": "Classic TI", - "category": "precomputed", - "idToWidget": "newTIPlanButton" - }, { - "resourceType": "template", - "expectedTemplateLabel": "mcTI Planning Tool", - "title": "Multichannel TI", - "newStudyLabel": "Multichannel TI", - "category": "precomputed", - "idToWidget": "newMTIPlanButton" - }, { - "resourceType": "template", - "expectedTemplateLabel": "pmTI Planning Tool", - "title": "Phase-modulation TI", - "newStudyLabel": "Phase-modulation TI", - "category": "precomputed", - "idToWidget": "newPMTIPlanButton" - }, { - "resourceType": "template", - "expectedTemplateLabel": "personalized TI Planning Tool", - "title": "Personalized Classic TI", - "newStudyLabel": "Personalized Classic TI", - "category": "personalized", - "idToWidget": "personalizationNewTIPlanButton" - }, { - "resourceType": "template", - "expectedTemplateLabel": "personalized mcTI Planning Tool", - "title": "Personalized Multichannel TI", - "newStudyLabel": "Personalized Multichannel TI", - "category": "personalized", - "idToWidget": "personalizationNewMTIPlanButton" - }, { - "resourceType": "template", - "expectedTemplateLabel": "personalized pmTI Planning Tool", - "title": "Personalized Phase-modulation TI", - "newStudyLabel": "Personalized Phase-modulation TI", - "category": "personalized", - "idToWidget": "personalizationNewPMTIPlanButton" - }] - }, - "tiplite": { - "categories": [{ - "id": "precomputed", - "title": "Precomputed" - }, { - "id": "personalized", - "title": "Personalized", - "description": "In the process, TIP will launch simulations on AWS.
The associated resource costs will be deduced from your Credits." - }], - "resources": [{ - "resourceType": "template", - "expectedTemplateLabel": "TI Planning Tool", - "title": "Classic TI", - "newStudyLabel": "Classic TI", - "category": "precomputed", - "idToWidget": "newTIPlanButton" - }, { - "showDisabled": true, - "reason": "Not available in ${replace_me_product_name}", - "resourceType": "template", - "expectedTemplateLabel": "mcTI Planning Tool", - "title": "Multichannel TI", - "newStudyLabel": "Multichannel TI", - "category": "precomputed", - "idToWidget": "newMTIPlanButton" - }, { - "showDisabled": true, - "reason": "Not available in ${replace_me_product_name}", - "resourceType": "template", - "expectedTemplateLabel": "pmTI Planning Tool", - "title": "Phase-modulation TI", - "newStudyLabel": "Phase-modulation TI", - "category": "precomputed", - "idToWidget": "newPMTIPlanButton" - }, { - "showDisabled": true, - "reason": "Not available in ${replace_me_product_name}", - "resourceType": "template", - "expectedTemplateLabel": "personalized TI Planning Tool", - "title": "Personalized Classic TI", - "newStudyLabel": "Personalized Classic TI", - "category": "personalized", - "idToWidget": "personalizationNewTIPlanButton" - }, { - "showDisabled": true, - "reason": "Not available in ${replace_me_product_name}", - "resourceType": "template", - "expectedTemplateLabel": "personalized mcTI Planning Tool", - "title": "Personalized Multichannel TI", - "newStudyLabel": "Personalized Multichannel TI", - "category": "personalized", - "idToWidget": "personalizationNewMTIPlanButton" - }, { - "showDisabled": true, - "reason": "Not available in ${replace_me_product_name}", - "resourceType": "template", - "expectedTemplateLabel": "personalized pmTI Planning Tool", - "title": "Personalized Phase-modulation TI", - "newStudyLabel": "Personalized Phase-modulation TI", - "category": "personalized", - "idToWidget": "personalizationNewPMTIPlanButton" - }] - }, - "s4l": { - "categories": [{ - "id": "apps", - "title": "Apps" - }, { - "id": "osparc", - "title": "oSPARC" - }], - "resources": [{ - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/s4l-ui", - "title": "Sim4Life", - "newStudyLabel": "New S4L project", - "idToWidget": "startS4LButton" - }, { - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/iseg-web", - "title": "iSEG", - "newStudyLabel": "New iSEG project" - }, { - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/s4l-jupyter", - "title": "Jupyter Lab", - "icon": "https://upload.wikimedia.org/wikipedia/commons/3/38/Jupyter_logo.svg", - "newStudyLabel": "New S4L Jupyter Lab" - }, { - "category": "osparc", - "resourceType": "study", - "icon": "@FontAwesome5Solid/file/18", - "title": "Empty Pipeline", - "newStudyLabel": "New Project", - "idToWidget": "emptyStudyBtn" - }] - }, - "s4lacad": { - "categories": [{ - "id": "apps", - "title": "Apps" - }, { - "id": "osparc", - "title": "oSPARC" - }], - "resources": [{ - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/s4l-ui", - "title": "Sim4Life", - "newStudyLabel": "New S4L project", - "idToWidget": "startS4LButton" - }, { - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/iseg-web", - "title": "iSEG", - "newStudyLabel": "New iSEG project" - }, { - "category": "apps", - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/s4l-jupyter", - "icon": "https://upload.wikimedia.org/wikipedia/commons/3/38/Jupyter_logo.svg", - "title": "Jupyter Lab", - "newStudyLabel": "New S4L Jupyter Lab" - }, { - "category": "osparc", - "resourceType": "study", - "icon": "@FontAwesome5Solid/file/18", - "title": "Empty Pipeline", - "newStudyLabel": "New Project", - "idToWidget": "emptyStudyBtn" - }] - }, - "s4llite": { - "resources": [{ - "resourceType": "service", - "expectedKey": "simcore/services/dynamic/s4l-ui-lite", - "title": "${replace_me_product_name}", - "newStudyLabel": "New Project", - "idToWidget": "startS4LButton" - }] - }, - "osparc": { - "resources": [{ - "resourceType": "study", - "icon": "@FontAwesome5Solid/file/18", - "title": "Empty Study", - "newStudyLabel": "New Study", - "idToWidget": "emptyStudyBtn" - }] - } -} diff --git a/services/static-webserver/client/source/resource/osparc/tours/osparc_tours.json b/services/static-webserver/client/source/resource/osparc/tours/osparc_tours.json new file mode 100644 index 00000000000..24ff1e63ef1 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/tours/osparc_tours.json @@ -0,0 +1,108 @@ +{ + "studies": { + "id": "studies", + "name": "Studies", + "description": "All you need to know about Study handling", + "context": "osparc-test-id=newPlusBtn", + "steps": [{ + "beforeClick": { + "selector": "osparc-test-id=newPlusBtn", + "action": "open" + }, + "anchorEl": "osparc-test-id=newPlusMenu", + "title": "Create Studies", + "text": "Clicking on the (+) New button, allows you to create new Studies or new Folders to organize the studies", + "placement": "right" + }, { + "anchorEl": "osparc-test-id=searchBarFilter-textField-study", + "title": "Filter and Search", + "text": "This tool allows you to search Studies, Tutorials and Services.
You can search and filter by:
- Title, description, owner, id...
- Tags
- Shared with", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=studyItemMenuButton", + "action": "open" + }, + "anchorEl": "osparc-test-id=studyItemMenuMenu", + "title": "More options button", + "text": "On the Study card, you can use the three dots button to access more information and operation on the Study.", + "placement": "left" + }, { + "anchorEl": "osparc-test-id=updateStudyBtn", + "title": "Update Services", + "text": "On the Study card, you can use the Update button to update the corresponding service to the latest version.", + "placement": "bottom" + }] + }, + "dashboard": { + "id": "dashboard", + "name": "Dashboard", + "description": "Introduction to the Dashboard tabs", + "context": "osparc-test-id=dashboardTabs", + "steps": [{ + "anchorEl": "osparc-test-id=dashboardTabs", + "title": "Dashboard Menu", + "text": "The menu tabs give you quick access to a set of core elements of the platform, namely Studies, Templates, Services and Data.", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=studiesTabBtn" + }, + "anchorEl": "osparc-test-id=studiesTabBtn", + "text": "Any Study is accessible via the Dashboard. The Studies, which belong to or are shared with you, can be found here. You can also create Folders to help you organize the Studies", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=templatesTabBtn" + }, + "anchorEl": "osparc-test-id=templatesTabBtn", + "text": "Clicking on a Template will create a copy of that Study, which will appear in your own Studies tab with the same name as the Template. Any changes you make to this copy will not affect the original Template.", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=servicesTabBtn" + }, + "anchorEl": "osparc-test-id=servicesTabBtn", + "text": "Every Study in oSparc is composed of so-called Services.
These are building blocks for Studies and can provide data/files, visualize results (2D, 3D), implement code in Jupyter notebooks or perform computations to execute simulations within a Study.", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=dataTabBtn" + }, + "anchorEl": "osparc-test-id=dataTabBtn", + "text": "All the Data of the Studies you have access to can bre explored here.", + "placement": "bottom" + }] + }, + "navbar": { + "id": "navbar", + "name": "Navigation Bar", + "description": "Introduction to the Navigation Bar", + "context": "osparc-test-id=navigationBar", + "steps": [{ + "beforeClick": { + "selector": "osparc-test-id=notificationsButton", + "event": "tap" + }, + "anchorEl": "osparc-test-id=notificationsContainer", + "text": "By clicking on the Bell, you will you see notifications about which Studies, Templates and Organizations have been shared with you.", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=helpNavigationBtn", + "action": "open" + }, + "anchorEl": "osparc-test-id=helpNavigationMenu", + "text": "Under the question mark, you will find Manuals, Support and ways to give us Feedback. It also provides quick access to other Guided Tours.", + "placement": "left" + }, { + "beforeClick": { + "selector": "osparc-test-id=userMenuBtn", + "action": "open" + }, + "anchorEl": "osparc-test-id=userMenuMenu", + "text": "The User Menu gives you access to Your Account, Organizations and more.", + "placement": "left" + }] + } +} diff --git a/services/static-webserver/client/source/resource/osparc/tours/s4l_tours.json b/services/static-webserver/client/source/resource/osparc/tours/s4l_tours.json index 492544fa598..0e5f056a68c 100644 --- a/services/static-webserver/client/source/resource/osparc/tours/s4l_tours.json +++ b/services/static-webserver/client/source/resource/osparc/tours/s4l_tours.json @@ -1,4 +1,39 @@ { + "projects": { + "id": "projects", + "name": "Projects", + "description": "All you need to know about Project handling", + "context": "osparc-test-id=newPlusBtn", + "steps": [{ + "beforeClick": { + "selector": "osparc-test-id=newPlusBtn", + "action": "open" + }, + "anchorEl": "osparc-test-id=newPlusMenu", + "title": "Start Sim4Life and more", + "text": "Clicking on the (+) New button, allows you to create new Sim4Life projects or new Folders to organize the projects", + "placement": "right" + }, { + "anchorEl": "osparc-test-id=searchBarFilter-textField-study", + "title": "Filter and Search", + "text": "This tool allows you to search Projects, Tutorials and Services.
You can search and filter by:
- Title, description, owner, id...
- Tags
- Shared with", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=studyItemMenuButton", + "action": "open" + }, + "anchorEl": "osparc-test-id=studyItemMenuMenu", + "title": "More options button", + "text": "On the Project card, you can use the three dots button to access more information and operation on the Project.", + "placement": "left" + }, { + "anchorEl": "osparc-test-id=updateStudyBtn", + "title": "Update Services", + "text": "On the Project card, you can use the Update button to update the corresponding service to the latest version.", + "placement": "bottom" + }] + }, "dashboard": { "id": "dashboard", "name": "Dashboard", @@ -28,7 +63,7 @@ "selector": "osparc-test-id=servicesTabBtn" }, "anchorEl": "osparc-test-id=servicesTabBtn", - "text": "Every Project in Sim4Life is composed of at lease one so-called Service.
Services are building blocks for Projects and can provide data/files, visualize results (2D, 3D), implement code in Jupyter notebooks or perform computations to execute simulations within a Project.", + "text": "Every Project in Sim4Life is composed of at least one so-called Service.
Services are building blocks for Projects and can provide data/files, visualize results (2D, 3D), implement code in Jupyter notebooks or perform computations to execute simulations within a Project.", "placement": "bottom" }] }, @@ -62,39 +97,5 @@ "text": "The User Menu gives you access to Your Account, Billing Center, Preferences, Organizations and more.", "placement": "left" }] - }, - "projects": { - "id": "projects", - "name": "Projects", - "description": "All you need to know about Project handling", - "context": "osparc-test-id=studiesTabBtn", - "steps": [{ - "beforeClick": { - "selector": "osparc-test-id=studiesTabBtn" - }, - "anchorEl": "osparc-test-id=startS4LButton", - "title": "Start Sim4Life", - "text": "Clicking on this (+) Start Sim4Life button, allows you to create and open a new Sim4Life project", - "placement": "right" - }, { - "anchorEl": "osparc-test-id=searchBarFilter-textField-study", - "title": "Filter and Search", - "text": "This tool allows you to filter Projects, Tutorials and Services.
You can search and filter by:
- Title, description, owner, id...
- Tags
- Shared with", - "placement": "bottom" - }, { - "beforeClick": { - "selector": "osparc-test-id=studyItemMenuButton", - "action": "open" - }, - "anchorEl": "osparc-test-id=studyItemMenuMenu", - "title": "More options button", - "text": "On the Project card, you can use the three dots button to access more information and operation on the Project.", - "placement": "left" - }, { - "anchorEl": "osparc-test-id=updateStudyBtn", - "title": "Update Services", - "text": "On the Project card, you can use the Update button to update the corresponding service to the latest version.", - "placement": "bottom" - }] } } diff --git a/services/static-webserver/client/source/resource/osparc/tours/s4llite_tours.json b/services/static-webserver/client/source/resource/osparc/tours/s4llite_tours.json index e1e509a6f83..7dfbd752dbc 100644 --- a/services/static-webserver/client/source/resource/osparc/tours/s4llite_tours.json +++ b/services/static-webserver/client/source/resource/osparc/tours/s4llite_tours.json @@ -1,4 +1,39 @@ { + "projects": { + "id": "projects", + "name": "Projects", + "description": "All you need to know about Project handling", + "context": "osparc-test-id=newPlusBtn", + "steps": [{ + "beforeClick": { + "selector": "osparc-test-id=newPlusBtn", + "action": "open" + }, + "anchorEl": "osparc-test-id=newPlusMenu", + "title": "Start Sim4Life.lite", + "text": "Clicking on the (+) New button, allows you to create new Sim4Life.lite projects or new Folders to organize the projects", + "placement": "right" + }, { + "anchorEl": "osparc-test-id=searchBarFilter-textField-study", + "title": "Filter and Search", + "text": "This tool allows you to filter Projects and Tutorials.
You can search and filter by:
- Title, description, owner, id...
- Tags
- Shared with", + "placement": "bottom" + }, { + "beforeClick": { + "selector": "osparc-test-id=studyItemMenuButton", + "action": "open" + }, + "anchorEl": "osparc-test-id=studyItemMenuMenu", + "title": "More options button", + "text": "On the Project card, you can use the three dots button to access more information and operation on the Project.", + "placement": "left" + }, { + "anchorEl": "osparc-test-id=updateStudyBtn", + "title": "Update Services", + "text": "On the Project card, you can use the Update button to update the corresponding service to the latest version.", + "placement": "bottom" + }] + }, "dashboard": { "id": "dashboard", "name": "Dashboard", @@ -52,39 +87,5 @@ "text": "The User Menu gives you access to Your Account, Preferences, Organizations and more.", "placement": "left" }] - }, - "projects": { - "id": "projects", - "name": "Projects", - "description": "All you need to know about Project handling", - "context": "osparc-test-id=studiesTabBtn", - "steps": [{ - "beforeClick": { - "selector": "osparc-test-id=studiesTabBtn" - }, - "anchorEl": "osparc-test-id=startS4LButton", - "title": "Start Sim4Life.lite", - "text": "Clicking on this (+) Start Sim4Life.lite button, allows you to create and open a new Sim4Life.lite project", - "placement": "right" - }, { - "anchorEl": "osparc-test-id=searchBarFilter-textField-study", - "title": "Filter and Search", - "text": "This tool allows you to filter Projects and Tutorials.
You can search and filter by:
- Title, description, owner, id...
- Tags
- Shared with", - "placement": "bottom" - }, { - "beforeClick": { - "selector": "osparc-test-id=studyItemMenuButton", - "action": "open" - }, - "anchorEl": "osparc-test-id=studyItemMenuMenu", - "title": "More options button", - "text": "On the Project card, you can use the three dots button to access more information and operation on the Project.", - "placement": "left" - }, { - "anchorEl": "osparc-test-id=updateStudyBtn", - "title": "Update Services", - "text": "On the Project card, you can use the Update button to update the corresponding service to the latest version.", - "placement": "bottom" - }] } } diff --git a/services/static-webserver/client/source/resource/osparc/trash-text.svg b/services/static-webserver/client/source/resource/osparc/trash-text.svg new file mode 100644 index 00000000000..0cbfe135be5 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/trash-text.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/static-webserver/client/source/resource/osparc/ui_config.json b/services/static-webserver/client/source/resource/osparc/ui_config.json new file mode 100644 index 00000000000..826f64cf9a9 --- /dev/null +++ b/services/static-webserver/client/source/resource/osparc/ui_config.json @@ -0,0 +1,166 @@ +{ + "tis": { + "newStudies": { + "categories": [{ + "id": "precomputed", + "title": "Precomputed" + }, { + "id": "personalized", + "title": "Personalized", + "description": "In the process, TIP will launch simulations on AWS.
The associated resource costs will be deduced from your Credits." + }], + "resources": [{ + "resourceType": "template", + "expectedTemplateLabel": "TI Planning Tool", + "title": "Classic TI", + "newStudyLabel": "Classic TI", + "category": "precomputed", + "idToWidget": "newTIPlanButton" + }, { + "resourceType": "template", + "expectedTemplateLabel": "mcTI Planning Tool", + "title": "Multichannel TI", + "newStudyLabel": "Multichannel TI", + "category": "precomputed" + }, { + "resourceType": "template", + "expectedTemplateLabel": "pmTI Planning Tool", + "title": "Phase-modulation TI", + "newStudyLabel": "Phase-modulation TI", + "category": "precomputed" + }, { + "resourceType": "template", + "expectedTemplateLabel": "personalized TI Planning Tool", + "title": "Personalized
Classic TI", + "newStudyLabel": "Personalized Classic TI", + "category": "personalized", + "idToWidget": "newPTIPlanButton" + }, { + "resourceType": "template", + "expectedTemplateLabel": "personalized mcTI Planning Tool", + "title": "Personalized
Multichannel TI", + "newStudyLabel": "Personalized Multichannel TI", + "category": "personalized" + }, { + "resourceType": "template", + "expectedTemplateLabel": "personalized pmTI Planning Tool", + "title": "Personalized
Phase-modulation TI", + "newStudyLabel": "Personalized Phase-modulation TI", + "category": "personalized" + }] + } + }, + "tiplite": { + "newStudies": { + "categories": [{ + "id": "precomputed", + "title": "Precomputed" + }, { + "id": "personalized", + "title": "Personalized", + "description": "In the process, TIP will launch simulations on AWS.
The associated resource costs will be deduced from your Credits." + }], + "resources": [{ + "resourceType": "template", + "expectedTemplateLabel": "TI Planning Tool", + "title": "Classic TI", + "newStudyLabel": "Classic TI", + "category": "precomputed", + "idToWidget": "newTIPlanButton" + }, { + "showDisabled": true, + "reason": "Not available in ${replace_me_product_name}", + "title": "Multichannel TI", + "category": "precomputed" + }, { + "showDisabled": true, + "reason": "Not available in ${replace_me_product_name}", + "title": "Phase-modulation TI", + "category": "precomputed" + }, { + "showDisabled": true, + "reason": "Not available in ${replace_me_product_name}", + "title": "Personalized
Classic TI", + "category": "personalized", + "idToWidget": "newPTIPlanButton" + }, { + "showDisabled": true, + "reason": "Not available in ${replace_me_product_name}", + "title": "Personalized
Multichannel TI", + "category": "personalized" + }, { + "showDisabled": true, + "reason": "Not available in ${replace_me_product_name}", + "title": "Personalized
Phase-modulation TI", + "category": "personalized" + }] + } + }, + "s4l": { + "plusButton": { + "resources": [{ + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/s4l-ui", + "title": "Sim4Life", + "newStudyLabel": "New Sim4Life", + "idToWidget": "startS4LButton" + }, { + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/s4l-jupyter", + "title": "Jupyter Lab", + "newStudyLabel": "New S4L Jupyter Lab" + }, { + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/iseg-web", + "title": "Segmentation", + "newStudyLabel": "Segmentation" + }] + } + }, + "s4lacad": { + "plusButton": { + "resources": [{ + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/s4l-ui", + "title": "Sim4Life", + "newStudyLabel": "New Sim4Life", + "idToWidget": "startS4LButton" + }, { + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/s4l-jupyter", + "title": "Jupyter Lab", + "newStudyLabel": "New S4L Jupyter Lab" + }, { + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/iseg-web", + "title": "Segmentation", + "newStudyLabel": "Segmentation" + }] + } + }, + "s4llite": { + "plusButton": { + "resources": [{ + "resourceType": "service", + "expectedKey": "simcore/services/dynamic/s4l-ui-lite", + "title": "${replace_me_product_name}", + "newStudyLabel": "New Project", + "idToWidget": "startS4LButton" + }] + } + }, + "osparc": { + "plusButton": { + "resources": [{ + "resourceType": "study", + "icon": "@FontAwesome5Solid/file/18", + "title": "Empty Study", + "newStudyLabel": "New Study", + "idToWidget": "emptyStudyBtn" + }, { + "resourceType": "service", + "myMostUsed": 2 + }] + } + } +} diff --git a/services/static-webserver/client/source/resource/schemas/product-ui.json b/services/static-webserver/client/source/resource/schemas/product-ui.json new file mode 100644 index 00000000000..a1454b8e6dd --- /dev/null +++ b/services/static-webserver/client/source/resource/schemas/product-ui.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "plusButton": { + "$ref": "#/definitions/buttonConfig" + }, + "newStudies": { + "$ref": "#/definitions/buttonConfig" + } + }, + "additionalProperties": false, + "definitions": { + "buttonConfig": { + "type": "object", + "properties": { + "categories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "description": { "type": "string" } + }, + "required": ["id", "title"] + } + }, + "resources": { + "type": "array", + "items": { + "oneOf": [{ + "type": "object", + "properties": { + "resourceType": { "enum": ["study"] }, + "title": { "type": "string" }, + "icon": { "type": "string" }, + "newStudyLabel": { "type": "string" }, + "idToWidget": { "type": "string" } + }, + "required": ["resourceType", "title"] + }, { + "type": "object", + "properties": { + "resourceType": { "enum": ["template"] }, + "expectedTemplateLabel": { "type": "string" }, + "title": { "type": "string" }, + "icon": { "type": "string" }, + "newStudyLabel": { "type": "string" }, + "category": { "type": "string" }, + "idToWidget": { "type": "string" } + }, + "required": ["resourceType", "expectedTemplateLabel", "title"] + }, { + "type": "object", + "properties": { + "resourceType": { "enum": ["service"] }, + "expectedKey": { "type": "string" }, + "title": { "type": "string" }, + "icon": { "type": "string" }, + "newStudyLabel": { "type": "string" }, + "category": { "type": "string" }, + "idToWidget": { "type": "string" } + }, + "required": ["resourceType", "expectedKey", "title"] + }, { + "type": "object", + "properties": { + "resourceType": { "enum": ["service"] }, + "myMostUsed": { "type": "integer" }, + "category": { "type": "string" } + }, + "required": ["resourceType", "myMostUsed"] + }, { + "type": "object", + "properties": { + "showDisabled": { + "type": "boolean", + "enum": [true] + }, + "title": { "type": "string" }, + "icon": { "type": "string" }, + "reason": { "type": "string" }, + "newStudyLabel": { "type": "string" }, + "category": { "type": "string" }, + "idToWidget": { "type": "string" } + }, + "required": ["showDisabled", "title"] + }] + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + } +} diff --git a/services/static-webserver/client/source/translation/en.po b/services/static-webserver/client/source/translation/en.po new file mode 100644 index 00000000000..cdb1069d242 --- /dev/null +++ b/services/static-webserver/client/source/translation/en.po @@ -0,0 +1,4411 @@ +#: osparc/Application.js +msgid "You have been logged out" +msgstr "" + +#: osparc/Application.js +msgid "New Version Released" +msgstr "" + +#: osparc/Application.js +msgid "Privacy Policy" +msgstr "" + +#: osparc/Application.js +msgid "Privacy Policy and License Terms" +msgstr "" + +#: osparc/Application.js +msgid "You have been logged out" +msgstr "" + +#: osparc/desktop/credits/ResourceInTableViewer.js +msgid "Reload" +msgstr "" + +#: osparc/desktop/credits/UsageTable.js +msgid "Service" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Copy to clipboard" +msgstr "" + +#: osparc/utils/Utils.js +msgid "Today" +msgstr "" + +#: osparc/utils/Utils.js +msgid "Yesterday" +msgstr "" + +#: osparc/utils/Utils.js +msgid "Tomorrow" +msgstr "" + +#: osparc/utils/Utils.js +msgid "Your account will expire today." +msgstr "" + +#: osparc/utils/Utils.js +msgid "Your account will expire tomorrow." +msgstr "" + +#: osparc/utils/Utils.js +msgid "Your account will expire in " +msgstr "" + +#: osparc/utils/Utils.js +msgid " days." +msgstr "" + +#: osparc/utils/Utils.js +msgid "Please contact us via email:" +msgstr "" + +#: osparc/utils/Utils.js +msgid "To use all " +msgstr "" + +#: osparc/utils/Utils.js +msgid ", please send us an e-mail to create an account:" +msgstr "" + +#: osparc/utils/Utils.js +msgid "Copied to clipboard" +msgstr "" + +#: osparc/WindowSizeTracker.js +msgid "This app performs better for larger window size: " +msgstr "" + +#: osparc/WindowSizeTracker.js +msgid "Touchscreen devices are not supported yet." +msgstr "" + +#: osparc/auth/Manager.js +msgid "Unsuccessful Login" +msgstr "" + +#: osparc/ErrorPage.js +msgid "Support email" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "Log in" +msgstr "" + +#: osparc/store/Support.js +msgid "Please send us an email to:" +msgstr "" + +#: osparc/product/Utils.js +msgid "projects" +msgstr "" + +#: osparc/product/Utils.js +msgid "project" +msgstr "" + +#: osparc/product/Utils.js +msgid "studies" +msgstr "" + +#: osparc/product/Utils.js +msgid "study" +msgstr "" + +#: osparc/product/Utils.js +msgid "tutorials" +msgstr "" + +#: osparc/product/Utils.js +msgid "tutorial" +msgstr "" + +#: osparc/product/Utils.js +msgid "templates" +msgstr "" + +#: osparc/product/Utils.js +msgid "template" +msgstr "" + +#: osparc/product/Utils.js +msgid "service" +msgstr "" + +#: osparc/product/Utils.js +msgid "services" +msgstr "" + +#: osparc/NewRelease.js +msgid "We are pleased to announce that some new features were deployed for you!" +msgstr "" + +#: osparc/NewRelease.js +msgid "What's new in " +msgstr "" + +#: osparc/CookiePolicy.js +msgid "This website applies cookies to personalize your experience and to make our site easier to navigate. By visiting the site, you agree to the " +msgstr "" + +#: osparc/CookiePolicy.js +msgid "By visiting the site, you agree to the " +msgstr "" + +#: osparc/CookiePolicy.js +msgid "It also uses third party software and libraries. By visiting the site, you agree to the " +msgstr "" + +#: osparc/CookiePolicy.js +msgid "Licensing." +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Accept" +msgstr "" + +#: osparc/data/Resources.js +msgid "You have been logged out. Your cookie might have expired." +msgstr "" + +#: osparc/FlashMessenger.js +msgid "Oops... something went wrong" +msgstr "" + +#: osparc/FlashMessenger.js +msgid "No message" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Do you want to close " +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Make sure you saved your changes to:" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "- current smash file" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "- current notebooks (jupyterlab session will be terminated)" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Close" +msgstr "" + +#: osparc/share/CollaboratorsStudy.js +msgid "Yes" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Do you want to save and close " +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Save & Close" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Platform logger" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Started template creation and added to the background tasks" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while duplicating the study
" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Loading Snapshot" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Closing previous snapshot..." +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "No snapshot found" +msgstr "" + +#: osparc/desktop/MainPageHandler.js +msgid "Study not found" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Loading Iteration" +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Closing..." +msgstr "" + +#: osparc/desktop/MainPage.js +msgid "Iteration not found" +msgstr "" + +#: osparc/MaintenanceTracker.js +msgid "The service is under maintenance. Please check back later" +msgstr "" + +#: osparc/CookieExpirationTracker.js +msgid "Your session has expired" +msgstr "" + +#: osparc/NewUITracker.js +msgid "A new version of the application is now available." +msgstr "" + +#: osparc/NewUITracker.js +msgid "Click the Reload button to get the latest features." +msgstr "" + +#: osparc/ui/switch/ThemeSwitcher.js +msgid "Dark theme" +msgstr "" + +#: osparc/ui/switch/ThemeSwitcher.js +msgid "Light theme" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Ten Simple Rules" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "TSR:" +msgstr "" + +#: osparc/share/NewCollaboratorsManager.js +msgid "Share" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "App mode" +msgstr "" + +#: osparc/service/ServiceListItem.js +msgid "Hits: " +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Unaccessible service(s):" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Service(s) retired, please update" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Service(s) deprecated, please update" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Update available" +msgstr "" + +#: osparc/navigation/PrevNextButtons.js +msgid "Running" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Ran successfully" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Run aborted" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Unsuccessful Run" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "A user" +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid " is closing it..." +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid " is cloning it..." +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid " is exporting it..." +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid " is opening it..." +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid " is using it." +msgstr "" + +#: osparc/dashboard/CardBase.js +msgid "Embargoed
Credits Required" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Empty" +msgstr "" + +#: osparc/product/quickStart/s4llite/Slides.js +msgid "Quick Start" +msgstr "" + +#: osparc/tours/Manager.js +msgid "Guided Tours" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Information" +msgstr "" + +#: osparc/store/Support.js +msgid "To create an issue, you must have an account and be already logged-in." +msgstr "" + +#: osparc/store/Support.js +msgid "Continue" +msgstr "" + +#: osparc/store/Support.js +msgid "Log in in " +msgstr "" + +#: osparc/store/Support.js +msgid "Release Notes" +msgstr "" + +#: osparc/store/Support.js +msgid "Registration is currently only available with an invitation." +msgstr "" + +#: osparc/store/Support.js +msgid "Please request access to " +msgstr "" + +#: osparc/store/Data.js +msgid "Oops... more than 10.000 items to be listed here. Maybe it's time to make a folder :)." +msgstr "" + +#: osparc/store/Data.js +msgid "Unsuccessful file copy" +msgstr "" + +#: osparc/store/Data.js +msgid "Unsuccessful file deletion" +msgstr "" + +#: osparc/notification/RibbonNotification.js +msgid "Maintenance scheduled." +msgstr "" + +#: osparc/notification/RibbonNotification.js +msgid "Please save your work and logout." +msgstr "" + +#: osparc/notification/RibbonNotification.js +msgid "Oops, your window is a bit small!" +msgstr "" + +#: osparc/product/quickStart/Utils.js +msgid "Don't show again" +msgstr "" + +#: osparc/TooSmallDialog.js +msgid "Window size too small" +msgstr "" + +#: osparc/TooSmallDialog.js +msgid "The application can't perform in such a small window." +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "Exit" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "Log out" +msgstr "" + +#: osparc/desktop/credits/Utils.js +msgid "You can't access this information" +msgstr "" + +#: osparc/desktop/credits/Utils.js +msgid "You can't access these operations" +msgstr "" + +#: osparc/share/Collaborators.js +msgid "Any logged-in user with access to the " +msgstr "" + +#: osparc/share/Collaborators.js +msgid " can open it" +msgstr "" + +#: osparc/share/Collaborators.js +msgid "Copy link" +msgstr "" + +#: osparc/share/Collaborators.js +msgid "Anyone on the internet with the link can open this " +msgstr "" + +#: osparc/share/Collaborators.js +msgid "Any logged-in user with the link can copy and open this " +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "Shared with" +msgstr "" + +#: osparc/share/Collaborators.js +msgid "Collaborator can't be removed:" +msgstr "" + +#: osparc/share/Collaborators.js +msgid " needs at least one owner." +msgstr "" + +#: osparc/share/Collaborators.js +msgid "You might want to delete it instead." +msgstr "" + +#: osparc/ui/list/MemberListItem.js +msgid "Leave" +msgstr "" + +#: osparc/auth/core/Utils.js +msgid "Passwords do not match" +msgstr "" + +#: osparc/auth/core/Utils.js +msgid "Invalid phone number. Please [+][country code][phone number]" +msgstr "" + +#: osparc/auth/core/Utils.js +msgid "Invalid email address.
Please register using your university email address" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Cancel" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Submit" +msgstr "" + +#: osparc/form/json/JsonSchemaForm.js +msgid "There was an issue generating the form or one or more schemas failed to validate. Check your Javascript console for more details." +msgstr "" + +#: osparc/auth/LoginWithDecorators.js +msgid "Your account has been created.
You can now use your credentials to login." +msgstr "" + +#: osparc/navigation/NavigationBar.js +msgid "This is TIP.lite, a light version of TIP.
Request access to TIP." +msgstr "" + +#: osparc/product/quickStart/tis/Dashboard.js +msgid "Dashboard" +msgstr "" + +#: osparc/navigation/NavigationBar.js +msgid "Read only" +msgstr "" + +#: osparc/ui/window/Confirmation.js +msgid "Confirmation" +msgstr "" + +#: osparc/snapshots/Loading.js +msgid "Loading " +msgstr "" + +#: osparc/desktop/MainPageHandler.js +msgid "is already open by" +msgstr "" + +#: osparc/desktop/MainPageHandler.js +msgid "another user." +msgstr "" + +#: osparc/desktop/MainPageHandler.js +msgid "We encountered an issue with the" +msgstr "" + +#: osparc/desktop/MainPageHandler.js +msgid "Please contact support." +msgstr "" + +#: osparc/dashboard/Dashboard.js +msgid "SERVICES" +msgstr "" + +#: osparc/dashboard/Dashboard.js +msgid "DATA" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "You do not have writing permissions.
Your changes will not be saved." +msgstr "" + +#: osparc/data/model/IframeHandler.js +msgid "Starting" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "Opening " +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid " is already opened" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "Error opening study" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "The Study contains more than " +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid " Interactive services." +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "Please start them manually." +msgstr "" + +#: osparc/desktop/SlideshowToolbar.js +msgid "Edit App Mode" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "The pipeline is up-to-date. Do you want to re-run it?" +msgstr "" + +#: osparc/widget/PreparingInputs.js +msgid "Re-run" +msgstr "" + +#: osparc/desktop/StartStopButtons.js +msgid "Run" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "Take Snapshot" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Checkpoints" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Iterations" +msgstr "" + +#: osparc/desktop/StudyEditor.js +msgid "Error saving the study" +msgstr "" + +#: osparc/data/PollTask.js +msgid "Unsuccessful polling status" +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "My Workspace" +msgstr "" + +#: osparc/study/Utils.js +msgid "CREATING " +msgstr "" + +#: osparc/service/Utils.js +msgid "Parameter" +msgstr "" + +#: osparc/service/Utils.js +msgid "File" +msgstr "" + +#: osparc/service/Utils.js +msgid "Iterator" +msgstr "" + +#: osparc/service/Utils.js +msgid "Computational" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Interactive" +msgstr "" + +#: osparc/service/Utils.js +msgid "Probe" +msgstr "" + +#: osparc/service/Utils.js +msgid "Service deprecated" +msgstr "" + +#: osparc/service/Utils.js +msgid "Please go back to the dashboard and Update the Service or download its data and upload it to an updated version" +msgstr "" + +#: osparc/service/Utils.js +msgid "Please instantiate an updated version" +msgstr "" + +#: osparc/service/Utils.js +msgid "Service retired" +msgstr "" + +#: osparc/service/Utils.js +msgid "Please download the Service data and upload it to an updated version" +msgstr "" + +#: osparc/service/Utils.js +msgid "Please Stop the Service and then Update it" +msgstr "" + +#: osparc/service/Utils.js +msgid "Please Update the Service" +msgstr "" + +#: osparc/service/Utils.js +msgid "It will be Retired: " +msgstr "" + +#: osparc/ui/basic/NodeStatusUI.js +msgid "Idle" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Waiting for resources" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Unsuccessful" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Ready" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Deprecated" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Retired" +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Starting..." +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Stopping..." +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Pending..." +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Pulling..." +msgstr "" + +#: osparc/service/StatusUI.js +msgid "Connecting..." +msgstr "" + +#: osparc/task/Export.js +msgid "Exporting " +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Details" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Credits required" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid "Open" +msgstr "" + +#: osparc/dashboard/NewPlusButton.js +msgid "New" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Some of your services are outdated. Please update to the latest version for better performance.\\\\\\\\n\\\\\\\\nDo you want to update now?" +msgstr "" + +#: osparc/workbench/ServiceCatalog.js +msgid "Version" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid " Files..." +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Overview" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Billing Settings" +msgstr "" + +#: osparc/node/TierSelectionView.js +msgid "Tiers" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Pipeline View" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Comments" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Sharing" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Classifiers" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Quality" +msgstr "" + +#: osparc/desktop/credits/UsageTable.js +msgid "Tags" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Services Updates" +msgstr "" + +#: osparc/node/BootOptionsView.js +msgid "Boot Options" +msgstr "" + +#: osparc/dashboard/ResourceDetails.js +msgid "Publish " +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "You need to be logged in to create a study" +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "Group" +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "None" +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "Grid view" +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "List view" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Creating " +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Share..." +msgstr "" + +#: osparc/dashboard/ResourceBrowserBase.js +msgid "Tags..." +msgstr "" + +#: osparc/widget/StudyDataManager.js +msgid " Files" +msgstr "" + +#: osparc/tours/Manager.js +msgid "This collection of Guided Tours will show you how to use the platform:" +msgstr "" + +#: osparc/desktop/credits/BuyCreditsStepper.js +msgid "Payment " +msgstr "" + +#: osparc/share/CollaboratorsWorkspace.js +msgid "Workspace successfully shared" +msgstr "" + +#: osparc/share/CollaboratorsWorkspace.js +msgid "Something went wrong while sharing the workspace" +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid " successfully removed" +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Something went wrong while removing " +msgstr "" + +#: osparc/share/CollaboratorsStudy.js +msgid "Demote" +msgstr "" + +#: osparc/share/CollaboratorsTag.js +msgid "Tag successfully shared" +msgstr "" + +#: osparc/share/CollaboratorsTag.js +msgid "Something went wrong while sharing the tag" +msgstr "" + +#: osparc/data/Roles.js +msgid "Restricted Member" +msgstr "" + +#: osparc/data/Roles.js +msgid "Restricted member: no Read access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can access content shared within the Organization" +msgstr "" + +#: osparc/data/Roles.js +msgid "Member" +msgstr "" + +#: osparc/data/Roles.js +msgid "Member: Read access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can see other members" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can share with other members" +msgstr "" + +#: osparc/data/Roles.js +msgid "Manager" +msgstr "" + +#: osparc/data/Roles.js +msgid "Manager: Read/Write access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can Add/Delete members" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can Promote/Demote members" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can Edit Organization details" +msgstr "" + +#: osparc/data/Roles.js +msgid "Administrator" +msgstr "" + +#: osparc/data/Roles.js +msgid "Admin: Read/Write/Delete access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can Delete the Organization" +msgstr "" + +#: osparc/desktop/credits/CheckoutsTable.js +msgid "User" +msgstr "" + +#: osparc/data/Roles.js +msgid "User: Read access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can open it" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Editor" +msgstr "" + +#: osparc/data/Roles.js +msgid "Editor: Read/Write access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can make changes" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can share it" +msgstr "" + +#: osparc/dashboard/SortedByMenuButton.js +msgid "Owner" +msgstr "" + +#: osparc/data/Roles.js +msgid "Owner: Read/Write/Delete access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can delete it" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can use it" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can use the credits" +msgstr "" + +#: osparc/data/Roles.js +msgid "Accountant" +msgstr "" + +#: osparc/data/Roles.js +msgid "Accountant: Read/Write access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can Edit Credit Account details" +msgstr "" + +#: osparc/data/Roles.js +msgid "Viewer" +msgstr "" + +#: osparc/data/Roles.js +msgid "Viewer: Read access" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can inspect the content and open " +msgstr "" + +#: osparc/data/Roles.js +msgid "- can add " +msgstr "" + +#: osparc/data/Roles.js +msgid "- can add folders" +msgstr "" + +#: osparc/data/Roles.js +msgid "- can rename workspace" +msgstr "" + +#: osparc/data/Roles.js +msgid "Roles" +msgstr "" + +#: osparc/share/AddCollaborators.js +msgid "Share with..." +msgstr "" + +#: osparc/share/AddCollaborators.js +msgid "My Organizations..." +msgstr "" + +#: osparc/share/AddCollaborators.js +msgid "Publish for" +msgstr "" + +#: osparc/share/CollaboratorsStudy.js +msgid " successfully shared" +msgstr "" + +#: osparc/share/CollaboratorsStudy.js +msgid "Something went wrong while sharing the " +msgstr "" + +#: osparc/share/CollaboratorsStudy.js +msgid "Sharee permissions" +msgstr "" + +#: osparc/ui/form/FileInput.js +msgid "Select File..." +msgstr "" + +#: osparc/auth/ui/LoginView.js +msgid "Sign in" +msgstr "" + +#: osparc/auth/ui/LoginView.js +msgid "Create Account" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Request Account" +msgstr "" + +#: osparc/auth/ui/LoginView.js +msgid "Forgot Password?" +msgstr "" + +#: osparc/auth/ui/LoginView.js +msgid "Disclaimer" +msgstr "" + +#: osparc/ui/basic/PoweredByOsparc.js +msgid "powered by" +msgstr "" + +#: osparc/auth/ui/LoginView.js +msgid "email or password don't look correct" +msgstr "" + +#: osparc/auth/ui/RegistrationView.js +msgid "Registration" +msgstr "" + +#: osparc/auth/ui/RegistrationView.js +msgid "Type your email" +msgstr "" + +#: osparc/auth/ui/RegistrationView.js +msgid "Type a password" +msgstr "" + +#: osparc/auth/ui/RegistrationView.js +msgid "Retype the password" +msgstr "" + +#: osparc/auth/ui/RegistrationView.js +msgid "Cannot register user" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "First Name" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Last Name" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "University Email" +msgstr "" + +#: osparc/po/Users.js +msgid "Email" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Phone Number" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Company Name" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "University" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Organization" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Research Group/Organization" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Address" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "City" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Postal code" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Country" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Application" +msgstr "" + +#: osparc/pricing/PlanEditor.js +msgid "Description" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "How did you hear about us?" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "I acknowledge that data will be processed in accordance to " +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Request" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "The request is being processed, you will hear from us in the coming hours" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Reload Captcha" +msgstr "" + +#: osparc/auth/ui/RequestAccount.js +msgid "Type the 6 digits:" +msgstr "" + +#: osparc/auth/ui/VerifyPhoneNumberView.js +msgid "Two-Factor Authentication (2FA)" +msgstr "" + +#: osparc/auth/ui/VerifyPhoneNumberView.js +msgid "A text message will be sent to your mobile phone for authentication each time you log in." +msgstr "" + +#: osparc/auth/ui/VerifyPhoneNumberView.js +msgid "Send SMS" +msgstr "" + +#: osparc/auth/ui/VerifyPhoneNumberView.js +msgid "Type the SMS code" +msgstr "" + +#: osparc/file/FileDownloadLink.js +msgid "Validate" +msgstr "" + +#: osparc/auth/ui/VerifyPhoneNumberView.js +msgid "Skip phone registration and send code via email" +msgstr "" + +#: osparc/auth/ui/Login2FAValidationCodeView.js +msgid "Invalid code" +msgstr "" + +#: osparc/auth/ui/ResetPassView.js +msgid "Reset Password" +msgstr "" + +#: osparc/auth/ui/ResetPassRequestView.js +msgid "Type your registration email" +msgstr "" + +#: osparc/auth/ui/ResetPassRequestView.js +msgid "Could not request password reset" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Your new password" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Retype your new password" +msgstr "" + +#: osparc/auth/ui/ResetPassView.js +msgid "Could not reset password" +msgstr "" + +#: osparc/auth/ui/Login2FAValidationCodeView.js +msgid "Type code" +msgstr "" + +#: osparc/auth/ui/Login2FAValidationCodeView.js +msgid "Didn't receive the code? Click to resend" +msgstr "" + +#: osparc/auth/ui/Login2FAValidationCodeView.js +msgid "Via SMS" +msgstr "" + +#: osparc/auth/ui/Login2FAValidationCodeView.js +msgid "Via email" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "Access Full TIP" +msgstr "" + +#: osparc/ui/basic/PoweredByOsparc.js +msgid "powered by " +msgstr "" + +#: osparc/workbench/BaseNodeUI.js +msgid "Information..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Convert to Pipeline" +msgstr "" + +#: osparc/navigation/StudyTitleWOptions.js +msgid "Convert to Standalone" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Restore" +msgstr "" + +#: osparc/navigation/StudyTitleWOptions.js +msgid "Platform Logs..." +msgstr "" + +#: osparc/task/TasksButton.js +msgid "Tasks" +msgstr "" + +#: osparc/jobs/JobsBrowser.js +msgid "Jobs" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "My Account" +msgstr "" + +#: osparc/desktop/credits/BillingCenter.js +msgid "Credit Accounts" +msgstr "" + +#: osparc/desktop/credits/BillingCenter.js +msgid "Payment Methods" +msgstr "" + +#: osparc/desktop/credits/BillingCenter.js +msgid "Payments" +msgstr "" + +#: osparc/desktop/account/MyAccount.js +msgid "Usage" +msgstr "" + +#: osparc/desktop/credits/BillingCenter.js +msgid "Purchases" +msgstr "" + +#: osparc/desktop/credits/BillingCenter.js +msgid "Checkouts" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "username" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Expiration date:" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Please contact us via email:
" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Avoid dots or numbers in text" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Update Profile" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Profile updated" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Unsuccessful profile update" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Privacy" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "For Privacy reasons, you might want to hide some personal data." +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Update Privacy" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Name is required" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Set the Name first" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Privacy updated" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Unsuccessful privacy update" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "If all searchable fields are hidden, you will not be discoverable." +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Two-Factor Authentication" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Set your preferred method to use for two-factor authentication when signing in:" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "You are about to disable the 2FA" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid " The Two-Factor Authentication is one more measure to prevent hackers from accessing your account with an additional layer of security. When you sign in, 2FA helps make sure that your resources and personal information stays private, safe and secure. " +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Yes, disable" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "2FA Method" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Password" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Your current password" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Unsuccessful password reset" +msgstr "" + +#: osparc/desktop/account/ProfilePage.js +msgid "Danger Zone" +msgstr "" + +#: osparc/desktop/account/DeleteAccount.js +msgid "Delete Account" +msgstr "" + +#: osparc/store/Services.js +msgid "Unable to fetch Services" +msgstr "" + +#: osparc/store/Services.js +msgid "Some services are inaccessible:
" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Successfully deleted" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Successfully restored" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "The permissions will be taken from the new workspace." +msgstr "" + +#: osparc/dashboard/MoveResourceTo.js +msgid "Move" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Move to..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Are you sure you want to delete the Folder and all its content?" +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "It will be permanently deleted after " +msgstr "" + +#: osparc/editor/AnnotationEditor.js +msgid "Delete" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Start with an empty study" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "New Plan" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Delete permanently" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "Import" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Import Study" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Move to" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Cancel Selection" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Select " +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Open location" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Rename..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Thumbnail..." +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Edit Thumbnail" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid "Something went wrong while renaming" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while updating the thumbnail" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid " files..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Billing Settings..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Duplicate" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while converting to pipeline" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Export cMIS" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Duplicate process started and added to the background tasks" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while duplicating" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Preparing files" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Exporting process started and added to the background tasks" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Download started" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while exporting the study" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Uploading file" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Importing process started and added to the background tasks" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Importing Study..." +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Processing study" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while fetching the study" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Something went wrong while importing the study" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Are you sure you want to delete" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "will be removed from your list. Collaborators will still have access." +msgstr "" + +#: osparc/form/json/JsonSchemaFormHeader.js +msgid "Remove" +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Duplicating " +msgstr "" + +#: osparc/dashboard/StudyBrowser.js +msgid "Duplication cancelled" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Update all" +msgstr "" + +#: osparc/dashboard/TemplateBrowser.js +msgid "Are you sure you want to update all " +msgstr "" + +#: osparc/node/LifeCycleView.js +msgid "Update" +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid "Are you sure you want to delete " +msgstr "" + +#: osparc/dashboard/TemplateBrowser.js +msgid "Study to Template cancelled" +msgstr "" + +#: osparc/dashboard/TemplateBrowser.js +msgid "Something went wrong while publishing the study
" +msgstr "" + +#: osparc/dashboard/TemplateBrowser.js +msgid "Publishing " +msgstr "" + +#: osparc/dashboard/ServiceBrowser.js +msgid "It seems you don't have access to this product." +msgstr "" + +#: osparc/dashboard/ServiceBrowser.js +msgid "Please contact us:" +msgstr "" + +#: osparc/dashboard/ServiceBrowser.js +msgid "Test with data" +msgstr "" + +#: osparc/dashboard/ServiceBrowser.js +msgid "Submit new service" +msgstr "" + +#: osparc/dashboard/ServiceBrowser.js +msgid "Submit a new service" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "New Node" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Nodes" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Storage" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Study options" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Service options" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Workbench" +msgstr "" + +#: osparc/node/slideshow/NodeView.js +msgid "Logger" +msgstr "" + +#: osparc/desktop/SlideshowToolbar.js +msgid "App Mode" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Start App Mode" +msgstr "" + +#: osparc/pricing/PlanListItem.js +msgid "Edit" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Start" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Annotations" +msgstr "" + +#: osparc/editor/AnnotationNoteCreator.js +msgid "Note" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Rectangle" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Text" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Show" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Create" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Inputs" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Service data" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Outputs" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Options" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Are you sure you want to delete the selected node?" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Delete Node" +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Are you sure you want to delete the selected " +msgstr "" + +#: osparc/desktop/WorkbenchView.js +msgid "Delete Nodes" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Settings" +msgstr "" + +#: osparc/desktop/SlideshowView.js +msgid "Are you sure you want to delete node?" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Show logs only from current node" +msgstr "" + +#: osparc/filter/TextFilter.js +msgid "Filter" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Min log-level" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Toggle auto-scroll" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Copy logs to clipboard" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Copy Selected log to clipboard" +msgstr "" + +#: osparc/widget/logger/LoggerView.js +msgid "Download logs" +msgstr "" + +#: osparc/form/renderer/PropForm.js +msgid "Unlink" +msgstr "" + +#: osparc/navigation/PrevNextButtons.js +msgid "Select File" +msgstr "" + +#: osparc/form/renderer/PropForm.js +msgid "Set new parameter" +msgstr "" + +#: osparc/form/renderer/PropForm.js +msgid "Set existing parameter" +msgstr "" + +#: osparc/form/renderer/PropForm.js +msgid "Required Input" +msgstr "" + +#: osparc/form/renderer/PropForm.js +msgid "Input" +msgstr "" + +#: osparc/widget/NodesSlidesTree.js +msgid "Use the eye icons to display/hide nodes in the App Mode." +msgstr "" + +#: osparc/widget/NodesSlidesTree.js +msgid "Use the up and down arrows to sort them." +msgstr "" + +#: osparc/widget/NodesSlidesTree.js +msgid "You can also display nodes by clicking on them on the Workbench or Nodes list." +msgstr "" + +#: osparc/widget/NodesSlidesTree.js +msgid "Disable" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Save" +msgstr "" + +#: osparc/snapshots/EditSnapshotView.js +msgid "OK" +msgstr "" + +#: osparc/snapshots/IterationsView.js +msgid "Edit Tag" +msgstr "" + +#: osparc/snapshots/SnapshotsView.js +msgid "Edit Snapshot" +msgstr "" + +#: osparc/snapshots/IterationsView.js +msgid "iterations" +msgstr "" + +#: osparc/snapshots/IterationsView.js +msgid "Edit Iteration" +msgstr "" + +#: osparc/desktop/StudyEditorIdlingTracker.js +msgid "Are you still there?" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "Nodes can't be added while the pipeline is running" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "Nodes can't be deleted while the pipeline is running" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "You are not allowed to add nodes" +msgstr "" + +#: osparc/data/model/IframeHandler.js +msgid " is retired" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "Error creating " +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "File couldn't be assigned" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "Parameter couldn't be assigned" +msgstr "" + +#: osparc/data/model/Workbench.js +msgid "Probe couldn't be assigned" +msgstr "" + +#: osparc/data/model/Node.js +msgid " ports auto connected" +msgstr "" + +#: osparc/data/model/Node.js +msgid "Do you really want Stop and Save the current state?" +msgstr "" + +#: osparc/desktop/StartStopButtons.js +msgid "Stop" +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Waiting ..." +msgstr "" + +#: osparc/widget/ProgressSequence.js +msgid "CREATING ..." +msgstr "" + +#: osparc/share/CollaboratorsService.js +msgid "Service successfully shared" +msgstr "" + +#: osparc/share/CollaboratorsService.js +msgid "Something went wrong while sharing the service" +msgstr "" + +#: osparc/share/CollaboratorsService.js +msgid "Something went wrong while removing member" +msgstr "" + +#: osparc/share/CollaboratorsService.js +msgid "Operation not available" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "Cancel upload" +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "Download" +msgstr "" + +#: osparc/node/UpdateResourceLimitsView.js +msgid "Reset" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Provide Link" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "In order to Select a File you have three options:" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "Drop file here" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "Select New File" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "Select Download Link" +msgstr "" + +#: osparc/file/FilePicker.js +msgid "Select File from other " +msgstr "" + +#: osparc/file/FileDownloadLink.js +msgid "Select" +msgstr "" + +#: osparc/task/TaskUI.js +msgid "Are you sure you want to cancel the task?" +msgstr "" + +#: osparc/task/TaskUI.js +msgid "Cancel Task" +msgstr "" + +#: osparc/task/TaskUI.js +msgid "Ignore" +msgstr "" + +#: osparc/dashboard/ResourceUpgradeHelper.js +msgid "Outdated services" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Copy Raw metadata" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Copy Service Id" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "SERVICE ID" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "KEY" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "INTEGRATION VERSION" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "VERSION" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "RELEASE DATE" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "CONTACT" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "AUTHORS" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "ACCESS RIGHTS" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "CLASSIFIERS" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "QUALITY" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Show Description only" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "From all the metadata shown in this view,\\\\\\\\nonly the Description will be shown to Users." +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Edit Icon" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Edit Name" +msgstr "" + +#: osparc/info/ServiceLarge.js +msgid "Edit Version Display" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Edit Description" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "An issue occurred while updating the information." +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Title:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Thumbnail:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Description:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Author:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Access:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Created:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Modified:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Tags:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Quality:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Classifiers:" +msgstr "" + +#: osparc/info/StudyLarge.js +msgid "Location:" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Edit Title" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid "Credit Account" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Buy Credits" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Transfer from this Credit Account" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Last charge:" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "credits" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "You don't have access to the last used Credit Account" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Top up the Credit Account:
Purchase additional credits to restore a positive balance." +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Transfer credits from another Account:
Use this Credit Account to cover the negative balance." +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "A credits transfer will be initiated to cover the negative balance:" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Credits to transfer: " +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "From: " +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "To: " +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Transfer" +msgstr "" + +#: osparc/study/BillingSettings.js +msgid "Credit Account saved" +msgstr "" + +#: osparc/service/PricingUnitsList.js +msgid "No Tiers found" +msgstr "" + +#: osparc/info/CommentsList.js +msgid "0 Comments" +msgstr "" + +#: osparc/info/CommentsList.js +msgid "Load more comments..." +msgstr "" + +#: osparc/info/CommentsList.js +msgid "1 Comment" +msgstr "" + +#: osparc/info/CommentsList.js +msgid " Comments" +msgstr "" + +#: osparc/info/CommentAdd.js +msgid "Add comment" +msgstr "" + +#: osparc/editor/AnnotationNoteCreator.js +msgid "Add" +msgstr "" + +#: osparc/metadata/ClassifiersEditor.js +msgid "RRID:" +msgstr "" + +#: osparc/metadata/ClassifiersEditor.js +msgid "Add Classifier" +msgstr "" + +#: osparc/metadata/ClassifiersEditor.js +msgid "RRID classifier successfully added" +msgstr "" + +#: osparc/metadata/ClassifiersEditor.js +msgid "Classifiers successfully edited" +msgstr "" + +#: osparc/metadata/ClassifiersEditor.js +msgid "Something went wrong while editing classifiers" +msgstr "" + +#: osparc/metadata/ClassifiersViewer.js +msgid "No Classifiers assigned" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Quality Assessment data not found" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "There was an issue validating the metadata." +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Enabled" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Rules" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Conformance Level" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Target" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Conformance Level Target" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "References" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "TSR SCORE" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Not Applicable" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "Edit References" +msgstr "" + +#: osparc/metadata/QualityEditor.js +msgid "There was an issue while updating the Quality Assessment." +msgstr "" + +#: osparc/form/tag/TagManager.js +msgid "Apply Tags" +msgstr "" + +#: osparc/desktop/preferences/pages/TagsPage.js +msgid "New Tag" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "This service is deprecated. Please update." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "This service has been retired. Please update." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Some services are inaccessible. Please contact the service owner:" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Services marked in red are retired and can no longer be used." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "If the Update button is disabled, they might require manual intervention to be updated:" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Open the study" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Click on the retired service, download the data" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Upload the data to a newer version" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Services marked in yellow are deprecated, they will be retired soon." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "They can be updated by pressing the Update button." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "All services are up to date to their latest compatible version." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Click Update to upgrade services to the latest compatible version." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Some services are not up to date." +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Current" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Compatible" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Latest compatible version" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Unknown" +msgstr "" + +#: osparc/metadata/ServicesInStudyUpdate.js +msgid "Up-to-date" +msgstr "" + +#: osparc/metadata/ServicesInStudyBootOpts.js +msgid "Here you can select in which mode the services will be started:" +msgstr "" + +#: osparc/metadata/ServicesInStudyBootOpts.js +msgid "Boot Mode" +msgstr "" + +#: osparc/metadata/ServicesInStudyBootOpts.js +msgid "Select boot type" +msgstr "" + +#: osparc/metadata/ServicesInStudy.js +msgid "Could not retrieve some service information" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "me" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Ten Simple Rules score" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Autostart services" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Disabling this will help opening and closing studies/projects faster" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Add tags" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Meta details" +msgstr "" + +#: osparc/info/StudyUtils.js +msgid "Share with Editors and Organizations" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Quality Assessment" +msgstr "" + +#: osparc/study/SaveAsTemplate.js +msgid "Publish with data" +msgstr "" + +#: osparc/study/SaveAsTemplate.js +msgid "Publish" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid " Options" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid "An issue occurred while selecting Credit Account" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Title" +msgstr "" + +#: osparc/study/StudyOptions.js +msgid "Advanced options" +msgstr "" + +#: osparc/service/ServiceListItem.js +msgid "Number of times you instantiated it" +msgstr "" + +#: osparc/dashboard/GridButtonItem.js +msgid "Viewer only" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Last modified" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Deleted" +msgstr "" + +#: osparc/dashboard/ListButtonItem.js +msgid "Click to filter by this Tag" +msgstr "" + +#: osparc/dashboard/ListButtonItem.js +msgid "More..." +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "All " +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "My " +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "Shared with Me" +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "Shared with Everyone" +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "search" +msgstr "" + +#: osparc/dashboard/SearchBarFilter.js +msgid "Service Type" +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "Recently Deleted" +msgstr "" + +#: osparc/pricing/PlanDetails.js +msgid "Services" +msgstr "" + +#: osparc/dashboard/ResourceFilter.js +msgid "Shared" +msgstr "" + +#: osparc/dashboard/ResourceFilter.js +msgid "All Tags..." +msgstr "" + +#: osparc/dashboard/ResourceFilter.js +msgid "Less Tags..." +msgstr "" + +#: osparc/dashboard/ResourceFilter.js +msgid "Edit Tags..." +msgstr "" + +#: osparc/study/NodePricingUnits.js +msgid "Cannot change Tier" +msgstr "" + +#: osparc/file/TreeFolderView.js +msgid "Calculating Size" +msgstr "" + +#: osparc/file/TreeFolderView.js +msgid "Total size: " +msgstr "" + +#: osparc/product/quickStart/s4l/Welcome.js +msgid "Welcome to Sim4Life" +msgstr "" + +#: osparc/product/quickStart/s4lacad/Welcome.js +msgid "Experience Most Advanced Simulations – All In The Cloud" +msgstr "" + +#: osparc/product/quickStart/s4llite/Welcome.js +msgid "Welcome onboard " +msgstr "" + +#: osparc/product/quickStart/s4lacad/Welcome.js +msgid " Sim4Life is a revolutionary simulation platform, combining computable human phantoms with the most powerful physics solvers and the most advanced tissue models, for directly analyzing biological real-world phenomena and complex technical devices in a validated biological and anatomical environment.
In order to facilitate the introduction to the platform, we have some Guided Tours that can be found under the User Menu.
For more specific technical information, please refer to the Manuals on the Navigation Bar. " +msgstr "" + +#: osparc/product/quickStart/s4lacad/Welcome.js +msgid "Welcome to Sim4Life Science" +msgstr "" + +#: osparc/tours/Step.js +msgid "Skip" +msgstr "" + +#: osparc/navigation/PrevNextButtons.js +msgid "Next" +msgstr "" + +#: osparc/tours/Step.js +msgid "To Tours" +msgstr "" + +#: osparc/tours/Step.js +msgid "Step: " +msgstr "" + +#: osparc/admin/Announcements.js +msgid "End" +msgstr "" + +#: osparc/desktop/credits/BuyCreditsForm.js +msgid "A one-off, non recurring payment." +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Pay with" +msgstr "" + +#: osparc/desktop/credits/BuyCreditsForm.js +msgid "Enter card details in the next step..." +msgstr "" + +#: osparc/share/NewCollaboratorsManager.js +msgid "Share with" +msgstr "" + +#: osparc/share/NewCollaboratorsManager.js +msgid "Select users or organizations from the list below." +msgstr "" + +#: osparc/share/NewCollaboratorsManager.js +msgid "Select users from the list below." +msgstr "" + +#: osparc/share/NewCollaboratorsManager.js +msgid "
Search them if they aren't listed." +msgstr "" + +#: osparc/po/Users.js +msgid "Search" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "Organizations" +msgstr "" + +#: osparc/desktop/organizations/OrganizationsWindow.js +msgid "Organization details" +msgstr "" + +#: osparc/share/ShareePermissions.js +msgid "The following users/groups will not be able to open the shared study, because they don't have access to some services. Please contact the service owner(s) to give permission." +msgstr "" + +#: osparc/utils/Validators.js +msgid "Color must be in hexadecimal form" +msgstr "" + +#: osparc/utils/Validators.js +msgid "Format is invalid" +msgstr "" + +#: osparc/widget/IntlTelInput.js +msgid "Invalid number" +msgstr "" + +#: osparc/widget/IntlTelInput.js +msgid "Invalid country code" +msgstr "" + +#: osparc/widget/IntlTelInput.js +msgid "Number too short" +msgstr "" + +#: osparc/widget/IntlTelInput.js +msgid "Number too long" +msgstr "" + +#: osparc/product/AboutProduct.js +msgid "About " +msgstr "" + +#: osparc/About.js +msgid " is an online-accessible, cloud-based, and collaborative computational modeling platform that was developed under the Common Fund’s Stimulating Peripheral Activity to Relieve Conditions (SPARC) program to ensure sustainable, reproducible, and FAIR (findable, accessible, interoperable, reusable) computational modeling in the field of bioelectronic medicine – from neural interfaces to peripheral nerve recruitment and the resulting effects on organ function.
For more information about SPARC and the services offered, visit the " +msgstr "" + +#: osparc/About.js +msgid "The platform is built upon a number of open-source resources - we can't do it all alone! Some of the technologies that we leverage include:" +msgstr "" + +#: osparc/About.js +msgid "Front-end" +msgstr "" + +#: osparc/About.js +msgid "Back-end" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Raw metadata" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Author" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Creation Date" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Last Modified" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Access Rights" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Service ID" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Service Key" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Service Integration Version" +msgstr "" + +#: osparc/info/MergedLarge.js +msgid "Service Version" +msgstr "" + +#: osparc/desktop/account/MyAccount.js +msgid "Profile" +msgstr "" + +#: osparc/desktop/account/MyAccount.js +msgid "Confirmations" +msgstr "" + +#: osparc/desktop/account/MyAccount.js +msgid "API Keys/Tokens" +msgstr "" + +#: osparc/desktop/account/MyAccount.js +msgid "Create/Edit Tags" +msgstr "" + +#: osparc/admin/AdminCenterWindow.js +msgid "Admin Center" +msgstr "" + +#: osparc/po/POCenterWindow.js +msgid "PO Center" +msgstr "" + +#: osparc/tester/TesterCenterWindow.js +msgid "Tester Center" +msgstr "" + +#: osparc/desktop/credits/BillingCenterWindow.js +msgid "Billing Center" +msgstr "" + +#: osparc/vipMarket/MarketWindow.js +msgid "The Shop" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "About oSPARC" +msgstr "" + +#: osparc/product/AboutProduct.js +msgid "About Product" +msgstr "" + +#: osparc/navigation/UserMenu.js +msgid "License" +msgstr "" + +#: osparc/desktop/credits/CreditsIndicator.js +msgid " credits" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethods.js +msgid "Credit cards used for payments in your personal Credit Account" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Add Payment Method" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethods.js +msgid "Fetching Payment Methods" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethods.js +msgid "The window was closed. Try again and follow the instructions inside the opened window." +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethods.js +msgid "No Payment Methods found" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethods.js +msgid "Could not retrieve your saved payment methods. Please try again later." +msgstr "" + +#: osparc/desktop/credits/Transactions.js +msgid "Top-up and refunds in US Dollars associated to your personal account show up here." +msgstr "" + +#: osparc/desktop/account/DeleteAccount.js +msgid "Your email" +msgstr "" + +#: osparc/desktop/account/DeleteAccount.js +msgid "Your password" +msgstr "" + +#: osparc/workbench/DiskUsageIndicator.js +msgid "Disk usage" +msgstr "" + +#: osparc/workbench/DiskUsageIndicator.js +msgid "Data storage: " +msgstr "" + +#: osparc/workbench/DiskUsageIndicator.js +msgid "I/O storage: " +msgstr "" + +#: osparc/dashboard/WorkspaceButtonNew.js +msgid "New Workspace" +msgstr "" + +#: osparc/dashboard/FolderButtonNew.js +msgid "New folder" +msgstr "" + +#: osparc/dashboard/NewPlusMenu.js +msgid "New Folder" +msgstr "" + +#: osparc/dashboard/MoveResourceTo.js +msgid "Current location:" +msgstr "" + +#: osparc/dashboard/MoveResourceTo.js +msgid "- Workspace: " +msgstr "" + +#: osparc/dashboard/MoveResourceTo.js +msgid "- Folder: " +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "Delete all" +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "All items will be permanently deleted" +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "Shared Workspaces" +msgstr "" + +#: osparc/dashboard/StudyBrowserHeader.js +msgid "Search results" +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Edit..." +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Edit Workspace" +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Share Workspace" +msgstr "" + +#: osparc/vipMarket/SortModelsButtons.js +msgid "Sort" +msgstr "" + +#: osparc/pricing/PlanEditor.js +msgid "Name" +msgstr "" + +#: osparc/dashboard/SortedByMenuButton.js +msgid "Created" +msgstr "" + +#: osparc/dashboard/SortedByMenuButton.js +msgid "Modified" +msgstr "" + +#: osparc/study/Import.js +msgid "Max file size 10GB" +msgstr "" + +#: osparc/workbench/BaseNodeUI.js +msgid "Rename" +msgstr "" + +#: osparc/widget/Renamer.js +msgid "Type text" +msgstr "" + +#: osparc/editor/ThumbnailEditor.js +msgid "Error checking link" +msgstr "" + +#: osparc/editor/ThumbnailEditor.js +msgid "url" +msgstr "" + +#: osparc/editor/ThumbnailEditor.js +msgid "or pick one from the list below:" +msgstr "" + +#: osparc/task/Import.js +msgid "Importing Study" +msgstr "" + +#: osparc/task/Duplicate.js +msgid "Duplicating:" +msgstr "" + +#: osparc/task/ToTemplate.js +msgid "Publishing:" +msgstr "" + +#: osparc/service/SortServicesButtons.js +msgid "Hits" +msgstr "" + +#: osparc/service/SortServicesButtons.js +msgid "Name Asc" +msgstr "" + +#: osparc/service/SortServicesButtons.js +msgid "Name Desc" +msgstr "" + +#: osparc/widget/CollapsibleViewLight.js +msgid "Expand" +msgstr "" + +#: osparc/widget/CollapsibleViewLight.js +msgid "Collapse" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "Study Information" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "DESCRIPTION" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "More Info" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "AUTHOR" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "CREATED" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "MODIFIED" +msgstr "" + +#: osparc/info/StudyMedium.js +msgid "TAGS" +msgstr "" + +#: osparc/widget/NodeOptions.js +msgid "To proceed with the following actions, the service needs to be Stopped." +msgstr "" + +#: osparc/desktop/SlideshowToolbar.js +msgid "Save App Mode" +msgstr "" + +#: osparc/desktop/SlideshowToolbar.js +msgid "Stop App Mode" +msgstr "" + +#: osparc/navigation/PrevNextButtons.js +msgid "Previous" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Service Information" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Instructions" +msgstr "" + +#: osparc/node/slideshow/BaseNodeView.js +msgid "Preparing Inputs" +msgstr "" + +#: osparc/workbench/ServiceCatalog.js +msgid "Service catalog" +msgstr "" + +#: osparc/form/renderer/PropFormBase.js +msgid "Required input: without it, the service will not start/run." +msgstr "" + +#: osparc/widget/NodeSlideTreeItem.js +msgid "Edit Instructions" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Progress" +msgstr "" + +#: osparc/workbench/NodeUI.js +msgid "Convert to Iterator" +msgstr "" + +#: osparc/workbench/NodeUI.js +msgid "Convert to Parameter" +msgstr "" + +#: osparc/widget/NodeTreeItem.js +msgid "Remove Marker" +msgstr "" + +#: osparc/widget/NodeTreeItem.js +msgid "Add Marker" +msgstr "" + +#: osparc/data/model/IframeHandler.js +msgid "There was an issue starting" +msgstr "" + +#: osparc/widget/NodeOutputs.js +msgid "Connects a Probe to this output" +msgstr "" + +#: osparc/file/FileDownloadLink.js +msgid "An issue occurred while checking link" +msgstr "" + +#: osparc/file/FileDownloadLink.js +msgid "Type a Link" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Upload file" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Drop file from File Explorer" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Drop file from tree" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "or" +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Only one file can be uploaded at a time." +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Please compress all files into a single zip file." +msgstr "" + +#: osparc/file/FileDrop.js +msgid "Drop me" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Folders are not accepted. Please upload a zip file instead." +msgstr "" + +#: osparc/file/FolderViewer.js +msgid "Select folder" +msgstr "" + +#: osparc/file/FolderViewer.js +msgid "Multiselect" +msgstr "" + +#: osparc/editor/TextEditor.js +msgid "Write" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Execute" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Public" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Limit" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Runtime check:
The service can consume a maximum of 'limit' resources - if it attempts to use more resources than this limit, it will be stopped" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Resources" +msgstr "" + +#: osparc/node/UpdateResourceLimitsView.js +msgid "Resource Limits" +msgstr "" + +#: osparc/info/ServiceUtils.js +msgid "Share with Collaborators and Organizations" +msgstr "" + +#: osparc/editor/MarkdownEditor.js +msgid "Markdown supported" +msgstr "" + +#: osparc/editor/HtmlEditor.js +msgid "Preview" +msgstr "" + +#: osparc/form/tag/TagItem.js +msgid "Share Tag" +msgstr "" + +#: osparc/form/tag/TagItem.js +msgid "Color" +msgstr "" + +#: osparc/metadata/ServicesInStudy.js +msgid "Something went wrong while updating the service" +msgstr "" + +#: osparc/metadata/ServicesInStudy.js +msgid "The Study is empty" +msgstr "" + +#: osparc/metadata/ServicesInStudy.js +msgid "Could not retrieve service information" +msgstr "" + +#: osparc/share/PublishTemplate.js +msgid "Make the " +msgstr "" + +#: osparc/share/PublishTemplate.js +msgid " also accessible to:" +msgstr "" + +#: osparc/share/PublishTemplate.js +msgid "Publish for..." +msgstr "" + +#: osparc/dashboard/GroupedCardContainer.js +msgid "Show less" +msgstr "" + +#: osparc/dashboard/GroupedCardContainer.js +msgid "Show all" +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Are you sure you want to delete the Workspace and all its content?" +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "All the content of the workspace will be deleted." +msgstr "" + +#: osparc/dashboard/WorkspaceButtonItem.js +msgid "Delete Workspace" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Edit Folder" +msgstr "" + +#: osparc/dashboard/FolderButtonItem.js +msgid "Delete Folder" +msgstr "" + +#: osparc/product/quickStart/s4llite/Welcome.js +msgid "Quick Start Guide" +msgstr "" + +#: osparc/product/quickStart/tis/Welcome.js +msgid " This quick tutorial gives a basic overview of how the TI Planning Tool works and how to navigate through the interface.
We will focus on two main aspects, how to:
- Use the platform
- Get started with a New Plan
" +msgstr "" + +#: osparc/product/quickStart/tis/Dashboard.js +msgid " The Dashboard is your private hub which contains all of your Plans as well as Plans that have been shared with you. From the Dashboard you are able to open your Plan or create a New Plan from scratch. " +msgstr "" + +#: osparc/product/quickStart/tis/Dashboard.js +msgid " 1) New Plan: by clicking on this card a new study will be created and open. The planning process will be presented in three successive steps that will be described more in detail in the following steps. " +msgstr "" + +#: osparc/product/quickStart/tis/Dashboard.js +msgid " 2) The other cards are TI Plans that were already created by you or shared with you. You can reopen them to do further analysis or by clicking three dots, on the top right corner, you can share, delete or check the details and metadata. " +msgstr "" + +#: osparc/product/quickStart/tis/ElectrodeSelector.js +msgid "Electrode Selector" +msgstr "" + +#: osparc/product/quickStart/tis/ElectrodeSelector.js +msgid " After pressing New Plan, three panels will be shown. " +msgstr "" + +#: osparc/product/quickStart/tis/ElectrodeSelector.js +msgid " In a first step, the relevant species, stimulation target, electrode shapes, electrode dimensions and potential electrode locations (currently required to narrow down the huge exposure configuration search space) are selected. " +msgstr "" + +#: osparc/product/quickStart/tis/ElectrodeSelector.js +msgid " After finishing the set up, the big button on the top right will turn blue and by clicking on it you will submit the configuration. " +msgstr "" + +#: osparc/product/quickStart/tis/ElectrodeSelector.js +msgid " Now the Arrow that says 'Next' can be pushed and the optimization will immediately start. " +msgstr "" + +#: osparc/product/quickStart/tis/PostPro.js +msgid "Post Processing" +msgstr "" + +#: osparc/product/quickStart/tis/PostPro.js +msgid " Based on extensive sweeping/optimization, a series of highly performing exposure parameters are proposed for the user to interactively explore, using predefined quantification metrics and visualizations. Identified conditions-of-interest can be documented and added to a report. " +msgstr "" + +#: osparc/product/quickStart/tis/PostPro.js +msgid " These metrics are reported in the Post Processing analysis environment for each electrode pair in the combination in a sorted tabular form that can be used to inspect the stimulation performances. By clicking on each pair, slice views of the maximum amplitude modulation (MAP) within the head are produced.
Pressing the `Load` button on the right, the selected configuration will be loaded. " +msgstr "" + +#: osparc/product/quickStart/tis/PostPro.js +msgid " Alternatively, slice views of the maximum interferential E-field can also be visualized and synchronous with the MAP slices to assess safety-related aspects (e.g., field intensity in proximity of the electrodes). These maps can be edited, thresholded, and saved offline for further inspection and analysis.
An isosurface of the TI stimulation distribution for the selected configuration can also be visualized within the head anatomy for inspection. " +msgstr "" + +#: osparc/product/quickStart/tis/PostPro.js +msgid " At the end of the optimization procedure, you can automatically generate a report.
It includes a summary of all the performance metrics calculated for each electrode pair combination, and a detailed performance report of the optimized electrode configuration. The report includes electrode placement, current intensities, performance metrics, TI and maximum high-frequency field distributions, cumulative dose histograms and all the graphs generated in the post-pro analysis tab. " +msgstr "" + +#: osparc/product/quickStart/tis/S4LPostPro.js +msgid "Not available in" +msgstr "" + +#: osparc/product/quickStart/tis/S4LPostPro.js +msgid "Sim4Life Post Processing" +msgstr "" + +#: osparc/product/quickStart/tis/S4LPostPro.js +msgid " Finally, and optionally, exposure conditions-of-interest can be visualized and analyzed freely, using the web-version of the Sim4Life (ZMT Zurich MedTech AG) computational life sciences platform. " +msgstr "" + +#: osparc/product/quickStart/tis/MoreInformation.js +msgid "For more information:" +msgstr "" + +#: osparc/product/quickStart/s4llite/Welcome.js +msgid " This quick user’s guide gives a short introduction to Sim4Life.lite. We will show:
- how to get started with a new project,
- how to get started from an existing tutorial project
- how to open Sim4Life desktop simulation projects in Sim4Life.lite,
- Sim4Life.lite features, limitations and user interface
For more specific technical information, please refer to the Dashboard Manual and the Sim4Life.lite Manual. " +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid "Dashboard - Projects & Tutorials" +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid " The Dashboard is the place where Projects and Tutorials can be accessed and organized. " +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid " 1) Start Sim4Life.lite: Click the + Start Sim4Life.lite button to create a new project. This will start the user interface of Sim4Life.lite. " +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid " 2) Other cards: Each card represents an existing project (own projects, or projects shared by other users) that can be accessed and managed. Click on the card to open the project. Click the “three dots” in the upper right corner of the card to perform operations such as rename, share, delete. " +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid " 3) TUTORIALS: A set of pre-built read-only tutorial projects with results is available to all Sim4Life.lite users. When a tutorial is selected, a copy is automatically created and added to the user’s Projects tab. This new copy is editable and can be shared. " +msgstr "" + +#: osparc/product/quickStart/s4llite/Dashboard.js +msgid " 4) To open an existing desktop project in Sim4Life.lite: - Click the + Start Sim4Life.lite button to create a new project.
- Click the menu and select “File Browser…”.
- Click “Upload File” for the .smash project and select the file from your desktop. Repeat the same step, but this time select “Upload Folder” and then select the result folder from your desktop. Close the window
- Click the Menu again and click “Open” to select the file you just uploaded.
" +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteSpecs.js +msgid "Sim4Life.lite: Features and Limitations" +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteSpecs.js +msgid " Sim4Life.lite is a powerful web-based simulation platform that allows you to model and analyze real-world phenomena and to design complex technical devices in a validated environment. Sim4Life.lite has been created specifically for students to facilitate their understanding of computational modeling and simulations for various topics, ranging from wireless communication to medical applications. The access to Sim4Life.lite is available free of charge to students enrolled at registered universities. " +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteSpecs.js +msgid " Sim4Life.lite offers
- Framework (GUI, Modeling, Postprocessing)
- 3D modeling environment (based on the ACIS toolkit) and CAD translators
- Postprocessing and visualization of the simulation results (2D and 3D viewers, 2D planar slice, volume rendering, streamlines, surface fields on arbitrary 3D structures, radiation and far-field data)
- No restrictions on number of modeling objects
- Solvers & Tissue Models:
- P-EM-FDTD: Electromagnetics Full-Wave Solver
- P-EM-QS: Quasi-Static Electromagnetics Solver
- P-Thermal: Thermodynamic Solver
- P-Acoustics: Acoustics Solver
- T-Neuro: Neuronal Tissue Models
- Computational anatomical model Yoon-sun, the first Korean model of the IT’IS Virtual Population
- Material database
- Python and Jupyter Notebook scripting " +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteSpecs.js +msgid " Limitations
The following limitations apply:
- Grid size of each simulation is limited to a maximum of 20 million grid cells
- High-Performance Computing is not supported:
- GPU acceleration is not available
- MPI multicore acceleration is not available
- 3rd-party tools are not available (e.g., MUSAIK, SYSSIM, IMAnalytics, etc…)
- Additional ViP models cannot be added
- 30 minutes idle time before logout
- Hardware resource limits
- 3 CPUs
- 3 GB of GPU RAM
- 5 GB disk space
- 16 GB RAM
" +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteUI.js +msgid "Sim4Life.lite" +msgstr "" + +#: osparc/product/quickStart/s4llite/S4LLiteUI.js +msgid " To check the Sim4Life.lite manual, please open a project and access the documentation via Help in the menu as shown below. Enjoy! " +msgstr "" + +#: osparc/desktop/credits/BuyCreditsInput.js +msgid "Credit Price" +msgstr "" + +#: osparc/desktop/credits/BuyCreditsInput.js +msgid "Credit Amount" +msgstr "" + +#: osparc/desktop/credits/BuyCreditsInput.js +msgid "Total" +msgstr "" + +#: osparc/filter/SearchingCollaborators.js +msgid "Searching..." +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid " An organization is a group of users who can share " +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid " and other resources.
Here you can see the list of organizations you belong to, create new organizations, or manage the membership by setting up the access rights of each member in the organization if you are a manager or administrator." +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid "New Organization" +msgstr "" + +#: osparc/desktop/organizations/OrganizationDetails.js +msgid "Organization Details Editor" +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid "Delete Organization" +msgstr "" + +#: osparc/desktop/organizations/OrganizationsList.js +msgid "Something went wrong while deleting " +msgstr "" + +#: osparc/pricing/PlanEditor.js +msgid " successfully created" +msgstr "" + +#: osparc/pricing/PlanEditor.js +msgid "Something went wrong while creating " +msgstr "" + +#: osparc/desktop/wallets/WalletDetails.js +msgid " successfully edited" +msgstr "" + +#: osparc/desktop/wallets/WalletDetails.js +msgid "Something went wrong while editing " +msgstr "" + +#: osparc/desktop/organizations/OrganizationDetails.js +msgid "Return to Organizations list" +msgstr "" + +#: osparc/desktop/wallets/WalletDetails.js +msgid "Members" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Job Id" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Solver" +msgstr "" + +#: osparc/desktop/credits/TransactionsTableModel.js +msgid "Status" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Submitted" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Started" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Info" +msgstr "" + +#: osparc/jobs/JobsTable.js +msgid "Instance" +msgstr "" + +#: osparc/notification/NotificationUI.js +msgid "You don't have access anymore" +msgstr "" + +#: osparc/notification/NotificationUI.js +msgid "Do you want to make it the default Credit Account?" +msgstr "" + +#: osparc/notification/NotificationUI.js +msgid "Default Credit Account" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Credits Indicator" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Show indicator" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Show warning when credits below" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Automatic Shutdown of Idle Instances" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Enter 0 to disable this function" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Idle time before closing (in minutes)" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Job Concurrency" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Maximum number of concurrent jobs" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Low Disk Space Threshold" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Set the warning Threshold for Low Disk Space availability" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Threshold (in GB)" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Help us improve Sim4Life user experience" +msgstr "" + +#: osparc/desktop/preferences/pages/GeneralPage.js +msgid "Share usage data" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Ask for confirmation for the following actions:" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Go back to the Dashboard" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Delete a " +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Warning: deleting a " +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid " cannot be undone" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Understood" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Delete a Node" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Warning: deleting a node cannot be undone" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Stop Node" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Snap Node to Grid" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "This is a list of experimental preferences" +msgstr "" + +#: osparc/desktop/preferences/pages/ConfirmationsPage.js +msgid "Connect ports automatically" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "API Keys" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "List API keys associated to your account." +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "New API Key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Cannot create API Key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Do you want to delete the API key?" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Delete API key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Cannot delete API Key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "API Tokens for External Services" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Provide the API tokens needed to access external services." +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Current Tokens" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Supported services" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Are you sure you want to delete this token?" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Delete Token" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Enter your token key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Key" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Enter your token secret" +msgstr "" + +#: osparc/desktop/preferences/pages/TokensPage.js +msgid "Secret" +msgstr "" + +#: osparc/desktop/preferences/pages/TagsPage.js +msgid " Tags help you organize the " +msgstr "" + +#: osparc/desktop/preferences/pages/TagsPage.js +msgid " in the Dashboard by categorizing topics, making it easier to search and filter. Once the tags are created, they can be assigned to the " +msgstr "" + +#: osparc/desktop/preferences/pages/TagsPage.js +msgid " via 'More options...' on the " +msgstr "" + +#: osparc/desktop/preferences/pages/TagsPage.js +msgid " cards." +msgstr "" + +#: osparc/desktop/credits/CreditsPerService.js +msgid "No usage records found" +msgstr "" + +#: osparc/product/AboutProduct.js +msgid "Information is unavailable" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "Personal" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "No personal Credit Account found" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "Shared with me" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "No shared Credit Accounts found" +msgstr "" + +#: osparc/desktop/wallets/WalletDetails.js +msgid "Credit Account Details Editor" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "Something went wrong while updating the Credit Account" +msgstr "" + +#: osparc/desktop/wallets/WalletsList.js +msgid "Currently in use" +msgstr "" + +#: osparc/desktop/wallets/WalletDetails.js +msgid "Return to Credit Accounts list" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodListItem.js +msgid "Are you sure you want to delete this Payment Method?" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodListItem.js +msgid "Delete Payment Method" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodDetails.js +msgid "Payment Method details" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodDetails.js +msgid "Card Holder name" +msgstr "" + +#: osparc/filter/NodeTypeFilter.js +msgid "Type" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodDetails.js +msgid "Number" +msgstr "" + +#: osparc/desktop/paymentMethods/PaymentMethodDetails.js +msgid "Expiration date" +msgstr "" + +#: osparc/desktop/credits/ResourceInTableViewer.js +msgid "Export" +msgstr "" + +#: osparc/desktop/credits/UsageTable.js +msgid "Node" +msgstr "" + +#: osparc/desktop/credits/CheckoutsTable.js +msgid "Duration" +msgstr "" + +#: osparc/study/PricingUnitLicense.js +msgid "Credits" +msgstr "" + +#: osparc/desktop/credits/PurchasesTable.js +msgid "PurchaseId" +msgstr "" + +#: osparc/desktop/credits/CheckoutsTable.js +msgid "ItemId" +msgstr "" + +#: osparc/desktop/credits/CheckoutsTable.js +msgid "Seats" +msgstr "" + +#: osparc/desktop/credits/CheckoutsTable.js +msgid "CheckoutId" +msgstr "" + +#: osparc/vipMarket/Market.js +msgid "My Models" +msgstr "" + +#: osparc/editor/OrganizationEditor.js +msgid "Thumbnail" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Double-click to add a node" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "INPUTS" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "OUTPUTS" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Pick the position" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Draw a rectangle" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Drop here" +msgstr "" + +#: osparc/workbench/WorkbenchUI.js +msgid "Draw a rectangle first" +msgstr "" + +#: osparc/widget/NodeTreeItem.js +msgid "Full Screen" +msgstr "" + +#: osparc/node/LifeCycleView.js +msgid "Update Service" +msgstr "" + +#: osparc/node/UpdateResourceLimitsView.js +msgid "Limits have been successfully updated" +msgstr "" + +#: osparc/node/UpdateResourceLimitsView.js +msgid "Something went wrong while updating the limits" +msgstr "" + +#: osparc/navigation/BreadcrumbsSlideshow.js +msgid "Pipeline is empty" +msgstr "" + +#: osparc/navigation/BreadcrumbsSlideshow.js +msgid "No visible nodes. Enable some by adjusting the app mode." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Please wait, this process may take a few minutes ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "LOADING ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Provisioning resources ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Setting up system software ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Retrieving your output data ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Retrieving your work ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Installing services ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Starting services ..." +msgstr "" + +#: osparc/data/model/NodeProgressSequence.js +msgid "Retrieving your input data ..." +msgstr "" + +#: osparc/ui/basic/NodeStatusUI.js +msgid "Select a file" +msgstr "" + +#: osparc/ui/basic/NodeStatusUI.js +msgid "Uploading..." +msgstr "" + +#: osparc/widget/PreparingInputs.js +msgid "To proceed, we need to prepare some inputs. You can check the progress logs here:" +msgstr "" + +#: osparc/widget/PreparingInputs.js +msgid "Run all" +msgstr "" + +#: osparc/widget/PreparingInputs.js +msgid "Logs" +msgstr "" + +#: osparc/ui/form/ContentSchemaHelper.js +msgid "Minimum items: " +msgstr "" + +#: osparc/ui/form/ContentSchemaHelper.js +msgid "Maximum items: " +msgstr "" + +#: osparc/ui/form/ContentSchemaHelper.js +msgid "Out of range" +msgstr "" + +#: osparc/ui/form/ContentSchemaHelper.js +msgid "Minimum value: " +msgstr "" + +#: osparc/ui/form/ContentSchemaHelper.js +msgid "Maximum value: " +msgstr "" + +#: osparc/file/FolderContent.js +msgid "Date Modified" +msgstr "" + +#: osparc/file/FolderContent.js +msgid "Size" +msgstr "" + +#: osparc/file/FolderContent.js +msgid "Id" +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "This action cannot be undone." +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "All contents within the folders will be deleted." +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "Do you want to proceed?" +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "Items successfully deleted" +msgstr "" + +#: osparc/file/FileLabelWithActions.js +msgid "Externally managed items cannot be deleted" +msgstr "" + +#: osparc/editor/HtmlEditor.js +msgid "Supports HTML" +msgstr "" + +#: osparc/study/PricingUnitTier.js +msgid "Credits/h" +msgstr "" + +#: osparc/study/PricingUnitTier.js +msgid "EC2" +msgstr "" + +#: osparc/ui/list/OrganizationListItem.js +msgid " members" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Edit details..." +msgstr "" + +#: osparc/desktop/wallets/MembersList.js +msgid "Add Members..." +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "You can add new members and assign roles." +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "You cannot add new members to this Organization. Please contact an Administrator or Manager." +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid " successfully added" +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Something went wrong while adding the user" +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Something went wrong while promoting to " +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Something went wrong while demoting to " +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Are you sure you want to leave?" +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "If you leave, the page will reload." +msgstr "" + +#: osparc/desktop/organizations/MembersList.js +msgid "Leave Organization" +msgstr "" + +#: osparc/desktop/organizations/TemplatesList.js +msgid "This is the list of " +msgstr "" + +#: osparc/desktop/organizations/TemplatesList.js +msgid " shared with this Organization" +msgstr "" + +#: osparc/desktop/organizations/ServicesList.js +msgid "This is the list of services shared with this Organization" +msgstr "" + +#: osparc/jobs/JobInfo.js +msgid "Job Info" +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Generate API Key" +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Key names must be unique." +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Key Name" +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Expiration Date" +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Select a future date" +msgstr "" + +#: osparc/desktop/preferences/window/CreateAPIKey.js +msgid "Confirm" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "API Key" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "For your security, store your access keys safely. You will not be able to access them again after closing this window." +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "Key:" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "Secret:" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "Base url:" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "API Secret" +msgstr "" + +#: osparc/desktop/preferences/window/ShowAPIKey.js +msgid "Base URL" +msgstr "" + +#: osparc/admin/AdminCenter.js +msgid "Pricing Plans" +msgstr "" + +#: osparc/admin/AdminCenter.js +msgid "Maintenance" +msgstr "" + +#: osparc/admin/AdminCenter.js +msgid "Announcements" +msgstr "" + +#: osparc/po/POCenter.js +msgid "Users" +msgstr "" + +#: osparc/po/POCenter.js +msgid "Pre-Registration" +msgstr "" + +#: osparc/po/POCenter.js +msgid "Invitations" +msgstr "" + +#: osparc/po/POCenter.js +msgid "Product Info" +msgstr "" + +#: osparc/po/POCenter.js +msgid "Message Templates" +msgstr "" + +#: osparc/tester/TesterCenter.js +msgid "Socket Messages" +msgstr "" + +#: osparc/tester/TesterCenter.js +msgid "Console Errors" +msgstr "" + +#: osparc/tester/Statics.js +msgid "Statics" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Something went wrong while updating the state" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Auto-recharge" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Auto-recharge: ON" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Auto-recharge: OFF" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "ON" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "OFF" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Credit Account enabled" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Credit Account blocked" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Currently being used" +msgstr "" + +#: osparc/desktop/wallets/WalletListItem.js +msgid "Switch to this Credit Account" +msgstr "" + +#: osparc/desktop/wallets/MembersList.js +msgid "Only Accountants of an Organization can share a wallet with other users." +msgstr "" + +#: osparc/vipMarket/SortModelsButtons.js +msgid "Date" +msgstr "" + +#: osparc/desktop/credits/TransactionsTableModel.js +msgid "Price USD" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Comment" +msgstr "" + +#: osparc/desktop/credits/TransactionsTableModel.js +msgid "Invoice" +msgstr "" + +#: osparc/vipMarket/VipMarket.js +msgid "Loading" +msgstr "" + +#: osparc/vipMarket/VipMarket.js +msgid "Cannot purchase model" +msgstr "" + +#: osparc/desktop/StartStopButtons.js +msgid "Run All" +msgstr "" + +#: osparc/desktop/StartStopButtons.js +msgid "Run Selection" +msgstr "" + +#: osparc/desktop/ZoomButtons.js +msgid "Zoom In" +msgstr "" + +#: osparc/desktop/ZoomButtons.js +msgid "Zoom Out" +msgstr "" + +#: osparc/desktop/ZoomButtons.js +msgid "Reset Zoom" +msgstr "" + +#: osparc/editor/AnnotationNoteCreator.js +msgid "Add Note" +msgstr "" + +#: osparc/editor/AnnotationNoteCreator.js +msgid "Add a recipient to be notified. Please make sure the user has access to the " +msgstr "" + +#: osparc/editor/AnnotationNoteCreator.js +msgid "Select recipient" +msgstr "" + +#: osparc/desktop/wallets/MemberListItem.js +msgid "Promote to " +msgstr "" + +#: osparc/desktop/wallets/MemberListItem.js +msgid "Demote to " +msgstr "" + +#: osparc/desktop/wallets/MemberListItem.js +msgid "Remove " +msgstr "" + +#: osparc/admin/Maintenance.js +msgid "Start and End dates go in UTC time zone" +msgstr "" + +#: osparc/admin/Maintenance.js +msgid "No Maintenance scheduled" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Create announcement" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "title" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "description" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "link" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Link" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Login" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Ribbon" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "User Menu" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Generate" +msgstr "" + +#: osparc/admin/Announcements.js +msgid "Copy announcement" +msgstr "" + +#: osparc/po/Users.js +msgid "user@email.address or user@*" +msgstr "" + +#: osparc/po/Users.js +msgid "Searching users..." +msgstr "" + +#: osparc/po/Users.js +msgid " user(s) found" +msgstr "" + +#: osparc/po/Users.js +msgid "Error searching users" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Pre-Registration" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Copy&Paste the Request Account Form in JSON format here ..." +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Request Form" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Unsuccessful Pre-Registration. See details below" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Searching Pre-Registered users..." +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "Pre-Registered as:" +msgstr "" + +#: osparc/po/PreRegistration.js +msgid "No Pre-Registered user found" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Create invitation" +msgstr "" + +#: osparc/po/Invitations.js +msgid "There is no invitation required in this product/deployment." +msgstr "" + +#: osparc/po/Invitations.js +msgid "new.user@email.address" +msgstr "" + +#: osparc/po/Invitations.js +msgid "User Email" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Welcome Credits (USD)" +msgstr "" + +#: osparc/po/Invitations.js +msgid "With expiration" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Trial Days" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Remember that this is a one time use link" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Copy invitation link" +msgstr "" + +#: osparc/po/Invitations.js +msgid "Data encrypted in the invitation" +msgstr "" + +#: osparc/po/MessageTemplates.js +msgid "Template updated" +msgstr "" + +#: osparc/tester/ConsoleErrors.js +msgid "Search in Message" +msgstr "" + +#: osparc/tester/WebSocketMessages.js +msgid "Channel" +msgstr "" + +#: osparc/tester/ConsoleErrors.js +msgid "Message" +msgstr "" + +#: osparc/tester/Statics.js +msgid "Local Storage" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Keep your balance running smoothly by automatically setting your credits to be recharged when it runs low." +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Before the auto-recharge function can be activated you need to add your first payment method" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Recharging amount (USD)" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Monthly limit (USD)" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Maximum amount in USD charged within a natural month." +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "To disable spending limit, clear input field" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Save and close" +msgstr "" + +#: osparc/desktop/credits/AutoRecharge.js +msgid "Changes on the Auto recharge were successfully saved" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "Select a model for more details" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "This bundle contains:" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "Terms and Conditions" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "Terms and Conditions" +msgstr "" + +#: osparc/vipMarket/LicensedItemDetails.js +msgid "Available for Importing" +msgstr "" + +#: osparc/form/ColorPicker.js +msgid "Pick a color" +msgstr "" + +#: osparc/pricing/Plans.js +msgid "New Pricing Plan" +msgstr "" + +#: osparc/pricing/Plans.js +msgid "Pricing Plan Creator" +msgstr "" + +#: osparc/pricing/Plans.js +msgid "Pricing Plan Editor" +msgstr "" + +#: osparc/pricing/PlanDetails.js +msgid "Return to Pricing Plans" +msgstr "" + +#: osparc/pricing/PlanDetails.js +msgid "Pricing Units" +msgstr "" + +#: osparc/study/PricingUnitLicense.js +msgid "Duration: 1 year" +msgstr "" + +#: osparc/study/PricingUnitLicense.js +msgid "Rent" +msgstr "" + +#: osparc/study/PricingUnitLicense.js +msgid "will be available until " +msgstr "" + +#: osparc/pricing/PlanEditor.js +msgid "Pricing Plan Key" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Successfully updated" +msgstr "" + +#: osparc/pricing/UnitsList.js +msgid "New Pricing Unit" +msgstr "" + +#: osparc/pricing/UnitsList.js +msgid "Pricing Unit Creator" +msgstr "" + +#: osparc/pricing/UnitsList.js +msgid "Pricing Unit Editor" +msgstr "" + +#: osparc/pricing/ServicesList.js +msgid "Add Service" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Unit Name" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Cost per unit" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Specific info" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "CPU" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "RAM" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "VRAM" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "More Extra Info" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Number of Seats" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Default" +msgstr "" + +#: osparc/pricing/UnitEditor.js +msgid "Successfully created" +msgstr "" diff --git a/services/static-webserver/client/source/translation/readme.txt b/services/static-webserver/client/source/translation/readme.txt index 08513ca8c3d..f7ff92f0c4d 100644 --- a/services/static-webserver/client/source/translation/readme.txt +++ b/services/static-webserver/client/source/translation/readme.txt @@ -1,4 +1,4 @@ This directory will contain translation (.po) files once you run the 'translation' job in your project. -NOTE: for the moment all translation files are in the .gitignore +Add more "locales" entries in compile.json to generate their .po files diff --git a/services/storage/.env-devel b/services/storage/.env-devel index b27bc54c999..96d465822d3 100644 --- a/services/storage/.env-devel +++ b/services/storage/.env-devel @@ -1,31 +1,20 @@ -# Environment used to configure storage services -# -# - To expose in cmd: export $(grep -v '^#' .env-devel | xargs -0) -# +CELERY_RESULT_EXPIRES=P7D -# environs in Dockerfile ---------------- -SC_BOOT_MODE=local-development +RABBIT_HOST=rabbit +RABBIT_PASSWORD=adminadmin +RABBIT_PORT=5672 +RABBIT_SECURE=false +RABBIT_USER=admin +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=adminadmin +REDIS_SECURE=false +REDIS_USER=null -# environs in docker-compose.yml -------- -POSTGRES_DB=testdb -POSTGRES_ENDPOINT=localhost:5432 -POSTGRES_HOST=localhost -POSTGRES_PASSWORD=secret -POSTGRES_PORT=5432 -POSTGRES_USER=test - +STORAGE_ENDPOINT=storage:8080 +STORAGE_HOST=storage STORAGE_LOGLEVEL=INFO -STORAGE_MONITORING_ENABLED=False - -S3_ACCESS_KEY=12345678 -S3_BUCKET_NAME=simcore -# 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface. -S3_ENDPOINT=http://172.17.0.1:9001 -S3_SECRET_KEY=12345678 -S3_REGION=us-east-1 - -BF_API_SECRET=none -BF_API_KEY=none - -TRACING_OBSERVABILITY_BACKEND_ENDPOINT=http://jaeger:9411 +STORAGE_PORT=8080 +STORAGE_PROFILING=1 +STORAGE_TRACING=null diff --git a/services/storage/Makefile b/services/storage/Makefile index 87234a46b62..ef350cae091 100644 --- a/services/storage/Makefile +++ b/services/storage/Makefile @@ -4,27 +4,15 @@ include ../../scripts/common.Makefile include ../../scripts/common-service.Makefile -APP_OPENAPI_RELPATH=src/$(APP_PACKAGE_NAME)/api/v0/openapi.yaml - -.PHONY: openapi-specs -openapi-specs: ## updates and validates openapi specifications - $(MAKE_C) $(REPO_BASE_DIR)/api/specs/$(APP_NAME) all - # validates OAS file: $(APP_OPENAPI_RELPATH) - $(call validate_openapi_specs,$(APP_OPENAPI_RELPATH)) - - -.PHONY: tests -tests: ## runs unit tests - # running unit tests - @pytest -vv --asyncio-mode=auto --failed-first --durations=10 --pdb $(CURDIR)/tests - - -# DEVELOPMENT ######## -.env: - cp .env-devel $@ - -run-devel: .env - # start app (within $<) in prod mode - export $(shell grep -v '^#' .env-devel | xargs -0); \ - $(APP_CLI_NAME) +.PHONY: openapi.json +openapi-specs: openapi.json +openapi.json: .env + # generating openapi specs file (need to have the environment set for this) + @set -o allexport; \ + source $<; \ + set +o allexport; \ + python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(app.openapi(), indent=2) )" > $@ + + # validates OAS file: $@ + $(call validate_openapi_specs,$@) diff --git a/services/storage/VERSION b/services/storage/VERSION index 8f0916f768f..a918a2aa18d 100644 --- a/services/storage/VERSION +++ b/services/storage/VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index ea9bf8bd877..4cc5128b43a 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -6,11 +6,16 @@ IFS=$(printf '\n\t') INFO="INFO: [$(basename "$0")] " -# BOOTING application --------------------------------------------- echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." echo "$INFO" "User :$(id "$(whoami)")" echo "$INFO" "Workdir : $(pwd)" +# +# DEVELOPMENT MODE +# +# - prints environ info +# - installs requirements in mounted volume +# if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "Environment :" printenv | sed 's/=/: /' | sed 's/^/ /' | sort @@ -19,35 +24,62 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/storage - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list - - echo "$INFO" "Setting entrypoint to use watchmedo autorestart..." - entrypoint='watchmedo auto-restart --recursive --pattern="*.py;*/src/*" --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" --ignore-directories --' - -elif [ "${SC_BUILD_TARGET}" = "production" ]; then - entrypoint="" fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi +# +# RUNNING application +# APP_LOG_LEVEL=${STORAGE_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') - -# RUNNING application ---------------------------------------- -echo "$INFO" "Selected config ${SC_BUILD_TARGET}" echo "$INFO" "Log-level app/server: $APP_LOG_LEVEL/$SERVER_LOG_LEVEL" -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: needs debupgy installed - echo "$INFO" "Debugpy initializing in port ${STORAGE_REMOTE_DEBUGGING_PORT} with ${SC_BUILD_TARGET}" - eval "$entrypoint" python3 -m debugpy --listen 0.0.0.0:"${STORAGE_REMOTE_DEBUGGING_PORT}" -m \ - simcore_service_storage run +if [ "${STORAGE_WORKER_MODE}" = "true" ]; then + if [ "${SC_BOOT_MODE}" = "debug" ]; then + exec watchmedo auto-restart \ + --directory /devel/packages \ + --directory services/storage \ + --pattern "*.py" \ + --recursive \ + -- \ + celery \ + --app=simcore_service_storage.modules.celery.worker_main:app \ + worker --pool=threads \ + --loglevel="${SERVER_LOG_LEVEL}" \ + --concurrency="${CELERY_CONCURRENCY}" + else + exec celery \ + --app=simcore_service_storage.modules.celery.worker_main:app \ + worker --pool=threads \ + --loglevel="${SERVER_LOG_LEVEL}" \ + --concurrency="${CELERY_CONCURRENCY}" + fi else - exec simcore-service-storage run + if [ "${SC_BOOT_MODE}" = "debug" ]; then + reload_dir_packages=$(find /devel/packages -maxdepth 3 -type d -path "*/src/*" ! -path "*.*" -exec echo '--reload-dir {} \' \;) + + exec sh -c " + cd services/storage/src/simcore_service_storage && \ + python -m debugpy --listen 0.0.0.0:${STORAGE_REMOTE_DEBUGGING_PORT} -m uvicorn main:app \ + --host 0.0.0.0 \ + --port ${STORAGE_PORT} \ + --reload \ + $reload_dir_packages + --reload-dir . \ + --log-level \"${SERVER_LOG_LEVEL}\" + " + else + exec uvicorn simcore_service_storage.main:app \ + --host 0.0.0.0 \ + --port ${STORAGE_PORT} \ + --log-level "${SERVER_LOG_LEVEL}" + fi fi diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py old mode 100644 new mode 100755 index b6711cd55eb..d938c860dab --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -20,18 +20,50 @@ import os +import socket +import subprocess import sys from urllib.request import urlopen +from simcore_service_storage.core.settings import ApplicationSettings + SUCCESS, UNHEALTHY = 0, 1 # Disabled if boots with debugger -ok = os.environ.get("SC_BOOT_MODE", "").lower() == "debug" +ok = os.getenv("SC_BOOT_MODE", "").lower() == "debug" # Queries host # pylint: disable=consider-using-with + +app_settings = ApplicationSettings.create_from_envs() + +def _is_celery_worker_healthy(): + assert app_settings.STORAGE_CELERY + broker_url = app_settings.STORAGE_CELERY.CELERY_RABBIT_BROKER.dsn + + try: + result = subprocess.run( + [ + "celery", + "--broker", + broker_url, + "inspect", + "ping", + "--destination", + "celery@" + socket.gethostname(), + ], + capture_output=True, + text=True, + check=True, + ) + return "pong" in result.stdout + except subprocess.CalledProcessError: + return False + + ok = ( ok + or (app_settings.STORAGE_WORKER_MODE and _is_celery_worker_healthy()) or urlopen( "{host}{baseurl}".format( host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") diff --git a/services/storage/openapi.json b/services/storage/openapi.json new file mode 100644 index 00000000000..2c5676b08bc --- /dev/null +++ b/services/storage/openapi.json @@ -0,0 +1,2730 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "simcore_service_storage", + "description": "Service that manages osparc storage backend", + "version": "0.6.0" + }, + "paths": { + "/v0/": { + "get": { + "tags": [ + "status" + ], + "summary": "Get Health", + "operationId": "get_health_v0__get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_HealthCheck_" + } + } + } + } + } + } + }, + "/v0/status": { + "get": { + "tags": [ + "status" + ], + "summary": "Get Status", + "operationId": "get_status_v0_status_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_AppStatusCheck_" + } + } + } + } + } + } + }, + "/v0/locations": { + "get": { + "tags": [ + "locations" + ], + "summary": "List Storage Locations", + "operationId": "list_storage_locations_v0_locations_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_list_FileLocation__" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/datasets": { + "get": { + "tags": [ + "datasets" + ], + "summary": "List Datasets Metadata", + "operationId": "list_datasets_metadata_v0_locations__location_id__datasets_get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_list_DatasetMetaDataGet__" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/datasets/{dataset_id}/metadata": { + "get": { + "tags": [ + "datasets" + ], + "summary": "List Dataset Files Metadata", + "operationId": "list_dataset_files_metadata_v0_locations__location_id__datasets__dataset_id__metadata_get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "dataset_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Dataset Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "expand_dirs", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true, + "title": "Expand Dirs" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_list_FileMetaDataGet__" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/metadata": { + "get": { + "tags": [ + "files" + ], + "summary": "List Files Metadata", + "operationId": "list_files_metadata_v0_locations__location_id__files_metadata_get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "project_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Project Id" + } + }, + { + "name": "uuid_filter", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "", + "title": "Uuid Filter" + } + }, + { + "name": "expand_dirs", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true, + "title": "Expand Dirs" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_list_FileMetaDataGet__" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/{file_id}/metadata": { + "get": { + "tags": [ + "files" + ], + "summary": "Get File Metadata", + "operationId": "get_file_metadata_v0_locations__location_id__files__file_id__metadata_get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "user-agent", + "in": "header", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User-Agent" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/Envelope_FileMetaDataGet_" + }, + { + "$ref": "#/components/schemas/Envelope_FileMetaDataGetv010_" + }, + { + "$ref": "#/components/schemas/Envelope_dict_" + } + ], + "title": "Response Get File Metadata V0 Locations Location Id Files File Id Metadata Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/{file_id}": { + "get": { + "tags": [ + "files" + ], + "summary": "Download File", + "operationId": "download_file_v0_locations__location_id__files__file_id__get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "link_type", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/LinkType", + "default": "PRESIGNED" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_FileDownloadResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "put": { + "tags": [ + "files" + ], + "summary": "Upload File", + "description": "creates upload file links:\n\nThis function covers v1 and v2 versions of the handler.\nNote: calling this entrypoint on an already existing file will overwrite that file. That file will be deleted\nbefore the upload takes place.\n\nv1 rationale:\n - client calls this handler, which returns a single link (either direct S3 or presigned) to the S3 backend\n - client uploads the file\n - storage relies on lazy update to find if the file is finished uploaded (when client calls get_file_meta_data, or if the dsm_cleaner goes over it after the upload time is expired)\n\nv2 rationale:\n - client calls this handler, which returns a FileUploadSchema object containing 1 or more links (either S3/presigned links)\n - client uploads the file (by chunking it if there are more than 1 presigned link)\n - client calls complete_upload handle which will reconstruct the file on S3 backend\n - client waits for completion to finish and then the file is accessible on S3 backend\n\n\nUse-case v1: query.file_size is not defined, returns a PresignedLink model (backward compatibility)\nUse-case v1.1: if query.link_type=presigned or None, returns a presigned link (limited to a single 5GB file)\nUse-case v1.2: if query.link_type=s3, returns a s3 direct link (limited to a single 5TB file)\n\nUser-case v2: query.is_directory is True (query.file_size is forced to -1), returns an s3 path where to upload all the content of the directory\nUser-case v2: if query.file_size is defined, returns a FileUploadSchema model, expects client to call \"complete_upload\" when the file is finished uploading\nUse-case v2.1: if query.file_size == 0 and query.link_type=presigned or None, returns a single presigned link inside FileUploadSchema (limited to a single 5Gb file)\nUse-case v2.2: if query.file_size > 0 and query.link_type=presigned or None, returns 1 or more presigned links depending on the file size (limited to a single 5TB file)\nUse-case v2.3: if query.link_type=s3 and query.file_size>=0, returns a single s3 direct link (limited to a single 5TB file)", + "operationId": "upload_file_v0_locations__location_id__files__file_id__put", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "link_type", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/LinkType", + "default": "PRESIGNED" + } + }, + { + "name": "file_size", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*(\\d*\\.?\\d+)\\s*(\\w+)?" + }, + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "File Size" + } + }, + { + "name": "is_directory", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": false, + "title": "Is Directory" + } + }, + { + "name": "sha256_checksum", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "type": "null" + } + ], + "title": "Sha256 Checksum" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/Envelope_FileUploadResponseV1_" + }, + { + "$ref": "#/components/schemas/Envelope_FileUploadSchema_" + } + ], + "title": "Response Upload File V0 Locations Location Id Files File Id Put" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "files" + ], + "summary": "Delete File", + "operationId": "delete_file_v0_locations__location_id__files__file_id__delete", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/{file_id}:abort": { + "post": { + "tags": [ + "files" + ], + "summary": "Abort Upload File", + "operationId": "abort_upload_file_v0_locations__location_id__files__file_id__abort_post", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/{file_id}:complete": { + "post": { + "tags": [ + "files" + ], + "summary": "Complete Upload File", + "operationId": "complete_upload_file_v0_locations__location_id__files__file_id__complete_post", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FileUploadCompletionBody" + } + } + } + }, + "responses": { + "202": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_FileUploadCompleteResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/files/{file_id}:complete/futures/{future_id}": { + "post": { + "tags": [ + "files" + ], + "summary": "Is Completed Upload File", + "operationId": "is_completed_upload_file_v0_locations__location_id__files__file_id__complete_futures__future_id__post", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "future_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Future Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_FileUploadCompleteFutureResponse_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/files/{file_id}:soft-copy": { + "post": { + "tags": [ + "files" + ], + "summary": "Copy As Soft Link", + "operationId": "copy_as_soft_link_v0_files__file_id__soft_copy_post", + "parameters": [ + { + "name": "file_id", + "in": "path", + "required": true, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SoftCopyBody" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_FileMetaDataGet_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/paths": { + "get": { + "tags": [ + "files" + ], + "summary": "List Paths", + "description": "Returns one level of files (paginated)", + "operationId": "list_paths_v0_locations__location_id__paths_get", + "parameters": [ + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "file_filter", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "path" + }, + { + "type": "null" + } + ], + "title": "File Filter" + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cursor" + } + }, + { + "name": "size", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 1000, + "minimum": 1, + "default": 50, + "title": "Size" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/locations/{location_id}/paths/{path}:size": { + "post": { + "tags": [ + "files" + ], + "summary": "Compute Path Size", + "operationId": "compute_path_size_v0_locations__location_id__paths__path__size_post", + "parameters": [ + { + "name": "path", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "path", + "title": "Path" + } + }, + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_PathTotalSizeCreate_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/simcore-s3:access": { + "post": { + "tags": [ + "simcore-s3" + ], + "summary": "Get Or Create Temporary S3 Access", + "operationId": "get_or_create_temporary_s3_access_v0_simcore_s3_access_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_S3Settings_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/simcore-s3/folders": { + "post": { + "tags": [ + "simcore-s3" + ], + "summary": "Copy Folders From Project", + "operationId": "copy_folders_from_project_v0_simcore_s3_folders_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FoldersBody" + } + } + } + }, + "responses": { + "202": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_TaskGet_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/simcore-s3/folders/{folder_id}": { + "delete": { + "tags": [ + "simcore-s3" + ], + "summary": "Delete Folders Of Project", + "operationId": "delete_folders_of_project_v0_simcore_s3_folders__folder_id__delete", + "parameters": [ + { + "name": "folder_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Folder Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "node_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Node Id" + } + } + ], + "responses": { + "204": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/simcore-s3/files/metadata:search": { + "post": { + "tags": [ + "simcore-s3" + ], + "summary": "Search Files", + "operationId": "search_files_v0_simcore_s3_files_metadata_search_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + }, + { + "name": "startswith", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Startswith" + } + }, + { + "name": "sha256_checksum", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "type": "null" + } + ], + "title": "Sha256 Checksum" + } + }, + { + "name": "kind", + "in": "query", + "required": true, + "schema": { + "const": "owned", + "type": "string", + "title": "Kind" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_list_FileMetaDataGet__" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "AppStatusCheck": { + "properties": { + "app_name": { + "type": "string", + "title": "App Name", + "description": "Application name" + }, + "version": { + "type": "string", + "title": "Version", + "description": "Application's version" + }, + "services": { + "type": "object", + "title": "Services", + "description": "Other backend services connected from this service", + "default": {} + }, + "sessions": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Sessions", + "description": "Client sessions info. If single session per app, then is denoted as main", + "default": {} + }, + "url": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "Url", + "description": "Link to current resource" + }, + "diagnostics_url": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "Diagnostics Url", + "description": "Link to diagnostics report sub-resource. This MIGHT take some time to compute" + } + }, + "type": "object", + "required": [ + "app_name", + "version" + ], + "title": "AppStatusCheck" + }, + "CursorPage___T_Customized_PathMetaDataGet_": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/PathMetaDataGet" + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Total", + "description": "Total items" + }, + "current_page": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Current Page", + "description": "Cursor to refetch the current page" + }, + "current_page_backwards": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Current Page Backwards", + "description": "Cursor to refetch the current page starting from the last item" + }, + "previous_page": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Previous Page", + "description": "Cursor for the previous page" + }, + "next_page": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next Page", + "description": "Cursor for the next page" + } + }, + "type": "object", + "required": [ + "items" + ], + "title": "CursorPage[~_T]Customized[PathMetaDataGet]" + }, + "DatasetMetaDataGet": { + "properties": { + "dataset_id": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "string", + "pattern": "^N:dataset:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "Dataset Id" + }, + "display_name": { + "type": "string", + "title": "Display Name" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "dataset_id", + "display_name" + ], + "title": "DatasetMetaDataGet" + }, + "Envelope_AppStatusCheck_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/AppStatusCheck" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[AppStatusCheck]" + }, + "Envelope_FileDownloadResponse_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileDownloadResponse" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileDownloadResponse]" + }, + "Envelope_FileMetaDataGet_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileMetaDataGet" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileMetaDataGet]" + }, + "Envelope_FileMetaDataGetv010_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileMetaDataGetv010" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileMetaDataGetv010]" + }, + "Envelope_FileUploadCompleteFutureResponse_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileUploadCompleteFutureResponse" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileUploadCompleteFutureResponse]" + }, + "Envelope_FileUploadCompleteResponse_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileUploadCompleteResponse" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileUploadCompleteResponse]" + }, + "Envelope_FileUploadResponseV1_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileUploadResponseV1" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileUploadResponseV1]" + }, + "Envelope_FileUploadSchema_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileUploadSchema" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[FileUploadSchema]" + }, + "Envelope_HealthCheck_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/HealthCheck" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[HealthCheck]" + }, + "Envelope_PathTotalSizeCreate_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/PathTotalSizeCreate" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[PathTotalSizeCreate]" + }, + "Envelope_S3Settings_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/S3Settings" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[S3Settings]" + }, + "Envelope_TaskGet_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/TaskGet" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[TaskGet]" + }, + "Envelope_dict_": { + "properties": { + "data": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Data" + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[dict]" + }, + "Envelope_list_DatasetMetaDataGet__": { + "properties": { + "data": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/DatasetMetaDataGet" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Data" + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[list[DatasetMetaDataGet]]" + }, + "Envelope_list_FileLocation__": { + "properties": { + "data": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/FileLocation" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Data" + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[list[FileLocation]]" + }, + "Envelope_list_FileMetaDataGet__": { + "properties": { + "data": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/FileMetaDataGet" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Data" + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[list[FileMetaDataGet]]" + }, + "FileDownloadResponse": { + "properties": { + "link": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Link" + } + }, + "type": "object", + "required": [ + "link" + ], + "title": "FileDownloadResponse" + }, + "FileLocation": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "id": { + "type": "integer", + "title": "Id" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "name", + "id" + ], + "title": "FileLocation" + }, + "FileMetaDataGet": { + "properties": { + "file_uuid": { + "type": "string", + "title": "File Uuid", + "description": "NOT a unique ID, like (api|uuid)/uuid/file_name or DATCORE folder structure" + }, + "location_id": { + "type": "integer", + "title": "Location Id", + "description": "Storage location" + }, + "project_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Project Name", + "description": "optional project name, used by frontend to display path" + }, + "node_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Node Name", + "description": "optional node name, used by frontend to display path" + }, + "file_name": { + "type": "string", + "title": "File Name", + "description": "Display name for a file" + }, + "file_id": { + "anyOf": [ + { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$" + }, + { + "type": "string", + "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" + } + ], + "title": "File Id", + "description": "THIS IS the unique ID for the file. either (api|project_id)/node_id/file_name.ext for S3 and N:package:UUID for datcore" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At" + }, + "last_modified": { + "type": "string", + "format": "date-time", + "title": "Last Modified" + }, + "file_size": { + "anyOf": [ + { + "type": "integer", + "const": -1 + }, + { + "type": "integer", + "minimum": 0 + } + ], + "title": "File Size", + "description": "File size in bytes (-1 means invalid)", + "default": -1 + }, + "entity_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Entity Tag", + "description": "Entity tag (or ETag), represents a specific version of the file, None if invalid upload or datcore" + }, + "is_soft_link": { + "type": "boolean", + "title": "Is Soft Link", + "description": "If true, this file is a soft link.i.e. is another entry with the same object_name", + "default": false + }, + "is_directory": { + "type": "boolean", + "title": "Is Directory", + "description": "if True this is a directory", + "default": false + }, + "sha256_checksum": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "type": "null" + } + ], + "title": "Sha256 Checksum", + "description": "SHA256 message digest of the file content. Main purpose: cheap lookup." + } + }, + "type": "object", + "required": [ + "file_uuid", + "location_id", + "file_name", + "file_id", + "created_at", + "last_modified" + ], + "title": "FileMetaDataGet" + }, + "FileMetaDataGetv010": { + "properties": { + "file_uuid": { + "type": "string", + "title": "File Uuid" + }, + "location_id": { + "type": "integer", + "title": "Location Id" + }, + "location": { + "type": "string", + "title": "Location" + }, + "bucket_name": { + "type": "string", + "title": "Bucket Name" + }, + "object_name": { + "type": "string", + "title": "Object Name" + }, + "project_id": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Project Id" + }, + "project_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Project Name" + }, + "node_id": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Node Id" + }, + "node_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Node Name" + }, + "file_name": { + "type": "string", + "title": "File Name" + }, + "user_id": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "User Id" + }, + "user_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User Name" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "file_uuid", + "location_id", + "location", + "bucket_name", + "object_name", + "project_id", + "project_name", + "node_id", + "node_name", + "file_name", + "user_id", + "user_name" + ], + "title": "FileMetaDataGetv010" + }, + "FileUploadCompleteFutureResponse": { + "properties": { + "state": { + "$ref": "#/components/schemas/FileUploadCompleteState" + }, + "e_tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "E Tag" + } + }, + "type": "object", + "required": [ + "state" + ], + "title": "FileUploadCompleteFutureResponse" + }, + "FileUploadCompleteLinks": { + "properties": { + "state": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "State" + } + }, + "type": "object", + "required": [ + "state" + ], + "title": "FileUploadCompleteLinks" + }, + "FileUploadCompleteResponse": { + "properties": { + "links": { + "$ref": "#/components/schemas/FileUploadCompleteLinks" + } + }, + "type": "object", + "required": [ + "links" + ], + "title": "FileUploadCompleteResponse" + }, + "FileUploadCompleteState": { + "type": "string", + "enum": [ + "ok", + "nok" + ], + "title": "FileUploadCompleteState" + }, + "FileUploadCompletionBody": { + "properties": { + "parts": { + "items": { + "$ref": "#/components/schemas/UploadedPart" + }, + "type": "array", + "title": "Parts" + } + }, + "type": "object", + "required": [ + "parts" + ], + "title": "FileUploadCompletionBody" + }, + "FileUploadLinks": { + "properties": { + "abort_upload": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Abort Upload" + }, + "complete_upload": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Complete Upload" + } + }, + "type": "object", + "required": [ + "abort_upload", + "complete_upload" + ], + "title": "FileUploadLinks" + }, + "FileUploadResponseV1": { + "properties": { + "link": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Link" + } + }, + "type": "object", + "required": [ + "link" + ], + "title": "FileUploadResponseV1" + }, + "FileUploadSchema": { + "properties": { + "chunk_size": { + "type": "integer", + "minimum": 0, + "title": "Chunk Size" + }, + "urls": { + "items": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "type": "array", + "title": "Urls" + }, + "links": { + "$ref": "#/components/schemas/FileUploadLinks" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "chunk_size", + "urls", + "links" + ], + "title": "FileUploadSchema" + }, + "FoldersBody": { + "properties": { + "source": { + "type": "object", + "title": "Source" + }, + "destination": { + "type": "object", + "title": "Destination" + }, + "nodes_map": { + "additionalProperties": { + "type": "string", + "format": "uuid" + }, + "propertyNames": { + "format": "uuid" + }, + "type": "object", + "title": "Nodes Map" + } + }, + "type": "object", + "title": "FoldersBody" + }, + "HTTPValidationError": { + "properties": { + "errors": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Validation errors" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "HealthCheck": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + }, + "api_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Version" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "type": "object", + "required": [ + "name", + "status", + "api_version", + "version" + ], + "title": "HealthCheck" + }, + "LinkType": { + "type": "string", + "enum": [ + "PRESIGNED", + "S3" + ], + "title": "LinkType" + }, + "PathMetaDataGet": { + "properties": { + "path": { + "type": "string", + "format": "path", + "title": "Path", + "description": "the path to the current path" + }, + "display_path": { + "type": "string", + "format": "path", + "title": "Display Path", + "description": "the path to display with UUID replaced (URL Encoded by parts as names may contain '/')" + }, + "file_meta_data": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileMetaDataGet" + }, + { + "type": "null" + } + ], + "description": "if filled, this is the file meta data of the s3 object" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "path", + "display_path" + ], + "title": "PathMetaDataGet" + }, + "PathTotalSizeCreate": { + "properties": { + "path": { + "type": "string", + "format": "path", + "title": "Path" + }, + "size": { + "type": "integer", + "minimum": 0, + "title": "Size" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "path", + "size" + ], + "title": "PathTotalSizeCreate" + }, + "S3Settings": { + "properties": { + "S3_ACCESS_KEY": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "S3 Access Key" + }, + "S3_BUCKET_NAME": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "S3 Bucket Name" + }, + "S3_ENDPOINT": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "S3 Endpoint", + "description": "do not define if using standard AWS" + }, + "S3_REGION": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "S3 Region" + }, + "S3_SECRET_KEY": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "S3 Secret Key" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "S3_ACCESS_KEY", + "S3_BUCKET_NAME", + "S3_REGION", + "S3_SECRET_KEY" + ], + "title": "S3Settings" + }, + "SoftCopyBody": { + "properties": { + "link_id": { + "type": "string", + "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$", + "title": "Link Id" + } + }, + "type": "object", + "required": [ + "link_id" + ], + "title": "SoftCopyBody" + }, + "TaskGet": { + "properties": { + "task_id": { + "type": "string", + "title": "Task Id" + }, + "task_name": { + "type": "string", + "title": "Task Name" + }, + "status_href": { + "type": "string", + "title": "Status Href" + }, + "result_href": { + "type": "string", + "title": "Result Href" + }, + "abort_href": { + "type": "string", + "title": "Abort Href" + } + }, + "type": "object", + "required": [ + "task_id", + "task_name", + "status_href", + "result_href", + "abort_href" + ], + "title": "TaskGet" + }, + "UploadedPart": { + "properties": { + "number": { + "type": "integer", + "exclusiveMinimum": true, + "title": "Number", + "minimum": 0 + }, + "e_tag": { + "type": "string", + "title": "E Tag" + } + }, + "type": "object", + "required": [ + "number", + "e_tag" + ], + "title": "UploadedPart" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + } + } + } +} diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index 5b1c3f13098..cf0ccfdba89 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -11,17 +11,22 @@ --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in -# service-library[aiohttp] --requirement ../../../packages/service-library/requirements/_base.in ---requirement ../../../packages/service-library/requirements/_aiohttp.in +--requirement ../../../packages/service-library/requirements/_fastapi.in aioboto3 # s3 storage aiofiles # i/o -aiohttp # server -aiohttp-swagger[performance] # server -aiopg[sa] # db +asgi_lifespan +asyncpg # database +celery[redis] +httpx opentelemetry-instrumentation-botocore packaging +fastapi[all] +fastapi-pagination +orjson +pydantic[dotenv] tenacity typer types-aiobotocore[s3] # s3 storage +watchdog diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index df1f0513a0e..99c101b84d6 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -1,14 +1,14 @@ -aio-pika==9.4.1 +aio-pika==9.5.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==13.1.0 +aioboto3==13.3.0 # via # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/_base.in -aiobotocore==2.13.1 +aiobotocore==2.16.0 # via aioboto3 -aiocache==0.12.2 +aiocache==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in @@ -17,17 +17,19 @@ aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in # aioboto3 -aiohttp==3.9.3 +aiohappyeyeballs==2.4.6 + # via aiohttp +aiohttp==3.11.12 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -55,31 +57,27 @@ aiohttp==3.9.3 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/service-library/requirements/_aiohttp.in - # -r requirements/_base.in # aiobotocore # aiodocker - # aiohttp-swagger -aiohttp-swagger==1.0.16 - # via -r requirements/_base.in -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiopg==1.4.0 - # via - # -r requirements/../../../packages/service-library/requirements/_aiohttp.in - # -r requirements/_base.in -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp -alembic==1.13.1 +alembic==1.14.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +amqp==5.3.1 + # via kombu annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.8.0 # via # fast-depends # faststream + # httpx + # starlette + # watchfiles arrow==1.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in @@ -89,28 +87,60 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -async-timeout==4.0.3 +asgi-lifespan==2.1.0 + # via -r requirements/_base.in +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +asyncpg==0.30.0 # via - # aiopg - # asyncpg -asyncpg==0.29.0 - # via sqlalchemy -attrs==23.2.0 + # -r requirements/_base.in + # sqlalchemy +attrs==25.1.0 # via - # -r requirements/../../../packages/service-library/requirements/_aiohttp.in # aiohttp # jsonschema # referencing -boto3==1.34.131 - # via aiobotocore -botocore==1.34.131 +billiard==4.2.1 + # via celery +boto3==1.35.81 + # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # aiobotocore +botocore==1.35.81 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.34.69 +botocore-stubs==1.36.17 # via types-aiobotocore -certifi==2024.2.2 +celery==5.4.0 + # via -r requirements/_base.in +certifi==2025.1.31 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -138,48 +168,120 @@ certifi==2024.2.2 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # httpcore + # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.1 # via requests -click==8.1.7 - # via typer -deprecated==1.2.14 +click==8.1.8 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # rich-toolkit + # typer + # uvicorn +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery +deprecated==1.2.18 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 - # via pydantic +email-validator==2.2.0 + # via + # fastapi + # pydantic +exceptiongroup==1.2.2 + # via aio-pika fast-depends==2.4.12 # via faststream -faststream==0.5.31 +fastapi==0.115.8 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # -r requirements/_base.in + # fastapi-lifespan-manager +fastapi-cli==0.0.7 + # via fastapi +fastapi-lifespan-manager==0.1.4 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +fastapi-pagination==0.12.34 + # via -r requirements/_base.in +faststream==0.5.34 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -grpcio==1.66.0 +grpcio==1.70.0 # via opentelemetry-exporter-otlp-proto-grpc -idna==3.6 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.28.1 + # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # -r requirements/_base.in + # fastapi +idna==3.10 # via # anyio # email-validator + # httpx # requests # yarl -importlib-metadata==8.0.0 +importlib-metadata==8.5.0 # via opentelemetry-api -jinja2==3.1.3 +itsdangerous==2.2.0 + # via fastapi +jinja2==3.1.5 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -207,21 +309,22 @@ jinja2==3.1.3 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # aiohttp-swagger + # fastapi jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../packages/service-library/requirements/_aiohttp.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via jsonschema -mako==1.3.2 +kombu==5.4.2 + # via celery +mako==1.3.9 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -252,114 +355,111 @@ mako==1.3.2 # alembic markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # jinja2 # mako - # werkzeug mdurl==0.1.2 # via markdown-it-py -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation - # opentelemetry-instrumentation-aiohttp-client - # opentelemetry-instrumentation-aiohttp-server - # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore - # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.30.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.30.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.30.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.51b0 # via - # opentelemetry-instrumentation-aiohttp-client - # opentelemetry-instrumentation-aiohttp-server - # opentelemetry-instrumentation-aiopg + # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore - # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-aiohttp-client==0.47b0 - # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in -opentelemetry-instrumentation-aiohttp-server==0.47b0 - # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in -opentelemetry-instrumentation-aiopg==0.47b0 - # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in -opentelemetry-instrumentation-asyncpg==0.47b0 +opentelemetry-instrumentation-asgi==0.51b0 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.51b0 # via -r requirements/../../../packages/postgres-database/requirements/_base.in -opentelemetry-instrumentation-botocore==0.47b0 +opentelemetry-instrumentation-botocore==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/_base.in -opentelemetry-instrumentation-dbapi==0.47b0 - # via opentelemetry-instrumentation-aiopg -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-fastapi==0.51b0 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-httpx==0.51b0 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-propagator-aws-xray==1.0.1 +opentelemetry-propagator-aws-xray==1.0.2 # via opentelemetry-instrumentation-botocore -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.30.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.51b0 # via - # opentelemetry-instrumentation-aiohttp-client - # opentelemetry-instrumentation-aiohttp-server + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore - # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.51b0 # via - # opentelemetry-instrumentation-aiohttp-client - # opentelemetry-instrumentation-aiohttp-server + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-requests -orjson==3.10.0 +orjson==3.10.15 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -404,25 +504,39 @@ orjson==3.10.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 - # via -r requirements/_base.in + # -r requirements/_base.in + # fastapi +packaging==24.2 + # via + # -r requirements/_base.in + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq -prometheus-client==0.20.0 - # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in -protobuf==4.25.4 +prometheus-client==0.21.1 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==7.0.2 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +prompt-toolkit==3.0.50 + # via click-repl +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.3 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -psycopg2-binary==2.9.9 - # via - # aiopg - # sqlalchemy -pydantic==2.10.2 +psycopg2-binary==2.9.10 + # via sqlalchemy +pycryptodome==3.21.0 + # via stream-zip +pydantic==2.10.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -475,12 +589,15 @@ pydantic==2.10.2 # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/_base.in # fast-depends + # fastapi + # fastapi-pagination # pydantic-extra-types # pydantic-settings -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -499,8 +616,35 @@ pydantic-extra-types==2.9.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -pydantic-settings==2.6.1 + # fastapi +pydantic-settings==2.7.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -509,9 +653,10 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -pygments==2.17.2 + # fastapi +pygments==2.19.1 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -519,9 +664,14 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore + # celery python-dotenv==1.0.1 - # via pydantic-settings -pyyaml==6.0.1 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.20 + # via fastapi +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -551,7 +701,8 @@ pyyaml==6.0.1 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in - # aiohttp-swagger + # fastapi + # uvicorn redis==5.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -582,36 +733,66 @@ redis==5.2.1 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -referencing==0.29.3 + # celery +referencing==0.35.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt # jsonschema # jsonschema-specifications -requests==2.32.2 +requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # rich-toolkit # typer -rpds-py==0.18.0 +rich-toolkit==0.13.2 + # via fastapi-cli +rpds-py==0.22.3 # via # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via boto3 -setuptools==74.0.0 - # via opentelemetry-instrumentation -sh==2.0.6 +sh==2.2.1 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 - # via anyio -sqlalchemy==1.4.52 + # via + # anyio + # asgi-lifespan +sqlalchemy==1.4.54 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -640,57 +821,99 @@ sqlalchemy==1.4.52 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in - # aiopg # alembic -tenacity==8.5.0 +starlette==0.45.3 + # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # fastapi + # prometheus-fastapi-instrumentator +stream-zip==0.0.83 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in +tenacity==9.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -tqdm==4.66.2 +tqdm==4.67.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.15.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in -types-aiobotocore==2.12.1 + # fastapi-cli +types-aiobotocore==2.19.0 # via # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/_base.in -types-aiobotocore-ec2==2.12.3 +types-aiobotocore-ec2==2.19.0 # via types-aiobotocore -types-aiobotocore-s3==2.12.1 +types-aiobotocore-s3==2.19.0.post1 # via types-aiobotocore -types-aiobotocore-ssm==2.12.3 +types-aiobotocore-ssm==2.19.0 # via types-aiobotocore -types-awscrt==0.20.5 +types-awscrt==0.23.10 # via botocore-stubs -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # aiodebug - # aiodocker # alembic + # anyio + # fastapi + # fastapi-pagination # faststream # opentelemetry-sdk # pydantic # pydantic-core + # pydantic-extra-types + # rich-toolkit # typer # types-aiobotocore # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -ujson==5.9.0 +tzdata==2025.1 + # via + # celery + # kombu +ujson==5.10.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -718,8 +941,8 @@ ujson==5.9.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # aiohttp-swagger -urllib3==2.2.3 + # fastapi +urllib3==2.3.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -749,19 +972,34 @@ urllib3==2.2.3 # -c requirements/../../../requirements/constraints.txt # botocore # requests -werkzeug==3.0.2 - # via -r requirements/../../../packages/service-library/requirements/_aiohttp.in -wrapt==1.16.0 +uvicorn==0.34.0 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn +vine==5.1.0 + # via + # amqp + # celery + # kombu +watchdog==6.0.0 + # via -r requirements/_base.in +watchfiles==1.0.4 + # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +websockets==14.2 + # via uvicorn +wrapt==1.17.2 # via # aiobotocore # deprecated # opentelemetry-instrumentation - # opentelemetry-instrumentation-aiohttp-client - # opentelemetry-instrumentation-aiohttp-server - # opentelemetry-instrumentation-aiopg - # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis -yarl==1.9.4 +yarl==1.18.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -769,5 +1007,5 @@ yarl==1.9.4 # aio-pika # aiohttp # aiormq -zipp==3.20.1 +zipp==3.21.0 # via importlib-metadata diff --git a/services/storage/requirements/_test.in b/services/storage/requirements/_test.in index 33b53a495bc..1af7ed5e188 100644 --- a/services/storage/requirements/_test.in +++ b/services/storage/requirements/_test.in @@ -6,24 +6,27 @@ --constraint _base.txt -aioresponses +asyncpg-stubs coverage docker faker fakeredis[lua] +flaky jsonref moto[server] pandas pytest -pytest-aiohttp pytest-asyncio +pytest-celery pytest-cov + pytest-icdiff pytest-instafail pytest-mock pytest-runner pytest-sugar python-dotenv +respx simcore-service-storage-sdk @ git+https://github.com/ITISFoundation/osparc-simcore.git@cfdf4f86d844ebb362f4f39e9c6571d561b72897#subdirectory=services/storage/client-sdk/python # to test backwards compatibility against deprecated client-sdk (used still in old versions of simcore-sdk) sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html types-aiofiles diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 26d91990499..c804dfd46d9 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -1,111 +1,174 @@ -aiohttp==3.9.3 +aiohappyeyeballs==2.4.6 + # via + # -c requirements/_base.txt + # aiohttp +aiohttp==3.11.12 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt - # aioresponses - # pytest-aiohttp # simcore-service-storage-sdk -aioresponses==0.7.8 - # via -r requirements/_test.in -aiosignal==1.3.1 +aiosignal==1.3.2 # via # -c requirements/_base.txt # aiohttp +amqp==5.3.1 + # via + # -c requirements/_base.txt + # kombu annotated-types==0.7.0 # via # -c requirements/_base.txt # pydantic antlr4-python3-runtime==4.13.2 # via moto -attrs==23.2.0 +anyio==4.8.0 + # via + # -c requirements/_base.txt + # httpx +asyncpg==0.30.0 + # via + # -c requirements/_base.txt + # asyncpg-stubs +asyncpg-stubs==0.30.0 + # via -r requirements/_test.in +attrs==25.1.0 # via # -c requirements/_base.txt # aiohttp # jsonschema # referencing -aws-sam-translator==1.94.0 +aws-sam-translator==1.95.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto +billiard==4.2.1 + # via + # -c requirements/_base.txt + # celery blinker==1.9.0 # via flask -boto3==1.34.131 +boto3==1.35.81 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.34.131 +botocore==1.35.81 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.2.2 +celery==5.4.0 + # via + # -c requirements/_base.txt + # pytest-celery +certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt + # httpcore + # httpx # requests # simcore-service-storage-sdk cffi==1.17.1 # via cryptography -cfn-lint==1.22.7 +cfn-lint==1.27.0 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.1 # via # -c requirements/_base.txt # requests -click==8.1.7 +click==8.1.8 # via # -c requirements/_base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # flask -coverage==7.6.10 +click-didyoumean==0.3.1 + # via + # -c requirements/_base.txt + # celery +click-plugins==1.1.1 + # via + # -c requirements/_base.txt + # celery +click-repl==0.3.0 + # via + # -c requirements/_base.txt + # celery +coverage==7.6.12 # via # -r requirements/_test.in # pytest-cov -cryptography==44.0.0 +cryptography==44.0.2 # via # -c requirements/../../../requirements/constraints.txt # joserfc # moto +debugpy==1.8.12 + # via pytest-celery docker==7.1.0 # via # -r requirements/_test.in # moto -faker==35.0.0 + # pytest-celery + # pytest-docker-tools +faker==36.1.1 + # via -r requirements/_test.in +fakeredis==2.27.0 # via -r requirements/_test.in -fakeredis==2.26.2 +flaky==3.8.1 # via -r requirements/_test.in -flask==3.0.3 +flask==3.1.0 # via # flask-cors # moto -flask-cors==5.0.0 +flask-cors==5.0.1 # via moto -frozenlist==1.4.1 +frozenlist==1.5.0 # via # -c requirements/_base.txt # aiohttp # aiosignal graphql-core==3.2.6 # via moto -greenlet==3.0.3 +greenlet==3.1.1 # via # -c requirements/_base.txt # sqlalchemy +h11==0.14.0 + # via + # -c requirements/_base.txt + # httpcore +httpcore==1.0.7 + # via + # -c requirements/_base.txt + # httpx +httpx==0.28.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # respx icdiff==2.0.7 # via pytest-icdiff -idna==3.6 +idna==3.10 # via # -c requirements/_base.txt + # anyio + # httpx # requests # yarl iniconfig==2.0.0 # via pytest itsdangerous==2.2.0 - # via flask -jinja2==3.1.3 + # via + # -c requirements/_base.txt + # flask +jinja2==3.1.5 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -116,9 +179,7 @@ jmespath==1.0.1 # -c requirements/_base.txt # boto3 # botocore -joserfc==1.0.2 - # via moto -jsondiff==2.2.1 +joserfc==1.0.4 # via moto jsonpatch==1.33 # via cfn-lint @@ -128,7 +189,7 @@ jsonpointer==3.0.0 # via jsonpatch jsonref==1.1.0 # via -r requirements/_test.in -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator @@ -136,47 +197,48 @@ jsonschema==4.21.1 # openapi-spec-validator jsonschema-path==0.3.4 # via openapi-spec-validator -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via # -c requirements/_base.txt # jsonschema # openapi-schema-validator +kombu==5.4.2 + # via + # -c requirements/_base.txt + # celery lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.4 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 # werkzeug -moto==5.0.20 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_test.in +moto==5.1.1 + # via -r requirements/_test.in mpmath==1.3.0 # via sympy -multidict==6.0.5 +multidict==6.1.0 # via # -c requirements/_base.txt # aiohttp # yarl -mypy==1.14.1 +mypy==1.15.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy networkx==3.4.2 # via cfn-lint -numpy==2.2.2 +numpy==2.2.3 # via pandas openapi-schema-validator==0.6.3 # via openapi-spec-validator openapi-spec-validator==0.7.1 # via moto -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt - # aioresponses # pytest # pytest-sugar pandas==2.2.3 @@ -189,40 +251,54 @@ ply==3.11 # via jsonpath-ng pprintpp==0.4.0 # via pytest-icdiff -py-partiql-parser==0.5.6 +prompt-toolkit==3.0.50 + # via + # -c requirements/_base.txt + # click-repl +propcache==0.2.1 + # via + # -c requirements/_base.txt + # aiohttp + # yarl +psutil==6.1.1 + # via + # -c requirements/_base.txt + # pytest-celery +py-partiql-parser==0.6.1 # via moto pycparser==2.22 # via cffi -pydantic==2.10.2 +pydantic==2.10.6 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via # -c requirements/_base.txt # pydantic pyparsing==3.2.1 # via moto -pytest==8.3.4 +pytest==8.3.5 # via # -r requirements/_test.in - # pytest-aiohttp # pytest-asyncio # pytest-cov + # pytest-docker-tools # pytest-icdiff # pytest-instafail # pytest-mock # pytest-sugar -pytest-aiohttp==1.0.5 - # via -r requirements/_test.in pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in - # pytest-aiohttp +pytest-celery==1.1.3 + # via -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in +pytest-docker-tools==3.1.3 + # via pytest-celery pytest-icdiff==0.9 # via -r requirements/_test.in pytest-instafail==0.5.0 @@ -237,7 +313,7 @@ python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt # botocore - # faker + # celery # moto # pandas # simcore-service-storage-sdk @@ -245,14 +321,13 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pytz==2024.2 +pytz==2025.1 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # cfn-lint - # jsondiff # jsonschema-path # moto # responses @@ -261,7 +336,7 @@ redis==5.2.1 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # fakeredis -referencing==0.29.3 +referencing==0.35.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -270,7 +345,7 @@ referencing==0.29.3 # jsonschema-specifications regex==2024.11.6 # via cfn-lint -requests==2.32.2 +requests==2.32.3 # via # -c requirements/_base.txt # docker @@ -279,32 +354,38 @@ requests==2.32.2 # responses responses==0.25.6 # via moto +respx==0.22.0 + # via -r requirements/_test.in rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.0 +rpds-py==0.22.3 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via # -c requirements/_base.txt # boto3 -setuptools==74.0.0 +setuptools==75.8.2 # via - # -c requirements/_base.txt # moto + # pytest-celery simcore-service-storage-sdk @ git+https://github.com/ITISFoundation/osparc-simcore.git@cfdf4f86d844ebb362f4f39e9c6571d561b72897#subdirectory=services/storage/client-sdk/python # via -r requirements/_test.in -six==1.16.0 +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil # rfc3339-validator # simcore-service-storage-sdk +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio sortedcontainers==2.4.0 # via fakeredis -sqlalchemy==1.4.52 +sqlalchemy==1.4.54 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -313,6 +394,10 @@ sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sympy==1.13.3 # via cfn-lint +tenacity==9.0.0 + # via + # -c requirements/_base.txt + # pytest-celery termcolor==2.5.0 # via pytest-sugar types-aiofiles==24.1.0.20241221 @@ -320,16 +405,22 @@ types-aiofiles==24.1.0.20241221 typing-extensions==4.12.2 # via # -c requirements/_base.txt + # anyio + # asyncpg-stubs # aws-sam-translator # cfn-lint - # faker # mypy # pydantic # pydantic-core # sqlalchemy2-stubs tzdata==2025.1 - # via pandas -urllib3==2.2.3 + # via + # -c requirements/_base.txt + # celery + # faker + # kombu + # pandas +urllib3==2.3.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -338,18 +429,28 @@ urllib3==2.2.3 # requests # responses # simcore-service-storage-sdk -werkzeug==3.0.2 +vine==5.1.0 # via # -c requirements/_base.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -c requirements/_base.txt + # prompt-toolkit +werkzeug==3.1.3 + # via # flask + # flask-cors # moto -wrapt==1.16.0 +wrapt==1.17.2 # via # -c requirements/_base.txt # aws-xray-sdk xmltodict==0.14.2 # via moto -yarl==1.9.4 +yarl==1.18.3 # via # -c requirements/_base.txt # aiohttp diff --git a/services/storage/requirements/_tools.txt b/services/storage/requirements/_tools.txt index 7bc674ecdf6..d57a2d475d5 100644 --- a/services/storage/requirements/_tools.txt +++ b/services/storage/requirements/_tools.txt @@ -1,6 +1,6 @@ astroid==3.3.8 # via pylint -black==24.10.0 +black==25.1.0 # via -r requirements/../../../requirements/devenv.txt build==1.2.2.post1 # via pip-tools @@ -8,7 +8,7 @@ bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt cfgv==3.4.0 # via pre-commit -click==8.1.7 +click==8.1.8 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -20,15 +20,15 @@ distlib==0.3.9 # via virtualenv filelock==3.17.0 # via virtualenv -identify==2.6.6 +identify==2.6.8 # via pre-commit -isort==5.13.2 +isort==6.0.1 # via # -r requirements/../../../requirements/devenv.txt # pylint mccabe==0.7.0 # via pylint -mypy==1.14.1 +mypy==1.15.0 # via # -c requirements/_test.txt # -r requirements/../../../requirements/devenv.txt @@ -39,7 +39,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -47,7 +47,7 @@ packaging==24.0 # build pathspec==0.12.1 # via black -pip==25.0 +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -58,24 +58,23 @@ platformdirs==4.3.6 # virtualenv pre-commit==4.1.0 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.3 +pylint==3.3.4 # via -r requirements/../../../requirements/devenv.txt pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.9.3 +ruff==0.9.9 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.8.2 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 @@ -85,9 +84,11 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.29.1 +virtualenv==20.29.2 # via pre-commit watchdog==6.0.0 - # via -r requirements/_tools.in + # via + # -c requirements/_base.txt + # -r requirements/_tools.in wheel==0.45.1 # via pip-tools diff --git a/services/storage/requirements/ci.txt b/services/storage/requirements/ci.txt index 26d5d78bff9..31b66afbe90 100644 --- a/services/storage/requirements/ci.txt +++ b/services/storage/requirements/ci.txt @@ -17,7 +17,7 @@ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ -simcore-service-library[aiohttp] @ ../../packages/service-library +simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library/ # installs current package diff --git a/services/storage/requirements/dev.txt b/services/storage/requirements/dev.txt index 97aefedee51..253cec8dbcb 100644 --- a/services/storage/requirements/dev.txt +++ b/services/storage/requirements/dev.txt @@ -17,7 +17,7 @@ --editable ../../packages/models-library --editable ../../packages/postgres-database/ --editable ../../packages/pytest-simcore/ ---editable ../../packages/service-library[aiohttp] +--editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library/ # installs current package diff --git a/services/storage/requirements/prod.txt b/services/storage/requirements/prod.txt index 5e0703f83a7..dc4a2da4805 100644 --- a/services/storage/requirements/prod.txt +++ b/services/storage/requirements/prod.txt @@ -14,7 +14,7 @@ simcore-aws-library @ ../../packages/aws-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ -simcore-service-library[aiohttp] @ ../../packages/service-library +simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library/ # installs current package diff --git a/services/storage/setup.cfg b/services/storage/setup.cfg index c703b2e8101..a185ddfb0a4 100644 --- a/services/storage/setup.cfg +++ b/services/storage/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.6.0 commit = True message = services/storage api version: {current_version} → {new_version} tag = False @@ -7,8 +7,6 @@ commit_args = --no-verify [bumpversion:file:VERSION] -[bumpversion:file:./src/simcore_service_storage/api/v0/openapi.yaml] - [tool:pytest] asyncio_mode = auto markers = diff --git a/services/storage/setup.py b/services/storage/setup.py index 2a0ca0d9c41..0ee0153333d 100644 --- a/services/storage/setup.py +++ b/services/storage/setup.py @@ -25,7 +25,7 @@ def read_reqs(reqs_path: Path) -> set[str]: | { "simcore-models-library", "simcore-postgres-database", - "simcore-service-library[aiohttp]>=1.2.0", + "simcore-service-library[fastapi]>=1.2.0", "simcore-settings-library", } ) diff --git a/services/storage/src/simcore_service_storage/_meta.py b/services/storage/src/simcore_service_storage/_meta.py index 8a0a28b89ea..7f5013dd0c3 100644 --- a/services/storage/src/simcore_service_storage/_meta.py +++ b/services/storage/src/simcore_service_storage/_meta.py @@ -27,3 +27,18 @@ """.format( f"v{__version__}" ) + +APP_WORKER_STARTED_BANNER_MSG = r""" + + ____ _ __ __ _ + / ___|| |_ ___ _ __ __ _ __ _ ___ \ \ / /__ _ __| | _____ _ __ + \___ \| __/ _ \| '__/ _` |/ _` |/ _ \____\ \ /\ / / _ \| '__| |/ / _ \ '__| + ___) | || (_) | | | (_| | (_| | __/_____\ V V / (_) | | | < __/ | + |____/ \__\___/|_| \__,_|\__, |\___| \_/\_/ \___/|_| |_|\_\___|_| + |___/ {} + +""".format( + f"v{__version__}" +) + +APP_FINISHED_BANNER_MSG = info.get_finished_banner() diff --git a/services/storage/src/simcore_service_storage/api/READ_ONLY_FOLDER b/services/storage/src/simcore_service_storage/api/READ_ONLY_FOLDER deleted file mode 100644 index 28208879513..00000000000 --- a/services/storage/src/simcore_service_storage/api/READ_ONLY_FOLDER +++ /dev/null @@ -1 +0,0 @@ -To modify the OpenAPI Specs, please do it in api/specs/storage and then run `make openapi-specs` in the current dir diff --git a/services/web/server/src/simcore_service_webserver/projects/_common/__init__.py b/services/storage/src/simcore_service_storage/api/__init__.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/projects/_common/__init__.py rename to services/storage/src/simcore_service_storage/api/__init__.py diff --git a/services/web/server/src/simcore_service_webserver/version_control/__init__.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/__init__.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/version_control/__init__.py rename to services/storage/src/simcore_service_storage/api/_worker_tasks/__init__.py diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py new file mode 100644 index 00000000000..fae0bdc770c --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py @@ -0,0 +1,27 @@ +import logging +from pathlib import Path + +from celery import Task # type: ignore[import-untyped] +from models_library.projects_nodes_io import LocationID +from models_library.users import UserID +from pydantic import ByteSize +from servicelib.logging_utils import log_context + +from ...dsm import get_dsm_provider +from ...modules.celery.models import TaskId +from ...modules.celery.utils import get_fastapi_app + +_logger = logging.getLogger(__name__) + + +async def compute_path_size( + task: Task, task_id: TaskId, user_id: UserID, location_id: LocationID, path: Path +) -> ByteSize: + assert task_id # nosec + with log_context( + _logger, + logging.INFO, + msg=f"computing path size {user_id=}, {location_id=}, {path=}", + ): + dsm = get_dsm_provider(get_fastapi_app(task.app)).get(location_id) + return await dsm.compute_path_size(user_id, path=Path(path)) diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py new file mode 100644 index 00000000000..557013de976 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py @@ -0,0 +1,22 @@ +import logging + +from celery import Celery # type: ignore[import-untyped] +from servicelib.logging_utils import log_context + +from ...modules.celery._celery_types import register_celery_types +from ...modules.celery._task import define_task +from ...modules.celery.tasks import export_data +from ._paths import compute_path_size + +_logger = logging.getLogger(__name__) + + +def setup_worker_tasks(app: Celery) -> None: + register_celery_types() + with log_context( + _logger, + logging.INFO, + msg="Storage setup Worker Tasks", + ): + define_task(app, export_data) + define_task(app, compute_path_size) diff --git a/services/storage/src/simcore_service_storage/api/rest/__init__.py b/services/storage/src/simcore_service_storage/api/rest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/api/rest/_datasets.py b/services/storage/src/simcore_service_storage/api/rest/_datasets.py new file mode 100644 index 00000000000..76e6e185068 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_datasets.py @@ -0,0 +1,58 @@ +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from models_library.api_schemas_storage.storage_schemas import ( + DatasetMetaDataGet, + FileMetaDataGet, +) +from models_library.generics import Envelope +from models_library.projects_nodes_io import LocationID + +from ...dsm import get_dsm_provider +from ...models import FilesMetadataDatasetQueryParams, StorageQueryParamsBase + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=[ + "datasets", + ], +) + + +@router.get( + "/locations/{location_id}/datasets", + response_model=Envelope[list[DatasetMetaDataGet]], +) +async def list_datasets_metadata( + query_params: Annotated[StorageQueryParamsBase, Depends()], + location_id: LocationID, + request: Request, +) -> Envelope[list[DatasetMetaDataGet]]: + dsm = get_dsm_provider(request.app).get(location_id) + data = await dsm.list_datasets(query_params.user_id) + return Envelope[list[DatasetMetaDataGet]]( + data=[DatasetMetaDataGet(**d.model_dump()) for d in data] + ) + + +@router.get( + "/locations/{location_id}/datasets/{dataset_id}/metadata", + response_model=Envelope[list[FileMetaDataGet]], +) +async def list_dataset_files_metadata( + query_params: Annotated[FilesMetadataDatasetQueryParams, Depends()], + location_id: LocationID, + dataset_id: str, + request: Request, +) -> Envelope[list[FileMetaDataGet]]: + dsm = get_dsm_provider(request.app).get(location_id) + data = await dsm.list_files_in_dataset( + user_id=query_params.user_id, + dataset_id=dataset_id, + expand_dirs=query_params.expand_dirs, + ) + return Envelope[list[FileMetaDataGet]]( + data=[FileMetaDataGet(**d.model_dump()) for d in data] + ) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py new file mode 100644 index 00000000000..c0b6a4f4a7c --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -0,0 +1,386 @@ +import asyncio +import logging +from typing import Annotated, cast +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Header, HTTPException, Request +from models_library.api_schemas_storage.storage_schemas import ( + FileMetaDataGet, + FileMetaDataGetv010, + FileUploadCompleteFutureResponse, + FileUploadCompleteLinks, + FileUploadCompleteResponse, + FileUploadCompleteState, + FileUploadCompletionBody, + FileUploadLinks, + FileUploadSchema, + SoftCopyBody, +) +from models_library.generics import Envelope +from models_library.projects_nodes_io import LocationID, StorageFileID +from pydantic import AnyUrl, ByteSize, TypeAdapter +from servicelib.aiohttp import status +from yarl import URL + +from ...dsm import get_dsm_provider +from ...exceptions.errors import FileMetaDataNotFoundError +from ...models import ( + FileDownloadQueryParams, + FileDownloadResponse, + FileMetaData, + FileMetadataListQueryParams, + FileUploadQueryParams, + FileUploadResponseV1, + StorageQueryParamsBase, + UploadLinks, +) +from ...modules.long_running_tasks import get_completed_upload_tasks +from ...simcore_s3_dsm import SimcoreS3DataManager +from ...utils.utils import create_upload_completion_task_name + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=[ + "files", + ], +) + + +@router.get( + "/locations/{location_id}/files/metadata", + response_model=Envelope[list[FileMetaDataGet]], +) +async def list_files_metadata( + query_params: Annotated[FileMetadataListQueryParams, Depends()], + location_id: LocationID, + request: Request, +): + dsm = get_dsm_provider(request.app).get(location_id) + data: list[FileMetaData] = await dsm.list_files( + user_id=query_params.user_id, + expand_dirs=query_params.expand_dirs, + uuid_filter=query_params.uuid_filter + or f"{query_params.project_id or ''}", # NOTE: https://github.com/ITISFoundation/osparc-issues/issues/1593 + project_id=query_params.project_id, + ) + return Envelope[list[FileMetaDataGet]]( + data=[FileMetaDataGet(**d.model_dump()) for d in data] + ) + + +@router.get( + "/locations/{location_id}/files/{file_id:path}/metadata", + response_model=Envelope[FileMetaDataGet] + | Envelope[FileMetaDataGetv010] + | Envelope[dict], +) +async def get_file_metadata( + query_params: Annotated[StorageQueryParamsBase, Depends()], + location_id: LocationID, + file_id: StorageFileID, + user_agent: Annotated[str | None, Header()], + request: Request, +): + # NOTE: Used by legacy dynamic services -> MUST BE BACKWARDS COMPATIBLE + dsm = get_dsm_provider(request.app).get(location_id) + try: + data = await dsm.get_file( + user_id=query_params.user_id, + file_id=file_id, + ) + except FileMetaDataNotFoundError: + # NOTE: LEGACY compatibility + # This is what happens Larry... data must be an empty {} or else some old dynamic services will FAIL (sic) + # Cannot remove until we retire all legacy services + # https://github.com/ITISFoundation/osparc-simcore/issues/5676 + # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_enveloped.py#L34 + return Envelope[dict]( + data={}, + error="No result found", # NOTE: LEGACY compatibility + ) + + if user_agent == "OpenAPI-Generator/0.1.0/python": + # NOTE: LEGACY compatiblity with API v0.1.0 + # SEE models used in sdk in: + # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_enveloped.py#L34 + # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_type.py#L34 + return Envelope[FileMetaDataGetv010]( + data=FileMetaDataGetv010( + file_uuid=data.file_uuid, + location_id=data.location_id, + location=data.location, + bucket_name=data.bucket_name, + object_name=data.object_name, + project_id=data.project_id, + project_name=data.project_name, + node_id=data.node_id, + node_name=data.node_name, + file_name=data.file_name, + user_id=data.user_id, + user_name=None, + ) + ) + + return Envelope[FileMetaDataGet](data=FileMetaDataGet(**data.model_dump())) + + +@router.get( + "/locations/{location_id}/files/{file_id:path}", + response_model=Envelope[FileDownloadResponse], +) +async def download_file( + location_id: LocationID, + file_id: StorageFileID, + query_params: Annotated[FileDownloadQueryParams, Depends()], + request: Request, +) -> Envelope[FileDownloadResponse]: + # NOTE: Used by legacy dynamic services -> MUST BE BACKWARDS COMPATIBLE + dsm = get_dsm_provider(request.app).get(location_id) + link = await dsm.create_file_download_link( + query_params.user_id, file_id, query_params.link_type + ) + return Envelope[FileDownloadResponse](data=FileDownloadResponse(link=link)) + + +@router.put( + "/locations/{location_id}/files/{file_id:path}", + response_model=Envelope[FileUploadResponseV1] | Envelope[FileUploadSchema], +) +async def upload_file( + location_id: LocationID, + file_id: StorageFileID, + query_params: Annotated[FileUploadQueryParams, Depends()], + request: Request, +): + """creates upload file links: + + This function covers v1 and v2 versions of the handler. + Note: calling this entrypoint on an already existing file will overwrite that file. That file will be deleted + before the upload takes place. + + v1 rationale: + - client calls this handler, which returns a single link (either direct S3 or presigned) to the S3 backend + - client uploads the file + - storage relies on lazy update to find if the file is finished uploaded (when client calls get_file_meta_data, or if the dsm_cleaner goes over it after the upload time is expired) + + v2 rationale: + - client calls this handler, which returns a FileUploadSchema object containing 1 or more links (either S3/presigned links) + - client uploads the file (by chunking it if there are more than 1 presigned link) + - client calls complete_upload handle which will reconstruct the file on S3 backend + - client waits for completion to finish and then the file is accessible on S3 backend + + + Use-case v1: query.file_size is not defined, returns a PresignedLink model (backward compatibility) + Use-case v1.1: if query.link_type=presigned or None, returns a presigned link (limited to a single 5GB file) + Use-case v1.2: if query.link_type=s3, returns a s3 direct link (limited to a single 5TB file) + + User-case v2: query.is_directory is True (query.file_size is forced to -1), returns an s3 path where to upload all the content of the directory + User-case v2: if query.file_size is defined, returns a FileUploadSchema model, expects client to call "complete_upload" when the file is finished uploading + Use-case v2.1: if query.file_size == 0 and query.link_type=presigned or None, returns a single presigned link inside FileUploadSchema (limited to a single 5Gb file) + Use-case v2.2: if query.file_size > 0 and query.link_type=presigned or None, returns 1 or more presigned links depending on the file size (limited to a single 5TB file) + Use-case v2.3: if query.link_type=s3 and query.file_size>=0, returns a single s3 direct link (limited to a single 5TB file) + """ + # NOTE: Used by legacy dynamic services with single presigned link -> MUST BE BACKWARDS COMPATIBLE + dsm = get_dsm_provider(request.app).get(location_id) + links: UploadLinks = await dsm.create_file_upload_links( + user_id=query_params.user_id, + file_id=file_id, + link_type=query_params.link_type, + file_size_bytes=query_params.file_size or ByteSize(0), + is_directory=query_params.is_directory, + sha256_checksum=query_params.sha256_checksum, + ) + if query_params.is_v1_upload: + # return v1 response + assert len(links.urls) == 1 # nosec + return Envelope[FileUploadResponseV1]( + data=FileUploadResponseV1(link=links.urls[0]) + ) + + # v2 response + + abort_url = ( + URL(f"{request.url}") + .with_path( + quote( + request.app.url_path_for( + "abort_upload_file", + location_id=f"{location_id}", + file_id=file_id, + ), + safe=":/", + ), + encoded=True, + ) + .with_query(user_id=query_params.user_id) + ) + + complete_url = ( + URL(f"{request.url}") + .with_path( + quote( + request.app.url_path_for( + "complete_upload_file", + location_id=f"{location_id}", + file_id=file_id, + ), + safe=":/", + ), + encoded=True, + ) + .with_query(user_id=query_params.user_id) + ) + + v2_response = FileUploadSchema( + chunk_size=links.chunk_size, + urls=links.urls, + links=FileUploadLinks( + abort_upload=TypeAdapter(AnyUrl).validate_python(f"{abort_url}"), + complete_upload=TypeAdapter(AnyUrl).validate_python(f"{complete_url}"), + ), + ) + return Envelope[FileUploadSchema](data=v2_response) + + +@router.post( + "/locations/{location_id}/files/{file_id:path}:abort", + status_code=status.HTTP_204_NO_CONTENT, +) +async def abort_upload_file( + location_id: LocationID, + file_id: StorageFileID, + query_params: Annotated[StorageQueryParamsBase, Depends()], + request: Request, +): + dsm = get_dsm_provider(request.app).get(location_id) + await dsm.abort_file_upload(query_params.user_id, file_id) + + +@router.post( + "/locations/{location_id}/files/{file_id:path}:complete", + response_model=Envelope[FileUploadCompleteResponse], + status_code=status.HTTP_202_ACCEPTED, +) +async def complete_upload_file( + query_params: Annotated[StorageQueryParamsBase, Depends()], + location_id: LocationID, + file_id: StorageFileID, + body: FileUploadCompletionBody, + request: Request, +): + dsm = get_dsm_provider(request.app).get(location_id) + # NOTE: completing a multipart upload on AWS can take up to several minutes + # if it returns slow we return a 202 - Accepted, the client will have to check later + # for completeness + task = asyncio.create_task( + dsm.complete_file_upload(file_id, query_params.user_id, body.parts), + name=create_upload_completion_task_name(query_params.user_id, file_id), + ) + get_completed_upload_tasks(request.app)[task.get_name()] = task + + route = ( + URL(f"{request.url}") + .with_path( + quote( + request.app.url_path_for( + "is_completed_upload_file", + location_id=f"{location_id}", + file_id=file_id, + future_id=task.get_name(), + ), + safe=":/", + ), + encoded=True, + ) + .with_query(user_id=query_params.user_id) + ) + complete_task_state_url = f"{route}" + + response = FileUploadCompleteResponse( + links=FileUploadCompleteLinks( + state=TypeAdapter(AnyUrl).validate_python(complete_task_state_url) + ) + ) + return Envelope[FileUploadCompleteResponse](data=response) + + +@router.post( + "/locations/{location_id}/files/{file_id:path}:complete/futures/{future_id}", + response_model=Envelope[FileUploadCompleteFutureResponse], +) +async def is_completed_upload_file( + query_params: Annotated[StorageQueryParamsBase, Depends()], + location_id: LocationID, + file_id: StorageFileID, + future_id: str, + request: Request, +): + # NOTE: completing a multipart upload on AWS can take up to several minutes + # therefore we wait a bit to see if it completes fast and return a 204 + # if it returns slow we return a 202 - Accepted, the client will have to check later + # for completeness + task_name = create_upload_completion_task_name(query_params.user_id, file_id) + assert task_name == future_id # nosec # NOTE: fastapi auto-decode path parameters + # first check if the task is in the app + if task := get_completed_upload_tasks(request.app).get(task_name): + if task.done(): + new_fmd: FileMetaData = task.result() + get_completed_upload_tasks(request.app).pop(task_name) + response = FileUploadCompleteFutureResponse( + state=FileUploadCompleteState.OK, e_tag=new_fmd.entity_tag + ) + else: + # the task is still running + response = FileUploadCompleteFutureResponse( + state=FileUploadCompleteState.NOK + ) + return Envelope[FileUploadCompleteFutureResponse](data=response) + # there is no task, either wrong call or storage was restarted + # we try to get the file to see if it exists in S3 + dsm = get_dsm_provider(request.app).get(location_id) + if fmd := await dsm.get_file( + user_id=query_params.user_id, + file_id=file_id, + ): + return Envelope[FileUploadCompleteFutureResponse]( + data=FileUploadCompleteFutureResponse( + state=FileUploadCompleteState.OK, e_tag=fmd.entity_tag + ) + ) + raise HTTPException( + status.HTTP_404_NOT_FOUND, + detail="Not found. Upload could not be completed. Please try again and contact support if it fails again.", + ) + + +@router.delete( + "/locations/{location_id}/files/{file_id:path}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_file( + query_params: Annotated[StorageQueryParamsBase, Depends()], + location_id: LocationID, + file_id: StorageFileID, + request: Request, +): + dsm = get_dsm_provider(request.app).get(location_id) + await dsm.delete_file(query_params.user_id, file_id) + + +@router.post( + "/files/{file_id:path}:soft-copy", response_model=Envelope[FileMetaDataGet] +) +async def copy_as_soft_link( + query_params: Annotated[StorageQueryParamsBase, Depends()], + file_id: StorageFileID, + body: SoftCopyBody, + request: Request, +): + dsm = cast( + SimcoreS3DataManager, + get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), + ) + file_link = await dsm.create_soft_link(query_params.user_id, file_id, body.link_id) + + return Envelope[FileMetaDataGet](data=FileMetaDataGet(**file_link.model_dump())) diff --git a/services/storage/src/simcore_service_storage/api/rest/_health.py b/services/storage/src/simcore_service_storage/api/rest/_health.py new file mode 100644 index 00000000000..7272066ee75 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_health.py @@ -0,0 +1,89 @@ +""" + +- Checks connectivity with other services in the backend + +""" + +import logging + +from aws_library.s3 import S3AccessError +from fastapi import APIRouter, Request +from models_library.api_schemas_storage.storage_schemas import HealthCheck, S3BucketName +from models_library.app_diagnostics import AppStatusCheck +from models_library.generics import Envelope +from pydantic import TypeAdapter +from servicelib.db_asyncpg_utils import check_postgres_liveness +from servicelib.fastapi.db_asyncpg_engine import get_engine +from simcore_postgres_database.utils_aiosqlalchemy import get_pg_engine_stateinfo + +from ..._meta import API_VERSION, PROJECT_NAME, VERSION +from ...core.settings import get_application_settings +from ...modules.s3 import get_s3_client + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=[ + "status", + ], +) + + +@router.get("/", include_in_schema=True, response_model=Envelope[HealthCheck]) +async def get_health( + request: Request, +) -> Envelope[HealthCheck]: + assert request # nosec + return Envelope[HealthCheck]( + data=HealthCheck( + name=PROJECT_NAME, + version=f"{VERSION}", + api_version=API_VERSION, + status=None, + ) + ) + + +@router.get("/status", response_model=Envelope[AppStatusCheck]) +async def get_status(request: Request) -> Envelope[AppStatusCheck]: + # NOTE: all calls here must NOT raise + assert request.app # nosec + app_settings = get_application_settings(request.app) + s3_state = "disabled" + if app_settings.STORAGE_S3: + try: + s3_state = ( + "connected" + if await get_s3_client(request.app).bucket_exists( + bucket=TypeAdapter(S3BucketName).validate_python( + app_settings.STORAGE_S3.S3_BUCKET_NAME + ) + ) + else "no access to S3 bucket" + ) + except S3AccessError: + s3_state = "failed" + + postgres_state = "disabled" + + if app_settings.STORAGE_POSTGRES: + postgres_state = ( + "connected" + if await check_postgres_liveness(get_engine(request.app)) + else "failed" + ) + + status = AppStatusCheck.model_validate( + { + "app_name": PROJECT_NAME, + "version": f"{VERSION}", + "services": { + "postgres": { + "healthy": postgres_state, + "pool": await get_pg_engine_stateinfo(get_engine(request.app)), + }, + "s3": {"healthy": s3_state}, + }, + } + ) + return Envelope[AppStatusCheck](data=status) diff --git a/services/storage/src/simcore_service_storage/api/rest/_locations.py b/services/storage/src/simcore_service_storage/api/rest/_locations.py new file mode 100644 index 00000000000..ec33f8e31c7 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_locations.py @@ -0,0 +1,33 @@ +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Request, status +from models_library.api_schemas_storage.storage_schemas import FileLocation +from models_library.generics import Envelope + +from ...dsm import get_dsm_provider +from ...models import StorageQueryParamsBase + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=["locations"], +) + + +@router.get( + "/locations", + status_code=status.HTTP_200_OK, + response_model=Envelope[list[FileLocation]], +) +async def list_storage_locations( + query_params: Annotated[StorageQueryParamsBase, Depends()], request: Request +): + dsm_provider = get_dsm_provider(request.app) + location_ids = dsm_provider.locations() + locs: list[FileLocation] = [] + for loc_id in location_ids: + dsm = dsm_provider.get(loc_id) + if await dsm.authorized(query_params.user_id): + locs.append(FileLocation(name=dsm.location_name, id=dsm.location_id)) + return Envelope[list[FileLocation]](data=locs) diff --git a/services/storage/src/simcore_service_storage/api/rest/_paths.py b/services/storage/src/simcore_service_storage/api/rest/_paths.py new file mode 100644 index 00000000000..bdb5a171f0c --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_paths.py @@ -0,0 +1,68 @@ +import logging +from pathlib import Path +from typing import Annotated + +from fastapi import APIRouter, Depends +from fastapi_pagination import create_page +from models_library.api_schemas_storage.storage_schemas import ( + PathMetaDataGet, + PathTotalSizeCreate, +) +from models_library.generics import Envelope +from models_library.users import UserID +from servicelib.fastapi.rest_pagination import ( + CustomizedPathsCursorPage, + CustomizedPathsCursorPageParams, +) + +from ...dsm_factory import BaseDataManager +from .dependencies.dsm_prodiver import get_data_manager + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=[ + "files", + ], +) + + +@router.get( + "/locations/{location_id}/paths", + response_model=CustomizedPathsCursorPage[PathMetaDataGet], +) +async def list_paths( + page_params: Annotated[CustomizedPathsCursorPageParams, Depends()], + dsm: Annotated[BaseDataManager, Depends(get_data_manager)], + user_id: UserID, + file_filter: Path | None = None, +): + """Returns one level of files (paginated)""" + items, next_cursor, total_number = await dsm.list_paths( + user_id=user_id, + file_filter=file_filter, + limit=page_params.size, + cursor=page_params.to_raw_params().cursor, + ) + return create_page( + [_.to_api_model() for _ in items], + total=total_number, + params=page_params, + next_=next_cursor, + ) + + +@router.post( + "/locations/{location_id}/paths/{path:path}:size", + response_model=Envelope[PathTotalSizeCreate], +) +async def compute_path_size( + dsm: Annotated[BaseDataManager, Depends(get_data_manager)], + user_id: UserID, + path: Path, +): + return Envelope[PathTotalSizeCreate]( + data=PathTotalSizeCreate( + path=path, size=await dsm.compute_path_size(user_id, path=path) + ) + ) diff --git a/services/storage/src/simcore_service_storage/api/rest/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/rest/_simcore_s3.py new file mode 100644 index 00000000000..29b199e6feb --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/_simcore_s3.py @@ -0,0 +1,163 @@ +import asyncio +import logging +from typing import Annotated, Any, cast + +from fastapi import APIRouter, Depends, FastAPI, Request +from models_library.api_schemas_long_running_tasks.base import TaskProgress +from models_library.api_schemas_long_running_tasks.tasks import TaskGet +from models_library.api_schemas_storage.storage_schemas import ( + FileMetaDataGet, + FoldersBody, +) +from models_library.generics import Envelope +from models_library.projects import ProjectID +from servicelib.aiohttp import status +from servicelib.fastapi.long_running_tasks._dependencies import get_tasks_manager +from servicelib.logging_utils import log_context +from servicelib.long_running_tasks._task import start_task +from settings_library.s3 import S3Settings +from yarl import URL + +from ...dsm import get_dsm_provider +from ...models import ( + DeleteFolderQueryParams, + FileMetaData, + SearchFilesQueryParams, + StorageQueryParamsBase, +) +from ...modules import sts +from ...simcore_s3_dsm import SimcoreS3DataManager + +_logger = logging.getLogger(__name__) + +router = APIRouter( + tags=[ + "simcore-s3", + ], +) + + +@router.post("/simcore-s3:access", response_model=Envelope[S3Settings]) +async def get_or_create_temporary_s3_access( + query_params: Annotated[StorageQueryParamsBase, Depends()], + request: Request, +): + # NOTE: the name of the method is not accurate, these are not temporary at all + # it returns the credentials of the s3 backend! + s3_settings: S3Settings = await sts.get_or_create_temporary_token_for_user( + request.app, query_params.user_id + ) + return Envelope[S3Settings](data=s3_settings) + + +async def _copy_folders_from_project( + progress: TaskProgress, + app: FastAPI, + query_params: StorageQueryParamsBase, + body: FoldersBody, +) -> Envelope[dict[str, Any]]: + dsm = cast( + SimcoreS3DataManager, + get_dsm_provider(app).get(SimcoreS3DataManager.get_location_id()), + ) + with log_context( + _logger, + logging.INFO, + msg=f"copying {body.source['uuid']} -> {body.destination['uuid']}", + ): + await dsm.deep_copy_project_simcore_s3( + query_params.user_id, + body.source, + body.destination, + body.nodes_map, + task_progress=progress, + ) + + return Envelope[dict[str, Any]](data=body.destination) + + +@router.post( + "/simcore-s3/folders", + response_model=Envelope[TaskGet], + status_code=status.HTTP_202_ACCEPTED, +) +async def copy_folders_from_project( + query_params: Annotated[StorageQueryParamsBase, Depends()], + body: FoldersBody, + request: Request, +): + task_id = None + try: + task_id = start_task( + get_tasks_manager(request), + _copy_folders_from_project, + app=request.app, + query_params=query_params, + body=body, + ) + relative_url = URL(f"{request.url}").relative() + + return Envelope[TaskGet]( + data=TaskGet( + task_id=task_id, + task_name=f"{request.method} {relative_url}", + status_href=f"{request.url_for('get_task_status', task_id=task_id)}", + result_href=f"{request.url_for('get_task_result', task_id=task_id)}", + abort_href=f"{request.url_for('cancel_and_delete_task', task_id=task_id)}", + ) + ) + except asyncio.CancelledError: + if task_id: + await get_tasks_manager(request).cancel_task( + task_id, with_task_context=None + ) + raise + + +@router.delete( + "/simcore-s3/folders/{folder_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_folders_of_project( + query_params: Annotated[DeleteFolderQueryParams, Depends()], + folder_id: str, + request: Request, +): + dsm = cast( + SimcoreS3DataManager, + get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), + ) + await dsm.delete_project_simcore_s3( + query_params.user_id, + ProjectID(folder_id), + query_params.node_id, + ) + + +@router.post( + "/simcore-s3/files/metadata:search", + response_model=Envelope[list[FileMetaDataGet]], +) +async def search_files( + query_params: Annotated[SearchFilesQueryParams, Depends()], request: Request +): + dsm = cast( + SimcoreS3DataManager, + get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), + ) + + data: list[FileMetaData] = await dsm.search_owned_files( + user_id=query_params.user_id, + file_id_prefix=query_params.startswith, + sha256_checksum=query_params.sha256_checksum, + limit=query_params.limit, + offset=query_params.offset, + ) + _logger.debug( + "Found %d files starting with '%s'", + len(data), + f"{query_params.startswith=}, {query_params.sha256_checksum=}", + ) + return Envelope[list[FileMetaDataGet]]( + data=[FileMetaDataGet(**d.model_dump()) for d in data] + ) diff --git a/services/storage/src/simcore_service_storage/api/rest/dependencies/__init__.py b/services/storage/src/simcore_service_storage/api/rest/dependencies/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/api/rest/dependencies/application.py b/services/storage/src/simcore_service_storage/api/rest/dependencies/application.py new file mode 100644 index 00000000000..706818ca793 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/dependencies/application.py @@ -0,0 +1,18 @@ +# mypy: disable-error-code=truthy-function +from fastapi import Request +from servicelib.fastapi.dependencies import get_app, get_reverse_url_mapper + +from ....core.settings import ApplicationSettings, get_application_settings + + +def get_settings(request: Request) -> ApplicationSettings: + return get_application_settings(request.app) + + +assert get_reverse_url_mapper # nosec +assert get_app # nosec + +__all__: tuple[str, ...] = ( + "get_app", + "get_reverse_url_mapper", +) diff --git a/services/storage/src/simcore_service_storage/api/rest/dependencies/dsm_prodiver.py b/services/storage/src/simcore_service_storage/api/rest/dependencies/dsm_prodiver.py new file mode 100644 index 00000000000..a0f16979e25 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/dependencies/dsm_prodiver.py @@ -0,0 +1,23 @@ +from typing import Annotated + +from fastapi import Depends, FastAPI +from models_library.projects_nodes_io import LocationID +from servicelib.fastapi.dependencies import get_app + +from ....dsm import get_dsm_provider +from ....dsm_factory import BaseDataManager, DataManagerProvider + + +def get_data_manager_provider( + app: Annotated[FastAPI, Depends(get_app)], +) -> DataManagerProvider: + return get_dsm_provider(app) + + +async def get_data_manager( + location_id: LocationID, + data_manager_provider: Annotated[ + DataManagerProvider, Depends(get_data_manager_provider) + ], +) -> BaseDataManager: + return data_manager_provider.get(location_id) diff --git a/services/storage/src/simcore_service_storage/api/rest/routes.py b/services/storage/src/simcore_service_storage/api/rest/routes.py new file mode 100644 index 00000000000..bb77c1d3f77 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rest/routes.py @@ -0,0 +1,20 @@ +from fastapi import APIRouter, FastAPI + +from . import _datasets, _files, _health, _locations, _paths, _simcore_s3 + +v0_router = APIRouter() + +health_router = _health.router +v0_router.include_router(_health.router) +v0_router.include_router(_locations.router) +v0_router.include_router(_datasets.router) +v0_router.include_router(_files.router) +v0_router.include_router(_paths.router) +v0_router.include_router(_simcore_s3.router) + + +def setup_rest_api_routes(app: FastAPI, vtag: str): + # healthcheck at / and at /v0/ + app.include_router(health_router, prefix=f"/{vtag}") + # api under /v* + app.include_router(v0_router, prefix=f"/{vtag}") diff --git a/services/storage/src/simcore_service_storage/api/rpc/__init__.py b/services/storage/src/simcore_service_storage/api/rpc/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py new file mode 100644 index 00000000000..6c928b18a3a --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -0,0 +1,147 @@ +# pylint: disable=unused-argument + +import logging + +from celery.exceptions import CeleryError # type: ignore[import-untyped] +from fastapi import FastAPI +from models_library.api_schemas_rpc_async_jobs.async_jobs import ( + AsyncJobGet, + AsyncJobId, + AsyncJobNameData, + AsyncJobResult, + AsyncJobStatus, +) +from models_library.api_schemas_rpc_async_jobs.exceptions import ( + JobAbortedError, + JobError, + JobMissingError, + JobNotDoneError, + JobSchedulerError, +) +from servicelib.logging_utils import log_catch +from servicelib.rabbitmq import RPCRouter + +from ...modules.celery import get_celery_client +from ...modules.celery.client import CeleryTaskQueueClient +from ...modules.celery.models import TaskError, TaskState + +_logger = logging.getLogger(__name__) +router = RPCRouter() + + +async def _assert_job_exists( + *, + job_id: AsyncJobId, + job_id_data: AsyncJobNameData, + celery_client: CeleryTaskQueueClient, +) -> None: + """Raises JobMissingError if job doesn't exist""" + job_ids = await celery_client.get_task_uuids( + task_context=job_id_data.model_dump(), + ) + if job_id not in job_ids: + raise JobMissingError(job_id=f"{job_id}") + + +@router.expose(reraise_if_error_type=(JobSchedulerError, JobMissingError)) +async def cancel(app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData): + assert app # nosec + assert job_id_data # nosec + try: + await _assert_job_exists( + job_id=job_id, job_id_data=job_id_data, celery_client=get_celery_client(app) + ) + await get_celery_client(app).abort_task( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) + except CeleryError as exc: + raise JobSchedulerError(exc=f"{exc}") from exc + + +@router.expose(reraise_if_error_type=(JobSchedulerError, JobMissingError)) +async def status( + app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData +) -> AsyncJobStatus: + assert app # nosec + assert job_id_data # nosec + + try: + await _assert_job_exists( + job_id=job_id, job_id_data=job_id_data, celery_client=get_celery_client(app) + ) + task_status = await get_celery_client(app).get_task_status( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) + except CeleryError as exc: + raise JobSchedulerError(exc=f"{exc}") from exc + + return AsyncJobStatus( + job_id=job_id, + progress=task_status.progress_report, + done=task_status.is_done, + ) + + +@router.expose( + reraise_if_error_type=( + JobError, + JobNotDoneError, + JobAbortedError, + JobSchedulerError, + JobMissingError, + ) +) +async def result( + app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData +) -> AsyncJobResult: + assert app # nosec + assert job_id # nosec + assert job_id_data # nosec + + try: + await _assert_job_exists( + job_id=job_id, job_id_data=job_id_data, celery_client=get_celery_client(app) + ) + _status = await get_celery_client(app).get_task_status( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) + if not _status.is_done: + raise JobNotDoneError(job_id=job_id) + _result = await get_celery_client(app).get_task_result( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) + except CeleryError as exc: + raise JobSchedulerError(exc=f"{exc}") from exc + + if _status.task_state == TaskState.ABORTED: + raise JobAbortedError(job_id=job_id) + if _status.task_state == TaskState.ERROR: + exc_type = "" + exc_msg = "" + with log_catch(logger=_logger, reraise=False): + task_error = TaskError.model_validate(_result) + exc_type = task_error.exc_type + exc_msg = task_error.exc_msg + raise JobError(job_id=job_id, exc_type=exc_type, exc_msg=exc_msg) + + return AsyncJobResult(result=_result) + + +@router.expose(reraise_if_error_type=(JobSchedulerError,)) +async def list_jobs( + app: FastAPI, filter_: str, job_id_data: AsyncJobNameData +) -> list[AsyncJobGet]: + assert app # nosec + + try: + task_uuids = await get_celery_client(app).get_task_uuids( + task_context=job_id_data.model_dump(), + ) + except CeleryError as exc: + raise JobSchedulerError(exc=f"{exc}") from exc + + return [AsyncJobGet(job_id=task_uuid) for task_uuid in task_uuids] diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py new file mode 100644 index 00000000000..7fe6612e5e3 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -0,0 +1,65 @@ +from celery.exceptions import CeleryError # type: ignore[import-untyped] +from fastapi import FastAPI +from models_library.api_schemas_rpc_async_jobs.async_jobs import ( + AsyncJobGet, + AsyncJobNameData, +) +from models_library.api_schemas_rpc_async_jobs.exceptions import JobSchedulerError +from models_library.api_schemas_storage.data_export_async_jobs import ( + AccessRightError, + DataExportTaskStartInput, + InvalidFileIdentifierError, +) +from servicelib.rabbitmq import RPCRouter + +from ...datcore_dsm import DatCoreDataManager +from ...dsm import get_dsm_provider +from ...exceptions.errors import FileAccessRightError +from ...modules.celery import get_celery_client +from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError +from ...simcore_s3_dsm import SimcoreS3DataManager + +router = RPCRouter() + + +@router.expose( + reraise_if_error_type=( + InvalidFileIdentifierError, + AccessRightError, + JobSchedulerError, + ) +) +async def start_data_export( + app: FastAPI, + data_export_start: DataExportTaskStartInput, + job_id_data: AsyncJobNameData, +) -> AsyncJobGet: + assert app # nosec + + dsm = get_dsm_provider(app).get(data_export_start.location_id) + + try: + for _id in data_export_start.file_and_folder_ids: + if isinstance(dsm, DatCoreDataManager): + _ = await dsm.get_file(user_id=job_id_data.user_id, file_id=_id) + elif isinstance(dsm, SimcoreS3DataManager): + await dsm.can_read_file(user_id=job_id_data.user_id, file_id=_id) + + except (FileAccessRightError, DatcoreAdapterError) as err: + raise AccessRightError( + user_id=job_id_data.user_id, + file_id=_id, + location_id=data_export_start.location_id, + ) from err + + try: + task_uuid = await get_celery_client(app).send_task( + "export_data", + task_context=job_id_data.model_dump(), + files=data_export_start.file_and_folder_ids, # ANE: adapt here your signature + ) + except CeleryError as exc: + raise JobSchedulerError(exc=f"{exc}") from exc + return AsyncJobGet( + job_id=task_uuid, + ) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_paths.py b/services/storage/src/simcore_service_storage/api/rpc/_paths.py new file mode 100644 index 00000000000..34ea3deeedb --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rpc/_paths.py @@ -0,0 +1,35 @@ +from pathlib import Path + +from fastapi import FastAPI +from models_library.api_schemas_rpc_async_jobs.async_jobs import ( + AsyncJobGet, + AsyncJobNameData, +) +from models_library.projects_nodes_io import LocationID +from servicelib.rabbitmq import RPCRouter + +from ...modules.celery import get_celery_client +from .._worker_tasks._paths import compute_path_size as remote_compute_path_size + +router = RPCRouter() + + +@router.expose(reraise_if_error_type=None) +async def compute_path_size( + app: FastAPI, + job_id_data: AsyncJobNameData, + # user_id: UserID, + location_id: LocationID, + path: Path, +) -> AsyncJobGet: + assert app # nosec + + task_uuid = await get_celery_client(app).send_task( + remote_compute_path_size.__name__, + task_context=job_id_data.model_dump(), + user_id=job_id_data.user_id, + location_id=location_id, + path=path, + ) + + return AsyncJobGet(job_id=task_uuid) diff --git a/services/storage/src/simcore_service_storage/api/rpc/routes.py b/services/storage/src/simcore_service_storage/api/rpc/routes.py new file mode 100644 index 00000000000..799a2b4e839 --- /dev/null +++ b/services/storage/src/simcore_service_storage/api/rpc/routes.py @@ -0,0 +1,32 @@ +import logging + +from fastapi import FastAPI +from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE +from servicelib.logging_utils import log_context +from servicelib.rabbitmq import RPCRouter + +from ...modules.rabbitmq import get_rabbitmq_rpc_server +from . import _async_jobs, _data_export, _paths + +_logger = logging.getLogger(__name__) + + +ROUTERS: list[RPCRouter] = [ + _async_jobs.router, + _data_export.router, + _paths.router, +] + + +def setup_rpc_api_routes(app: FastAPI) -> None: + async def startup() -> None: + with log_context( + _logger, + logging.INFO, + msg="Storage startup RPC API Routes", + ): + rpc_server = get_rabbitmq_rpc_server(app) + for router in ROUTERS: + await rpc_server.register_router(router, STORAGE_RPC_NAMESPACE, app) + + app.add_event_handler("startup", startup) diff --git a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml deleted file mode 100644 index 22f27d960ac..00000000000 --- a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml +++ /dev/null @@ -1,1596 +0,0 @@ -openapi: 3.1.0 -info: - title: simcore-service-storage API - description: API definition for simcore-service-storage service - contact: - name: IT'IS Foundation - email: support@simcore.io - license: - name: MIT - url: https://github.com/ITISFoundation/osparc-simcore/blob/master/LICENSE - version: 0.5.0 -servers: -- url: / - description: 'Default server: requests directed to serving url' -- url: http://{host}:{port}/ - description: 'Development server: can configure any base url' - variables: - host: - default: 127.0.0.1 - port: - default: '8000' -paths: - /v0/locations/{location_id}/datasets: - get: - tags: - - datasets - summary: Get datasets metadata - description: returns all the top level datasets a user has access to - operationId: get_datasets_metadata - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_DatasetMetaData__' - /v0/locations/{location_id}/datasets/{dataset_id}/metadata: - get: - tags: - - datasets - summary: Get Files Metadata - description: returns all the file meta data inside dataset with dataset_id - operationId: get_files_metadata_dataset - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: dataset_id - in: path - required: true - schema: - type: string - title: Dataset Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - - name: expand_dirs - in: query - required: false - schema: - type: boolean - description: Automatic directory expansion. This will be replaced by pagination - the future - default: true - title: Expand Dirs - description: Automatic directory expansion. This will be replaced by pagination - the future - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_FileMetaDataGet__' - /v0/locations: - get: - tags: - - locations - summary: Get available storage locations - description: Returns the list of available storage locations - operationId: get_storage_locations - parameters: - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/DatasetMetaData' - title: Response Get Storage Locations - /v0/locations/{location_id}:sync: - post: - tags: - - locations - summary: Manually triggers the synchronisation of the file meta data table in - the database - description: Returns an object containing added, changed and removed paths - operationId: synchronise_meta_data_table - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: dry_run - in: query - required: false - schema: - type: boolean - default: false - title: Dry Run - - name: fire_and_forget - in: query - required: false - schema: - type: boolean - default: false - title: Fire And Forget - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_TableSynchronisation_' - /v0/locations/{location_id}/files/metadata: - get: - tags: - - files - summary: Get datasets metadata - description: returns all the file meta data a user has access to (uuid_filter - may be used) - operationId: get_files_metadata - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: uuid_filter - in: query - required: false - schema: - type: string - default: '' - title: Uuid Filter - - name: expand_dirs - in: query - required: false - schema: - type: boolean - description: Automatic directory expansion. This will be replaced by pagination - the future - default: true - title: Expand Dirs - description: Automatic directory expansion. This will be replaced by pagination - the future - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_DatasetMetaData__' - /v0/locations/{location_id}/files/{file_id}/metadata: - get: - tags: - - files - summary: Get File Metadata - description: returns the file meta data of file_id if user_id has the rights - to - operationId: get_file_metadata - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - anyOf: - - $ref: '#/components/schemas/FileMetaData' - - $ref: '#/components/schemas/Envelope_FileMetaDataGet_' - title: Response Get File Metadata - /v0/locations/{location_id}/files/{file_id}: - get: - tags: - - files - summary: Returns download link for requested file - description: creates a download file link if user has the rights to - operationId: download_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - - name: link_type - in: query - required: false - schema: - $ref: '#/components/schemas/LinkType' - default: PRESIGNED - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_PresignedLink_' - put: - tags: - - files - summary: Returns upload link - description: creates one or more upload file links if user has the rights to, - expects the client to complete/abort upload - operationId: upload_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: file_size - in: query - required: true - schema: - anyOf: - - type: string - pattern: ^\s*(\d*\.?\d+)\s*(\w+)? - - type: integer - minimum: 0 - - type: 'null' - title: File Size - - name: link_type - in: query - required: false - schema: - $ref: '#/components/schemas/LinkType' - default: PRESIGNED - - name: is_directory - in: query - required: false - schema: - type: boolean - default: false - title: Is Directory - responses: - '200': - description: Successful Response - content: - application/json: - schema: - anyOf: - - $ref: '#/components/schemas/Envelope_FileUploadSchema_' - - $ref: '#/components/schemas/Envelope_Url_' - title: Response Upload File - delete: - tags: - - files - summary: Deletes File - description: deletes file if user has the rights to - operationId: delete_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '204': - description: Successful Response - /v0/locations/{location_id}/files/{file_id}:abort: - post: - tags: - - files - summary: Abort Upload File - description: 'aborts an upload if user has the rights to, and reverts - - to the latest version if available, else will delete the file' - operationId: abort_upload_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '204': - description: Successful Response - /v0/locations/{location_id}/files/{file_id}:complete: - post: - tags: - - files - summary: Complete Upload File - description: completes an upload if the user has the rights to - operationId: complete_upload_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FileUploadCompletionBody_' - responses: - '202': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FileUploadCompleteResponse_' - /v0/locations/{location_id}/files/{file_id}:complete/futures/{future_id}: - post: - tags: - - files - summary: Check for upload completion - description: Returns state of upload completion - operationId: is_completed_upload_file - parameters: - - name: location_id - in: path - required: true - schema: - type: integer - title: Location Id - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: future_id - in: path - required: true - schema: - type: string - title: Future Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FileUploadCompleteFutureResponse_' - /v0/: - get: - tags: - - health - summary: health check endpoint - description: Current service health - operationId: health_check - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_HealthCheck_' - /v0/status: - get: - tags: - - health - summary: returns the status of the services inside - description: returns the status of all the external dependencies - operationId: get_status - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_AppStatusCheck_' - /v0/files/{file_id}:soft-copy: - post: - tags: - - files - summary: copy file as soft link - description: creates and returns a soft link - operationId: copy_as_soft_link - parameters: - - name: file_id - in: path - required: true - schema: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/SoftCopyBody' - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/FileMetaDataGet' - /v0/simcore-s3:access: - post: - tags: - - simcore-s3 - summary: gets or creates the a temporary access - description: returns a set of S3 credentials - operationId: get_or_create_temporary_s3_access - parameters: - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_S3Settings_' - /v0/simcore-s3/folders: - post: - tags: - - simcore-s3 - summary: copies folders from project - description: copies folders from project - operationId: copy_folders_from_project - parameters: - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/FoldersBody' - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_TaskGet_' - /v0/simcore-s3/folders/{folder_id}: - delete: - tags: - - simcore-s3 - summary: delete folders from project - description: removes folders from a project - operationId: delete_folders_of_project - parameters: - - name: folder_id - in: path - required: true - schema: - type: string - title: Folder Id - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - - name: node_id - in: query - required: false - schema: - anyOf: - - type: string - format: uuid - - type: 'null' - title: Node Id - responses: - '204': - description: Successful Response - /v0/simcore-s3/files/metadata:search: - post: - tags: - - simcore-s3 - summary: search for owned files - description: search for files starting with `startswith` and/or matching a sha256_checksum - in the file_meta_data table - operationId: search_files - parameters: - - name: user_id - in: query - required: true - schema: - type: integer - exclusiveMinimum: true - title: User Id - minimum: 0 - - name: startswith - in: query - required: false - schema: - anyOf: - - type: string - - type: 'null' - title: Startswith - - name: sha256_checksum - in: query - required: false - schema: - anyOf: - - type: string - pattern: ^[a-fA-F0-9]{64}$ - - type: 'null' - title: Sha256 Checksum - - name: kind - in: query - required: true - schema: - enum: - - owned - const: owned - type: string - title: Kind - - name: limit - in: query - required: false - schema: - type: integer - maximum: 50 - minimum: 1 - default: 20 - title: Limit - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_FileMetaDataGet_' - /v0/futures: - get: - tags: - - tasks - summary: list current long running tasks - description: list current long running tasks - operationId: list_tasks - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_TaskGet_' - /v0/futures/{task_id}: - get: - tags: - - tasks - summary: gets the status of the task - description: gets the status of the task - operationId: get_task_status - parameters: - - name: task_id - in: path - required: true - schema: - type: string - title: Task Id - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_TaskStatus_' - delete: - tags: - - tasks - summary: cancels and removes the task - description: cancels and removes the task - operationId: cancel_and_delete_task - parameters: - - name: task_id - in: path - required: true - schema: - type: string - title: Task Id - responses: - '204': - description: Successful Response - /v0/futures/{task_id}/result: - get: - tags: - - tasks - summary: get result of the task - description: get result of the task - operationId: get_task_result - parameters: - - name: task_id - in: path - required: true - schema: - type: string - title: Task Id - responses: - '200': - description: Successful Response - content: - application/json: - schema: - title: Response Get Task Result -components: - schemas: - AppStatusCheck: - properties: - app_name: - type: string - title: App Name - description: Application name - version: - type: string - title: Version - description: Application's version - services: - type: object - title: Services - description: Other backend services connected from this service - default: {} - sessions: - anyOf: - - type: object - - type: 'null' - title: Sessions - description: Client sessions info. If single session per app, then is denoted - as main - default: {} - url: - anyOf: - - type: string - minLength: 1 - format: uri - - type: 'null' - title: Url - description: Link to current resource - diagnostics_url: - anyOf: - - type: string - minLength: 1 - format: uri - - type: 'null' - title: Diagnostics Url - description: Link to diagnostics report sub-resource. This MIGHT take some - time to compute - type: object - required: - - app_name - - version - title: AppStatusCheck - DatasetMetaData: - properties: - dataset_id: - anyOf: - - type: string - format: uuid - - type: string - pattern: ^N:dataset:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: Dataset Id - display_name: - type: string - title: Display Name - additionalProperties: false - type: object - required: - - dataset_id - - display_name - title: DatasetMetaData - Envelope_AppStatusCheck_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/AppStatusCheck' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[AppStatusCheck] - Envelope_FileMetaDataGet_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FileMetaDataGet' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FileMetaDataGet] - Envelope_FileUploadCompleteFutureResponse_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FileUploadCompleteFutureResponse' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FileUploadCompleteFutureResponse] - Envelope_FileUploadCompleteResponse_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FileUploadCompleteResponse' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FileUploadCompleteResponse] - Envelope_FileUploadCompletionBody_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FileUploadCompletionBody' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FileUploadCompletionBody] - Envelope_FileUploadSchema_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/FileUploadSchema' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[FileUploadSchema] - Envelope_HealthCheck_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/HealthCheck' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[HealthCheck] - Envelope_PresignedLink_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/PresignedLink' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[PresignedLink] - Envelope_S3Settings_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/S3Settings' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[S3Settings] - Envelope_TableSynchronisation_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/TableSynchronisation' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[TableSynchronisation] - Envelope_TaskGet_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/TaskGet' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[TaskGet] - Envelope_TaskStatus_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/TaskStatus' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[TaskStatus] - Envelope_Url_: - properties: - data: - anyOf: - - type: string - minLength: 1 - format: uri - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[Url] - Envelope_list_DatasetMetaData__: - properties: - data: - anyOf: - - items: - $ref: '#/components/schemas/DatasetMetaData' - type: array - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[list[DatasetMetaData]] - Envelope_list_FileMetaDataGet__: - properties: - data: - anyOf: - - items: - $ref: '#/components/schemas/FileMetaDataGet' - type: array - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[list[FileMetaDataGet]] - FileMetaData: - properties: - file_uuid: - type: string - title: File Uuid - description: NOT a unique ID, like (api|uuid)/uuid/file_name or DATCORE - folder structure - location_id: - type: integer - title: Location Id - description: Storage location - project_name: - anyOf: - - type: string - - type: 'null' - title: Project Name - description: optional project name, used by frontend to display path - node_name: - anyOf: - - type: string - - type: 'null' - title: Node Name - description: optional node name, used by frontend to display path - file_name: - type: string - title: File Name - description: Display name for a file - file_id: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - description: THIS IS the unique ID for the file. either (api|project_id)/node_id/file_name.ext - for S3 and N:package:UUID for datcore - created_at: - type: string - format: date-time - title: Created At - last_modified: - type: string - format: date-time - title: Last Modified - file_size: - anyOf: - - type: integer - enum: - - -1 - const: -1 - - type: integer - minimum: 0 - title: File Size - description: File size in bytes (-1 means invalid) - default: -1 - entity_tag: - anyOf: - - type: string - - type: 'null' - title: Entity Tag - description: Entity tag (or ETag), represents a specific version of the - file, None if invalid upload or datcore - is_soft_link: - type: boolean - title: Is Soft Link - description: If true, this file is a soft link.i.e. is another entry with - the same object_name - default: false - is_directory: - type: boolean - title: Is Directory - description: if True this is a directory - default: false - sha256_checksum: - anyOf: - - type: string - pattern: ^[a-fA-F0-9]{64}$ - - type: 'null' - title: Sha256 Checksum - upload_id: - anyOf: - - type: string - - type: 'null' - title: Upload Id - upload_expires_at: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Upload Expires At - location: - type: string - title: Location - bucket_name: - type: string - title: Bucket Name - object_name: - type: string - title: Object Name - project_id: - anyOf: - - type: string - format: uuid - - type: 'null' - title: Project Id - node_id: - anyOf: - - type: string - format: uuid - - type: 'null' - title: Node Id - user_id: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: User Id - type: object - required: - - file_uuid - - location_id - - file_name - - file_id - - created_at - - last_modified - - sha256_checksum - - location - - bucket_name - - object_name - - project_id - - node_id - - user_id - title: FileMetaData - FileMetaDataGet: - properties: - file_uuid: - type: string - title: File Uuid - description: NOT a unique ID, like (api|uuid)/uuid/file_name or DATCORE - folder structure - location_id: - type: integer - title: Location Id - description: Storage location - project_name: - anyOf: - - type: string - - type: 'null' - title: Project Name - description: optional project name, used by frontend to display path - node_name: - anyOf: - - type: string - - type: 'null' - title: Node Name - description: optional node name, used by frontend to display path - file_name: - type: string - title: File Name - description: Display name for a file - file_id: - anyOf: - - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - - type: string - pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ - title: File Id - description: THIS IS the unique ID for the file. either (api|project_id)/node_id/file_name.ext - for S3 and N:package:UUID for datcore - created_at: - type: string - format: date-time - title: Created At - last_modified: - type: string - format: date-time - title: Last Modified - file_size: - anyOf: - - type: integer - enum: - - -1 - const: -1 - - type: integer - minimum: 0 - title: File Size - description: File size in bytes (-1 means invalid) - default: -1 - entity_tag: - anyOf: - - type: string - - type: 'null' - title: Entity Tag - description: Entity tag (or ETag), represents a specific version of the - file, None if invalid upload or datcore - is_soft_link: - type: boolean - title: Is Soft Link - description: If true, this file is a soft link.i.e. is another entry with - the same object_name - default: false - is_directory: - type: boolean - title: Is Directory - description: if True this is a directory - default: false - sha256_checksum: - anyOf: - - type: string - pattern: ^[a-fA-F0-9]{64}$ - - type: 'null' - title: Sha256 Checksum - description: 'SHA256 message digest of the file content. Main purpose: cheap - lookup.' - type: object - required: - - file_uuid - - location_id - - file_name - - file_id - - created_at - - last_modified - title: FileMetaDataGet - FileUploadCompleteFutureResponse: - properties: - state: - $ref: '#/components/schemas/FileUploadCompleteState' - e_tag: - anyOf: - - type: string - - type: 'null' - title: E Tag - type: object - required: - - state - title: FileUploadCompleteFutureResponse - FileUploadCompleteLinks: - properties: - state: - type: string - minLength: 1 - format: uri - title: State - type: object - required: - - state - title: FileUploadCompleteLinks - FileUploadCompleteResponse: - properties: - links: - $ref: '#/components/schemas/FileUploadCompleteLinks' - type: object - required: - - links - title: FileUploadCompleteResponse - FileUploadCompleteState: - type: string - enum: - - ok - - nok - title: FileUploadCompleteState - FileUploadCompletionBody: - properties: - parts: - items: - $ref: '#/components/schemas/UploadedPart' - type: array - title: Parts - type: object - required: - - parts - title: FileUploadCompletionBody - FileUploadLinks: - properties: - abort_upload: - type: string - minLength: 1 - format: uri - title: Abort Upload - complete_upload: - type: string - minLength: 1 - format: uri - title: Complete Upload - type: object - required: - - abort_upload - - complete_upload - title: FileUploadLinks - FileUploadSchema: - properties: - chunk_size: - type: integer - minimum: 0 - title: Chunk Size - urls: - items: - type: string - minLength: 1 - format: uri - type: array - title: Urls - links: - $ref: '#/components/schemas/FileUploadLinks' - type: object - required: - - chunk_size - - urls - - links - title: FileUploadSchema - FoldersBody: - properties: - source: - type: object - title: Source - destination: - type: object - title: Destination - nodes_map: - additionalProperties: - type: string - format: uuid - type: object - title: Nodes Map - type: object - title: FoldersBody - HealthCheck: - properties: - name: - anyOf: - - type: string - - type: 'null' - title: Name - status: - anyOf: - - type: string - - type: 'null' - title: Status - api_version: - anyOf: - - type: string - - type: 'null' - title: Api Version - version: - anyOf: - - type: string - - type: 'null' - title: Version - type: object - required: - - name - - status - - api_version - - version - title: HealthCheck - LinkType: - type: string - enum: - - PRESIGNED - - S3 - title: LinkType - PresignedLink: - properties: - link: - type: string - minLength: 1 - format: uri - title: Link - type: object - required: - - link - title: PresignedLink - S3Settings: - properties: - S3_ACCESS_KEY: - type: string - maxLength: 50 - minLength: 1 - title: S3 Access Key - S3_BUCKET_NAME: - type: string - maxLength: 50 - minLength: 1 - title: S3 Bucket Name - S3_ENDPOINT: - anyOf: - - type: string - minLength: 1 - format: uri - - type: 'null' - title: S3 Endpoint - description: do not define if using standard AWS - S3_REGION: - type: string - maxLength: 50 - minLength: 1 - title: S3 Region - S3_SECRET_KEY: - type: string - maxLength: 50 - minLength: 1 - title: S3 Secret Key - additionalProperties: false - type: object - required: - - S3_ACCESS_KEY - - S3_BUCKET_NAME - - S3_REGION - - S3_SECRET_KEY - title: S3Settings - SoftCopyBody: - properties: - link_id: - type: string - pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ - title: Link Id - type: object - required: - - link_id - title: SoftCopyBody - TableSynchronisation: - properties: - dry_run: - anyOf: - - type: boolean - - type: 'null' - title: Dry Run - fire_and_forget: - anyOf: - - type: boolean - - type: 'null' - title: Fire And Forget - removed: - items: - type: string - type: array - title: Removed - type: object - required: - - removed - title: TableSynchronisation - TaskGet: - properties: - task_id: - type: string - title: Task Id - task_name: - type: string - title: Task Name - status_href: - type: string - title: Status Href - result_href: - type: string - title: Result Href - abort_href: - type: string - title: Abort Href - type: object - required: - - task_id - - task_name - - status_href - - result_href - - abort_href - title: TaskGet - TaskProgress: - properties: - task_id: - anyOf: - - type: string - - type: 'null' - title: Task Id - message: - type: string - title: Message - default: '' - percent: - type: number - maximum: 1.0 - minimum: 0.0 - title: Percent - default: 0.0 - type: object - title: TaskProgress - description: 'Helps the user to keep track of the progress. Progress is expected - to be - - defined as a float bound between 0.0 and 1.0' - TaskStatus: - properties: - task_progress: - $ref: '#/components/schemas/TaskProgress' - done: - type: boolean - title: Done - started: - type: string - format: date-time - title: Started - type: object - required: - - task_progress - - done - - started - title: TaskStatus - UploadedPart: - properties: - number: - type: integer - exclusiveMinimum: true - title: Number - minimum: 0 - e_tag: - type: string - title: E Tag - type: object - required: - - number - - e_tag - title: UploadedPart -tags: -- name: datasets -- name: files -- name: health -- name: locations -- name: tasks -- name: simcore-s3 diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py deleted file mode 100644 index 16aa8f837eb..00000000000 --- a/services/storage/src/simcore_service_storage/application.py +++ /dev/null @@ -1,108 +0,0 @@ -""" Main's application module for simcore_service_storage service - - Functions to create, setup and run an aiohttp application provided a settingsuration object -""" - -import logging -from typing import Final - -from aiohttp import web -from servicelib.aiohttp.application import create_safe_application -from servicelib.aiohttp.application_keys import APP_CONFIG_KEY -from servicelib.aiohttp.dev_error_logger import setup_dev_error_logger -from servicelib.aiohttp.monitoring import setup_monitoring -from servicelib.aiohttp.profiler_middleware import profiling_middleware -from servicelib.aiohttp.tracing import setup_tracing -from settings_library.tracing import TracingSettings - -from ._meta import APP_NAME, APP_STARTED_BANNER_MSG, VERSION -from .db import setup_db -from .dsm import setup_dsm -from .dsm_cleaner import setup_dsm_cleaner -from .long_running_tasks import setup_rest_api_long_running_tasks -from .redis import setup_redis -from .rest import setup_rest -from .s3 import setup_s3 -from .settings import Settings -from .utils_handlers import dsm_exception_handler - -_ACCESS_LOG_FORMAT: Final[ - str -] = '%a %t "%r" %s %b [%Dus] "%{Referer}i" "%{User-Agent}i"' - -_LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR -_NOISY_LOGGERS = ( - "aiobotocore", - "aio_pika", - "aiormq", - "botocore", - "sqlalchemy", -) -_logger = logging.getLogger(__name__) - - -def create(settings: Settings) -> web.Application: - _logger.debug( - "Initializing app with settings:\n%s", - settings.model_dump_json(indent=2), - ) - - app = create_safe_application(None) - app[APP_CONFIG_KEY] = settings - # Tracing - tracing_settings: TracingSettings | None = app[APP_CONFIG_KEY].STORAGE_TRACING - if tracing_settings: - setup_tracing( - app, - tracing_settings=tracing_settings, - service_name=APP_NAME, - ) - - setup_db(app) - setup_s3(app) - - setup_rest_api_long_running_tasks(app) - setup_rest(app) - - setup_dsm(app) - if settings.STORAGE_CLEANER_INTERVAL_S: - setup_redis(app) - setup_dsm_cleaner(app) - - app.middlewares.append(dsm_exception_handler) - - if settings.STORAGE_PROFILING: - app.middlewares.append(profiling_middleware) - - if settings.LOG_LEVEL == "DEBUG": - setup_dev_error_logger(app) - - if settings.STORAGE_MONITORING_ENABLED: - setup_monitoring(app, APP_NAME, version=f"{VERSION}") - - # keep mostly quiet noisy loggers - quiet_level: int = max( - min(logging.root.level + _LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING - ) - for name in _NOISY_LOGGERS: - logging.getLogger(name).setLevel(quiet_level) - - return app - - -def run(settings: Settings, app: web.Application | None = None): - _logger.debug("Serving application ") - if not app: - app = create(settings) - - async def welcome_banner(_app: web.Application): - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 - - app.on_startup.append(welcome_banner) - - web.run_app( - app, - host=settings.STORAGE_HOST, - port=settings.STORAGE_PORT, - access_log_format=_ACCESS_LOG_FORMAT, - ) diff --git a/services/storage/src/simcore_service_storage/cli.py b/services/storage/src/simcore_service_storage/cli.py index bf37855e6c8..bcf4086f4aa 100644 --- a/services/storage/src/simcore_service_storage/cli.py +++ b/services/storage/src/simcore_service_storage/cli.py @@ -1,44 +1,97 @@ import logging +import os import typer -from servicelib.logging_utils import config_all_loggers -from settings_library.utils_cli import create_settings_command +from settings_library.postgres import PostgresSettings +from settings_library.s3 import S3Settings +from settings_library.utils_cli import ( + create_settings_command, + create_version_callback, + print_as_envfile, +) -from . import application -from .settings import Settings +from ._meta import PROJECT_NAME, __version__ +from .core.settings import ApplicationSettings LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) -main = typer.Typer(name="simcore-service-storage service") +# NOTE: 'main' variable is referred in the setup's entrypoint! +main = typer.Typer(name=PROJECT_NAME) -main.command()(create_settings_command(settings_cls=Settings, logger=log)) +main.command()( + create_settings_command(settings_cls=ApplicationSettings, logger=_logger) +) +main.callback()(create_version_callback(__version__)) @main.command() def run(): """Runs application""" - typer.secho("Resolving settings ...", nl=False) - settings_obj = Settings.create_from_envs() - typer.secho("DONE", fg=typer.colors.GREEN) - - logging.basicConfig(level=settings_obj.log_level) - logging.root.setLevel(settings_obj.log_level) - config_all_loggers( - log_format_local_dev_enabled=settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, - logger_filter_mapping=settings_obj.STORAGE_LOG_FILTER_MAPPING, - tracing_settings=settings_obj.STORAGE_TRACING, + typer.secho("Sorry, this entrypoint is intentionally disabled. Use instead") + typer.secho( + f"$ uvicorn {PROJECT_NAME}.main:the_app", + fg=typer.colors.BLUE, ) - # keep mostly quiet noisy loggers - quiet_level: int = max( - min(logging.root.level + LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING + +@main.command() +def echo_dotenv(ctx: typer.Context, *, minimal: bool = True) -> None: + """Generates and displays a valid environment variables file (also known as dot-envfile) + + Usage: + $ simcore-service echo-dotenv > .env + $ cat .env + $ set -o allexport; source .env; set +o allexport + """ + assert ctx # nosec + + # NOTE: we normally DO NOT USE `os.environ` to capture env vars but this is a special case + # The idea here is to have a command that can generate a **valid** `.env` file that can be used + # to initialized the app. For that reason we fill required fields of the `ApplicationSettings` with + # "fake" but valid values (e.g. generating a password or adding tags as `replace-with-api-key). + # Nonetheless, if the caller of this CLI has already some **valid** env vars in the environment we want to use them ... + # and that is why we use `os.environ`. + + settings = ApplicationSettings.create_from_envs( + STORAGE_POSTGRES=os.environ.get( + "STORAGE_POSTGRES", + PostgresSettings.create_from_envs( + POSTGRES_HOST=os.environ.get( + "POSTGRES_HOST", "replace-with-postgres-host" + ), + POSTGRES_USER=os.environ.get( + "POSTGRES_USER", "replace-with-postgres-user" + ), + POSTGRES_DB=os.environ.get("POSTGRES_DB", "replace-with-postgres-db"), + POSTGRES_PASSWORD=os.environ.get( + "POSTGRES_PASSWORD", "replace-with-postgres-password" + ), + ), + ), + STORAGE_S3=os.environ.get( # nosec + "STORAGE_S3", + S3Settings.create_from_envs( + S3_BUCKET_NAME=os.environ.get("S3_BUCKET", "replace-with-s3-bucket"), + S3_ACCESS_KEY=os.environ.get( + "S3_ACCESS_KEY", "replace-with-s3-access-key" + ), + S3_SECRET_KEY=os.environ.get( + "S3_SECRET_KEY", "replace-with-s3-secret-key" + ), + S3_ENDPOINT=os.environ.get( + "S3_ENDPOINT", "https://s3.replace-with-s3-endpoint" + ), + S3_REGION=os.environ.get("S3_REGION", "replace-with-s3-region"), + ), + ), ) - logging.getLogger("engineio").setLevel(quiet_level) - logging.getLogger("openapi_spec_validator").setLevel(quiet_level) - logging.getLogger("sqlalchemy").setLevel(quiet_level) - logging.getLogger("sqlalchemy.engine").setLevel(quiet_level) - typer.secho("Starting app ... ") - application.run(settings_obj) + print_as_envfile( + settings, + compact=False, + verbose=True, + show_secrets=True, + exclude_unset=minimal, + ) diff --git a/services/storage/src/simcore_service_storage/constants.py b/services/storage/src/simcore_service_storage/constants.py index 498a7c0eebb..fecbfb54e87 100644 --- a/services/storage/src/simcore_service_storage/constants.py +++ b/services/storage/src/simcore_service_storage/constants.py @@ -1,16 +1,14 @@ from typing import Final from aws_library.s3 import PRESIGNED_LINK_MAX_SIZE, S3_MAX_FILE_SIZE -from models_library.api_schemas_storage import LinkType +from models_library.api_schemas_storage.storage_schemas import LinkType from pydantic import ByteSize -from servicelib.aiohttp import application_keys RETRY_WAIT_SECS = 2 MAX_CHUNK_SIZE = 1024 MINUTE = 60 - -APP_CONFIG_KEY = application_keys.APP_CONFIG_KEY # app-storage-key for config object +UPLOAD_TASKS_KEY = f"{__name__}.upload_tasks" # DSM locations SIMCORE_S3_ID = 0 @@ -39,12 +37,7 @@ MAX_CONCURRENT_REST_CALLS: Final[int] = 10 # DATABASE ---------------------------- -APP_AIOPG_ENGINE_KEY = f"{__name__}.aiopg_engine" MAX_CONCURRENT_DB_TASKS: Final[int] = 2 -# DATA STORAGE MANAGER ---------------------------------- -APP_DSM_KEY = f"{__name__}.DSM" -APP_S3_KEY = f"{__name__}.S3_CLIENT" - EXPAND_DIR_MAX_ITEM_COUNT: Final[int] = 1000 diff --git a/services/storage/src/simcore_service_storage/core/__init__.py b/services/storage/src/simcore_service_storage/core/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py new file mode 100644 index 00000000000..b70c2cef302 --- /dev/null +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -0,0 +1,131 @@ +"""Main's application module for simcore_service_storage service + +Functions to create, setup and run an aiohttp application provided a settingsuration object +""" + +import logging + +from common_library.basic_types import BootModeEnum +from fastapi import FastAPI +from fastapi.middleware.gzip import GZipMiddleware +from fastapi_pagination import add_pagination +from servicelib.fastapi import timing_middleware +from servicelib.fastapi.cancellation_middleware import RequestCancellationMiddleware +from servicelib.fastapi.client_session import setup_client_session +from servicelib.fastapi.openapi import override_fastapi_openapi_method +from servicelib.fastapi.profiler import ProfilerMiddleware +from servicelib.fastapi.prometheus_instrumentation import ( + setup_prometheus_instrumentation, +) +from servicelib.fastapi.tracing import initialize_tracing +from starlette.middleware.base import BaseHTTPMiddleware + +from .._meta import ( + API_VERSION, + API_VTAG, + APP_FINISHED_BANNER_MSG, + APP_NAME, + APP_STARTED_BANNER_MSG, + APP_WORKER_STARTED_BANNER_MSG, +) +from ..api.rest.routes import setup_rest_api_routes +from ..api.rpc.routes import setup_rpc_api_routes +from ..dsm import setup_dsm +from ..dsm_cleaner import setup_dsm_cleaner +from ..exceptions.handlers import set_exception_handlers +from ..modules.celery import setup_celery_client +from ..modules.db import setup_db +from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads +from ..modules.rabbitmq import setup as setup_rabbitmq +from ..modules.redis import setup as setup_redis +from ..modules.s3 import setup_s3 +from .settings import ApplicationSettings + +_LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR +_NOISY_LOGGERS = ( + "aio_pika", + "aiobotocore", + "aiormq", + "botocore", + "httpcore", + "urllib3", + "werkzeug", +) +_logger = logging.getLogger(__name__) + + +def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 + # keep mostly quiet noisy loggers + quiet_level: int = max( + min(logging.root.level + _LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING + ) + for name in _NOISY_LOGGERS: + logging.getLogger(name).setLevel(quiet_level) + + _logger.info("app settings: %s", settings.model_dump_json(indent=1)) + + app = FastAPI( + debug=settings.SC_BOOT_MODE + in [BootModeEnum.DEBUG, BootModeEnum.DEVELOPMENT, BootModeEnum.LOCAL], + title=APP_NAME, + description="Service that manages osparc storage backend", + version=API_VERSION, + openapi_url=f"/api/{API_VTAG}/openapi.json", + docs_url="/dev/doc", + redoc_url=None, # default disabled + ) + override_fastapi_openapi_method(app) + add_pagination(app) + + # STATE + app.state.settings = settings + + setup_db(app) + setup_s3(app) + setup_client_session(app) + + if not settings.STORAGE_WORKER_MODE: + setup_rabbitmq(app) + setup_rpc_api_routes(app) + setup_celery_client(app) + setup_rest_api_long_running_tasks_for_uploads(app) + setup_rest_api_routes(app, API_VTAG) + set_exception_handlers(app) + + setup_redis(app) + + setup_dsm(app) + if settings.STORAGE_CLEANER_INTERVAL_S and not settings.STORAGE_WORKER_MODE: + setup_dsm_cleaner(app) + + if settings.STORAGE_PROFILING: + app.add_middleware(ProfilerMiddleware) + + if settings.SC_BOOT_MODE != BootModeEnum.PRODUCTION: + # middleware to time requests (ONLY for development) + app.add_middleware( + BaseHTTPMiddleware, dispatch=timing_middleware.add_process_time_header + ) + + app.add_middleware(GZipMiddleware) + + app.add_middleware(RequestCancellationMiddleware) + + if settings.STORAGE_TRACING: + initialize_tracing(app, settings.STORAGE_TRACING, APP_NAME) + if settings.STORAGE_MONITORING_ENABLED: + setup_prometheus_instrumentation(app) + + async def _on_startup() -> None: + if settings.STORAGE_WORKER_MODE: + print(APP_WORKER_STARTED_BANNER_MSG, flush=True) # noqa: T201 + else: + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + + async def _on_shutdown() -> None: + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + return app diff --git a/services/storage/src/simcore_service_storage/resources.py b/services/storage/src/simcore_service_storage/core/resources.py similarity index 100% rename from services/storage/src/simcore_service_storage/resources.py rename to services/storage/src/simcore_service_storage/core/resources.py diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py new file mode 100644 index 00000000000..4d246a89eeb --- /dev/null +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -0,0 +1,138 @@ +from typing import Annotated, Self + +from fastapi import FastAPI +from pydantic import AliasChoices, Field, PositiveInt, field_validator, model_validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.application import BaseApplicationSettings +from settings_library.basic_types import LogLevel, PortInt +from settings_library.celery import CelerySettings +from settings_library.postgres import PostgresSettings +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings +from settings_library.s3 import S3Settings +from settings_library.tracing import TracingSettings +from settings_library.utils_logging import MixinLoggingSettings + +from ..modules.datcore_adapter.datcore_adapter_settings import DatcoreAdapterSettings + + +class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): + STORAGE_HOST: str = "0.0.0.0" # nosec + STORAGE_PORT: PortInt = 8080 + + LOG_LEVEL: Annotated[ + LogLevel, + Field( + validation_alias=AliasChoices("STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), + ), + ] = LogLevel.INFO + + STORAGE_MONITORING_ENABLED: bool = False + STORAGE_PROFILING: bool = False + + STORAGE_POSTGRES: Annotated[ + PostgresSettings | None, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + + STORAGE_REDIS: Annotated[ + RedisSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_S3: Annotated[ + S3Settings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_CELERY: Annotated[ + CelerySettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_TRACING: Annotated[ + TracingSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + DATCORE_ADAPTER: Annotated[ + DatcoreAdapterSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_SYNC_METADATA_TIMEOUT: Annotated[ + PositiveInt, Field(180, description="Timeout (seconds) for metadata sync task") + ] + + STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS: Annotated[ + int, + Field( + 3600, description="Default expiration time in seconds for presigned links" + ), + ] + + STORAGE_CLEANER_INTERVAL_S: Annotated[ + int | None, + Field( + 30, + description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", + ), + ] + + STORAGE_RABBITMQ: Annotated[ + RabbitSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] + + STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: Annotated[ + int, + Field( + 4, + description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", + ), + ] + + STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + bool, + Field( + default=False, + validation_alias=AliasChoices( + "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + description="Enables local development _logger format. WARNING: make sure it is disabled if you want to have structured logs!", + ), + ] + + STORAGE_LOG_FILTER_MAPPING: Annotated[ + dict[LoggerName, list[MessageSubstring]], + Field( + default_factory=dict, + validation_alias=AliasChoices( + "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", + ), + ] + + STORAGE_WORKER_MODE: Annotated[ + bool, Field(description="If True, run as a worker") + ] = False + + @field_validator("LOG_LEVEL", mode="before") + @classmethod + def _validate_loglevel(cls, value: str) -> str: + log_level: str = cls.validate_log_level(value) + return log_level + + @model_validator(mode="after") + def _ensure_settings_consistency(self) -> Self: + if self.STORAGE_CLEANER_INTERVAL_S is not None and not self.STORAGE_REDIS: + msg = ( + "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! " + "Please correct settings." + ) + raise ValueError(msg) + return self + + +def get_application_settings(app: FastAPI) -> ApplicationSettings: + assert isinstance(app.state.settings, ApplicationSettings) # nosec + return app.state.settings diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py deleted file mode 100644 index 8fb9a162c52..00000000000 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py +++ /dev/null @@ -1,253 +0,0 @@ -import logging -from collections.abc import Callable -from math import ceil -from typing import Any, TypeVar, cast - -import aiohttp -from aiohttp import web -from aiohttp.client import ClientSession -from models_library.api_schemas_storage import DatCoreDatasetName -from models_library.users import UserID -from pydantic import AnyUrl, TypeAdapter -from servicelib.aiohttp.application_keys import APP_CONFIG_KEY -from servicelib.aiohttp.client_session import get_client_session -from servicelib.utils import logged_gather - -from ..constants import DATCORE_ID, DATCORE_STR, MAX_CONCURRENT_REST_CALLS -from ..models import DatasetMetaData, FileMetaData -from .datcore_adapter_exceptions import ( - DatcoreAdapterClientError, - DatcoreAdapterError, - DatcoreAdapterTimeoutError, -) - -log = logging.getLogger(__file__) - - -class _DatcoreAdapterResponseError(DatcoreAdapterError): - """Basic exception for response errors""" - - def __init__(self, status: int, reason: str) -> None: - self.status = status - self.reason = reason - super().__init__( - msg=f"forwarded call failed with status {status}, reason {reason}" - ) - - -async def _request( - app: web.Application, - api_key: str, - api_secret: str, - method: str, - path: str, - *, - json: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - **request_kwargs, -) -> dict[str, Any] | list[dict[str, Any]]: - datcore_adapter_settings = app[APP_CONFIG_KEY].DATCORE_ADAPTER - url = datcore_adapter_settings.endpoint + path - session: ClientSession = get_client_session(app) - - try: - if request_kwargs is None: - request_kwargs = {} - async with session.request( - method, - url, - raise_for_status=True, - headers={ - "x-datcore-api-key": api_key, - "x-datcore-api-secret": api_secret, - }, - json=json, - params=params, - **request_kwargs, - ) as response: - response_data = await response.json() - assert isinstance(response_data, dict | list) # nosec - return response_data - - except aiohttp.ClientResponseError as exc: - raise _DatcoreAdapterResponseError(status=exc.status, reason=f"{exc}") from exc - - except TimeoutError as exc: - msg = f"datcore-adapter server timed-out: {exc}" - raise DatcoreAdapterTimeoutError(msg) from exc - - except aiohttp.ClientError as exc: - msg = f"unexpected client error: {exc}" - raise DatcoreAdapterClientError(msg) from exc - - -_T = TypeVar("_T") - - -async def _retrieve_all_pages( - app: web.Application, - api_key: str, - api_secret: str, - method: str, - path: str, - return_type_creator: Callable[..., _T], -) -> list[_T]: - page = 1 - objs = [] - while ( - response := cast( - dict[str, Any], - await _request( - app, api_key, api_secret, method, path, params={"page": page} - ), - ) - ) and response.get("items"): - log.debug( - "called %s [%d/%d], received %d objects", - path, - page, - ceil(response.get("total", -1) / response.get("size", 1)), - len(response.get("items", [])), - ) - - objs += [return_type_creator(d) for d in response.get("items", [])] - page += 1 - return objs - - -async def check_service_health(app: web.Application) -> bool: - datcore_adapter_settings = app[APP_CONFIG_KEY].DATCORE_ADAPTER - url = datcore_adapter_settings.endpoint + "/ready" - session: ClientSession = get_client_session(app) - try: - await session.get(url, raise_for_status=True) - except (TimeoutError, aiohttp.ClientError): - return False - return True - - -async def check_user_can_connect( - app: web.Application, api_key: str, api_secret: str -) -> bool: - if not api_key or not api_secret: - # no need to ask, datcore is an authenticated service - return False - - try: - await _request(app, api_key, api_secret, "GET", "/user/profile") - return True - except DatcoreAdapterError: - return False - - -async def list_all_datasets_files_metadatas( - app: web.Application, user_id: UserID, api_key: str, api_secret: str -) -> list[FileMetaData]: - all_datasets: list[DatasetMetaData] = await list_datasets(app, api_key, api_secret) - results = await logged_gather( - *( - list_all_files_metadatas_in_dataset( - app, - user_id, - api_key, - api_secret, - cast(DatCoreDatasetName, d.dataset_id), - ) - for d in all_datasets - ), - log=log, - max_concurrency=MAX_CONCURRENT_REST_CALLS, - ) - all_files_of_all_datasets: list[FileMetaData] = [] - for data in results: - all_files_of_all_datasets += data - return all_files_of_all_datasets - - -_LIST_ALL_DATASETS_TIMEOUT_S = 60 - - -async def list_all_files_metadatas_in_dataset( - app: web.Application, - user_id: UserID, - api_key: str, - api_secret: str, - dataset_id: DatCoreDatasetName, -) -> list[FileMetaData]: - all_files: list[dict[str, Any]] = cast( - list[dict[str, Any]], - await _request( - app, - api_key, - api_secret, - "GET", - f"/datasets/{dataset_id}/files_legacy", - timeout=aiohttp.ClientTimeout(total=_LIST_ALL_DATASETS_TIMEOUT_S), - ), - ) - return [ - FileMetaData.model_construct( - file_uuid=d["path"], - location_id=DATCORE_ID, - location=DATCORE_STR, - bucket_name=d["dataset_id"], - object_name=d["path"], - file_name=d["name"], - file_id=d["package_id"], - file_size=d["size"], - created_at=d["created_at"], - last_modified=d["last_modified_at"], - project_id=None, - node_id=None, - user_id=user_id, - is_soft_link=False, - ) - for d in all_files - ] - - -async def list_datasets( - app: web.Application, api_key: str, api_secret: str -) -> list[DatasetMetaData]: - all_datasets: list[DatasetMetaData] = await _retrieve_all_pages( - app, - api_key, - api_secret, - "GET", - "/datasets", - lambda d: DatasetMetaData(dataset_id=d["id"], display_name=d["display_name"]), - ) - - return all_datasets - - -async def get_file_download_presigned_link( - app: web.Application, api_key: str, api_secret: str, file_id: str -) -> AnyUrl: - file_download_data = cast( - dict[str, Any], - await _request(app, api_key, api_secret, "GET", f"/files/{file_id}"), - ) - url: AnyUrl = TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) - return url - - -async def get_package_files( - app: web.Application, api_key: str, api_secret: str, package_id: str -) -> list[dict[str, Any]]: - return cast( - list[dict[str, Any]], - await _request( - app, - api_key, - api_secret, - "GET", - f"/packages/{package_id}/files", - ), - ) - - -async def delete_file( - app: web.Application, api_key: str, api_secret: str, file_id: str -) -> None: - await _request(app, api_key, api_secret, "DELETE", f"/files/{file_id}") diff --git a/services/storage/src/simcore_service_storage/datcore_dsm.py b/services/storage/src/simcore_service_storage/datcore_dsm.py index 3ced68fa3f4..fef1aa992d6 100644 --- a/services/storage/src/simcore_service_storage/datcore_dsm.py +++ b/services/storage/src/simcore_service_storage/datcore_dsm.py @@ -1,8 +1,14 @@ +import contextlib from dataclasses import dataclass +from pathlib import Path -from aiohttp import web -from models_library.api_schemas_storage import ( +import arrow +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( + UNDEFINED_SIZE_TYPE, + DatCoreCollectionName, DatCoreDatasetName, + DatCorePackageName, LinkType, UploadedPart, ) @@ -10,22 +16,54 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize +from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter, ValidationError from .constants import DATCORE_ID, DATCORE_STR -from .datcore_adapter import datcore_adapter -from .datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterMultipleFilesError -from .db_tokens import get_api_token_and_secret from .dsm_factory import BaseDataManager -from .models import DatasetMetaData, FileMetaData, UploadLinks +from .exceptions.errors import DatCoreCredentialsMissingError +from .models import ( + DatasetMetaData, + FileMetaData, + GenericCursor, + PathMetaData, + TotalNumber, + UploadLinks, +) +from .modules.datcore_adapter import datcore_adapter +from .modules.datcore_adapter.datcore_adapter_exceptions import ( + DatcoreAdapterMultipleFilesError, +) +from .modules.db import get_db_engine +from .modules.db.tokens import TokenRepository + + +def _check_api_credentials( + api_token: str | None, api_secret: str | None +) -> tuple[str, str]: + if not api_token or not api_secret: + raise DatCoreCredentialsMissingError + assert api_token is not None + assert api_secret is not None + return api_token, api_secret + + +def _is_collection(file_filter: Path) -> bool: + with contextlib.suppress(ValidationError): + TypeAdapter(DatCoreCollectionName).validate_python(file_filter.parts[1]) + return True + return False @dataclass class DatCoreDataManager(BaseDataManager): - app: web.Application + app: FastAPI - async def _get_datcore_tokens(self, user_id: UserID): - return await get_api_token_and_secret(self.app, user_id) + async def _get_datcore_tokens( + self, user_id: UserID + ) -> tuple[str | None, str | None]: + return await TokenRepository.instance( + get_db_engine(self.app) + ).get_api_token_and_secret(user_id=user_id) @classmethod def get_location_id(cls) -> LocationID: @@ -45,16 +83,171 @@ async def authorized(self, user_id: UserID) -> bool: async def list_datasets(self, user_id: UserID) -> list[DatasetMetaData]: api_token, api_secret = await self._get_datcore_tokens(user_id) - return await datcore_adapter.list_datasets(self.app, api_token, api_secret) + api_token, api_secret = _check_api_credentials(api_token, api_secret) + return await datcore_adapter.list_all_datasets(self.app, api_token, api_secret) async def list_files_in_dataset( self, user_id: UserID, dataset_id: str, *, expand_dirs: bool ) -> list[FileMetaData]: api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) return await datcore_adapter.list_all_files_metadatas_in_dataset( - self.app, user_id, api_token, api_secret, DatCoreDatasetName(dataset_id) + self.app, user_id, api_token, api_secret, dataset_id ) + async def list_paths( + self, + user_id: UserID, + *, + file_filter: Path | None, + cursor: GenericCursor | None, + limit: NonNegativeInt, + ) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber | None]: + """returns a page of the file meta data a user has access to""" + api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) + if not file_filter: + datasets, next_cursor, total = await datcore_adapter.list_datasets( + self.app, + api_key=api_token, + api_secret=api_secret, + cursor=cursor, + limit=limit, + ) + return ( + [ + PathMetaData( + path=Path(f"{dataset.dataset_id}"), + display_path=Path(f"{dataset.display_name}"), + location_id=self.location_id, + location=self.location_name, + bucket_name="fake", + project_id=None, + node_id=None, + user_id=user_id, + created_at=arrow.utcnow().datetime, + last_modified=arrow.utcnow().datetime, + file_meta_data=None, + ) + for dataset in datasets + ], + next_cursor, + total, + ) + assert len(file_filter.parts) + + if len(file_filter.parts) == 1: + # this is looking into a dataset + return await datcore_adapter.list_top_level_objects_in_dataset( + self.app, + user_id=user_id, + api_key=api_token, + api_secret=api_secret, + dataset_id=TypeAdapter(DatCoreDatasetName).validate_python( + file_filter.parts[0] + ), + cursor=cursor, + limit=limit, + ) + assert len(file_filter.parts) == 2 + + if _is_collection(file_filter): + # this is a collection + return await datcore_adapter.list_top_level_objects_in_collection( + self.app, + user_id=user_id, + api_key=api_token, + api_secret=api_secret, + dataset_id=TypeAdapter(DatCoreDatasetName).validate_python( + file_filter.parts[0] + ), + collection_id=TypeAdapter(DatCoreCollectionName).validate_python( + file_filter.parts[1] + ), + cursor=cursor, + limit=limit, + ) + assert TypeAdapter(DatCorePackageName).validate_python( + file_filter.parts[1] + ) # nosec + + # only other option is a file or maybe a partial?? that would be bad + return ( + [ + await datcore_adapter.get_package_file_as_path( + self.app, + user_id=user_id, + api_key=api_token, + api_secret=api_secret, + dataset_id=TypeAdapter(DatCoreDatasetName).validate_python( + file_filter.parts[0] + ), + package_id=TypeAdapter(DatCorePackageName).validate_python( + file_filter.parts[1] + ), + ) + ], + None, + 1, + ) + + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) + + # if this is a dataset we might have the size directly + with contextlib.suppress(ValidationError): + dataset_id = TypeAdapter(DatCoreDatasetName).validate_python(f"{path}") + _, dataset_size = await datcore_adapter.get_dataset( + self.app, + api_key=api_token, + api_secret=api_secret, + dataset_id=dataset_id, + ) + if dataset_size is not None: + return dataset_size + + # generic computation (slow and unoptimized - could be improved if necessary by using datcore data better) + try: + accumulated_size = ByteSize(0) + paths_to_process = [path] + + while paths_to_process: + current_path = paths_to_process.pop() + paths, cursor, _ = await self.list_paths( + user_id, file_filter=current_path, cursor=None, limit=50 + ) + + while paths: + for p in paths: + if p.file_meta_data is not None: + # this is a file + assert ( + p.file_meta_data.file_size is not UNDEFINED_SIZE_TYPE + ) # nosec + assert isinstance( + p.file_meta_data.file_size, ByteSize + ) # nosec + accumulated_size = ByteSize( + accumulated_size + p.file_meta_data.file_size + ) + continue + paths_to_process.append(p.path) + + if cursor: + paths, cursor, _ = await self.list_paths( + user_id, file_filter=current_path, cursor=cursor, limit=50 + ) + else: + break + + return accumulated_size + + except ValidationError: + # invalid path + return ByteSize(0) + async def list_files( self, user_id: UserID, @@ -64,34 +257,37 @@ async def list_files( project_id: ProjectID | None, ) -> list[FileMetaData]: api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) return await datcore_adapter.list_all_datasets_files_metadatas( self.app, user_id, api_token, api_secret ) async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaData: api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) package_files = await datcore_adapter.get_package_files( - self.app, api_token, api_secret, file_id + self.app, api_key=api_token, api_secret=api_secret, package_id=file_id ) if not len(package_files) == 1: raise DatcoreAdapterMultipleFilesError( msg=f"{len(package_files)} files in package, this breaks the current assumption" ) - resp_data = package_files[0]["content"] + + file = package_files[0] return FileMetaData( file_uuid=file_id, location_id=DATCORE_ID, location=DATCORE_STR, - bucket_name=resp_data["s3bucket"], + bucket_name=file.s3_bucket, object_name=file_id, - file_name=resp_data["filename"], + file_name=file.filename, file_id=file_id, - file_size=resp_data["size"], - created_at=resp_data["createdAt"], - last_modified=resp_data["updatedAt"], + file_size=file.size, + created_at=file.created_at, + last_modified=file.updated_at, project_id=None, node_id=None, user_id=user_id, @@ -126,14 +322,16 @@ async def create_file_download_link( self, user_id: UserID, file_id: StorageFileID, link_type: LinkType ) -> AnyUrl: api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) return await datcore_adapter.get_file_download_presigned_link( self.app, api_token, api_secret, file_id ) async def delete_file(self, user_id: UserID, file_id: StorageFileID) -> None: api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) await datcore_adapter.delete_file(self.app, api_token, api_secret, file_id) -def create_datcore_data_manager(app: web.Application) -> DatCoreDataManager: +def create_datcore_data_manager(app: FastAPI) -> DatCoreDataManager: return DatCoreDataManager(app) diff --git a/services/storage/src/simcore_service_storage/db.py b/services/storage/src/simcore_service_storage/db.py deleted file mode 100644 index 2dbb7dc8704..00000000000 --- a/services/storage/src/simcore_service_storage/db.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -from typing import Any - -from aiohttp import web -from aiopg.sa.engine import Engine -from servicelib.aiohttp.aiopg_utils import is_pg_responsive -from servicelib.common_aiopg_utils import DataSourceName, create_pg_engine -from servicelib.retry_policies import PostgresRetryPolicyUponInitialization -from settings_library.postgres import PostgresSettings -from simcore_postgres_database.utils_aiopg import ( - get_pg_engine_stateinfo, - raise_if_migration_not_ready, -) -from tenacity import retry - -from .constants import APP_AIOPG_ENGINE_KEY, APP_CONFIG_KEY - -_logger = logging.getLogger(__name__) - - -@retry(**PostgresRetryPolicyUponInitialization(_logger).kwargs) -async def _ensure_pg_ready(dsn: DataSourceName, min_size: int, max_size: int) -> None: - _logger.info("Checking pg is ready %s", dsn) - - async with create_pg_engine(dsn, minsize=min_size, maxsize=max_size) as engine: - await raise_if_migration_not_ready(engine) - - -async def postgres_cleanup_ctx(app: web.Application): - pg_cfg: PostgresSettings = app[APP_CONFIG_KEY].STORAGE_POSTGRES - dsn = DataSourceName( - application_name=f"{__name__}_{id(app)}", - database=pg_cfg.POSTGRES_DB, - user=pg_cfg.POSTGRES_USER, - password=pg_cfg.POSTGRES_PASSWORD.get_secret_value(), - host=pg_cfg.POSTGRES_HOST, - port=pg_cfg.POSTGRES_PORT, - ) - - await _ensure_pg_ready( - dsn, min_size=pg_cfg.POSTGRES_MINSIZE, max_size=pg_cfg.POSTGRES_MAXSIZE - ) - _logger.info("Creating pg engine for %s", dsn) - async with create_pg_engine( - dsn, minsize=pg_cfg.POSTGRES_MINSIZE, maxsize=pg_cfg.POSTGRES_MAXSIZE - ) as engine: - - assert engine # nosec - app[APP_AIOPG_ENGINE_KEY] = engine - - _logger.info("Created pg engine for %s", dsn) - yield # ---------- - _logger.info("Deleting pg engine for %s", dsn) - _logger.info("Deleted pg engine for %s", dsn) - - -async def is_service_responsive(app: web.Application) -> bool: - """Returns true if the app can connect to db service""" - return await is_pg_responsive(engine=app[APP_AIOPG_ENGINE_KEY]) - - -def get_engine_state(app: web.Application) -> dict[str, Any]: - engine: Engine | None = app.get(APP_AIOPG_ENGINE_KEY) - if engine: - engine_info: dict[str, Any] = get_pg_engine_stateinfo(engine) - return engine_info - return {} - - -def setup_db(app: web.Application): - app[APP_AIOPG_ENGINE_KEY] = None - - # app is created at this point but not yet started - _logger.debug("Setting up %s [service: %s] ...", __name__, "postgres") - - # async connection to db - app.cleanup_ctx.append(postgres_cleanup_ctx) diff --git a/services/storage/src/simcore_service_storage/db_file_meta_data.py b/services/storage/src/simcore_service_storage/db_file_meta_data.py deleted file mode 100644 index 593a48f72b2..00000000000 --- a/services/storage/src/simcore_service_storage/db_file_meta_data.py +++ /dev/null @@ -1,207 +0,0 @@ -import datetime -from collections.abc import AsyncGenerator - -import sqlalchemy as sa -from aiopg.sa.connection import SAConnection -from models_library.basic_types import SHA256Str -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from models_library.users import UserID -from models_library.utils.fastapi_encoders import jsonable_encoder -from simcore_postgres_database.storage_models import file_meta_data -from sqlalchemy import and_, literal_column -from sqlalchemy.dialects.postgresql import insert as pg_insert - -from .exceptions import FileMetaDataNotFoundError -from .models import FileMetaData, FileMetaDataAtDB, UserOrProjectFilter - - -async def exists(conn: SAConnection, file_id: SimcoreS3FileID) -> bool: - return bool( - await conn.scalar( - sa.select(sa.func.count()) - .select_from(file_meta_data) - .where(file_meta_data.c.file_id == file_id) - ) - == 1 - ) - - -async def upsert( - conn: SAConnection, fmd: FileMetaData | FileMetaDataAtDB -) -> FileMetaDataAtDB: - # NOTE: upsert file_meta_data, if the file already exists, we update the whole row - # so we get the correct time stamps - fmd_db = ( - FileMetaDataAtDB.model_validate(fmd) if isinstance(fmd, FileMetaData) else fmd - ) - insert_statement = pg_insert(file_meta_data).values(**jsonable_encoder(fmd_db)) - on_update_statement = insert_statement.on_conflict_do_update( - index_elements=[file_meta_data.c.file_id], set_=jsonable_encoder(fmd_db) - ).returning(literal_column("*")) - result = await conn.execute(on_update_statement) - row = await result.first() - assert row # nosec - return FileMetaDataAtDB.model_validate(row) - - -async def insert(conn: SAConnection, fmd: FileMetaData) -> FileMetaDataAtDB: - fmd_db = FileMetaDataAtDB.model_validate(fmd) - result = await conn.execute( - file_meta_data.insert() - .values(jsonable_encoder(fmd_db)) - .returning(literal_column("*")) - ) - row = await result.first() - assert row # nosec - return FileMetaDataAtDB.model_validate(row) - - -async def get(conn: SAConnection, file_id: SimcoreS3FileID) -> FileMetaDataAtDB: - result = await conn.execute( - query=sa.select(file_meta_data).where(file_meta_data.c.file_id == file_id) - ) - if row := await result.first(): - return FileMetaDataAtDB.model_validate(row) - raise FileMetaDataNotFoundError(file_id=file_id) - - -def _list_filter_with_partial_file_id_stmt( - *, - user_or_project_filter: UserOrProjectFilter, - file_id_prefix: str | None, - partial_file_id: str | None, - sha256_checksum: SHA256Str | None, - is_directory: bool | None, - limit: int | None = None, - offset: int | None = None, -): - conditions: list = [] - - # Checks access rights (project can be owned or shared) - user_id = user_or_project_filter.user_id - if user_id is not None: - project_ids = user_or_project_filter.project_ids - conditions.append( - sa.or_( - file_meta_data.c.user_id == f"{user_id}", - ( - file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) - if project_ids - else False - ), - ) - ) - - # Optional filters - if file_id_prefix: - conditions.append(file_meta_data.c.file_id.startswith(file_id_prefix)) - if partial_file_id: - conditions.append(file_meta_data.c.file_id.ilike(f"%{partial_file_id}%")) - if is_directory is not None: - conditions.append(file_meta_data.c.is_directory.is_(is_directory)) - if sha256_checksum: - conditions.append(file_meta_data.c.sha256_checksum == sha256_checksum) - - return ( - sa.select(file_meta_data) - .where(sa.and_(*conditions)) - .order_by(file_meta_data.c.created_at.asc()) # sorted as oldest first - .offset(offset) - .limit(limit) - ) - - -async def list_filter_with_partial_file_id( - conn: SAConnection, - *, - user_or_project_filter: UserOrProjectFilter, - file_id_prefix: str | None, - partial_file_id: str | None, - sha256_checksum: SHA256Str | None, - is_directory: bool | None, - limit: int | None = None, - offset: int | None = None, -) -> list[FileMetaDataAtDB]: - - stmt = _list_filter_with_partial_file_id_stmt( - user_or_project_filter=user_or_project_filter, - file_id_prefix=file_id_prefix, - partial_file_id=partial_file_id, - sha256_checksum=sha256_checksum, - is_directory=is_directory, - limit=limit, - offset=offset, - ) - - return [ - FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) - ] - - -async def list_fmds( - conn: SAConnection, - *, - user_id: UserID | None = None, - project_ids: list[ProjectID] | None = None, - file_ids: list[SimcoreS3FileID] | None = None, - expired_after: datetime.datetime | None = None, -) -> list[FileMetaDataAtDB]: - stmt = sa.select(file_meta_data).where( - and_( - (file_meta_data.c.user_id == f"{user_id}") if user_id else True, - ( - (file_meta_data.c.project_id.in_([f"{p}" for p in project_ids])) - if project_ids - else True - ), - (file_meta_data.c.file_id.in_(file_ids)) if file_ids else True, - ( - (file_meta_data.c.upload_expires_at < expired_after) - if expired_after - else True - ), - ) - ) - - return [ - FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) - ] - - -async def total(conn: SAConnection) -> int: - """returns the number of uploaded file entries""" - return ( - await conn.scalar(sa.select(sa.func.count()).select_from(file_meta_data)) or 0 - ) - - -async def list_valid_uploads( - conn: SAConnection, -) -> AsyncGenerator[FileMetaDataAtDB, None]: - """returns all the theoretically valid fmds (e.g. upload_expires_at column is null)""" - async for row in conn.execute( - sa.select(file_meta_data).where( - file_meta_data.c.upload_expires_at == None # lgtm [py/test-equals-none] - ) - ): - fmd_at_db = FileMetaDataAtDB.model_validate(row) - yield fmd_at_db - - -async def delete(conn: SAConnection, file_ids: list[SimcoreS3FileID]) -> None: - await conn.execute( - file_meta_data.delete().where(file_meta_data.c.file_id.in_(file_ids)) - ) - - -async def delete_all_from_project(conn: SAConnection, project_id: ProjectID) -> None: - await conn.execute( - file_meta_data.delete().where(file_meta_data.c.project_id == f"{project_id}") - ) - - -async def delete_all_from_node(conn: SAConnection, node_id: NodeID) -> None: - await conn.execute( - file_meta_data.delete().where(file_meta_data.c.node_id == f"{node_id}") - ) diff --git a/services/storage/src/simcore_service_storage/db_projects.py b/services/storage/src/simcore_service_storage/db_projects.py deleted file mode 100644 index dc680c491ee..00000000000 --- a/services/storage/src/simcore_service_storage/db_projects.py +++ /dev/null @@ -1,39 +0,0 @@ -from collections.abc import AsyncIterator -from contextlib import suppress - -import sqlalchemy as sa -from aiopg.sa.connection import SAConnection -from models_library.projects import ProjectAtDB, ProjectID -from pydantic import ValidationError -from simcore_postgres_database.storage_models import projects - - -async def list_valid_projects_in( - conn: SAConnection, - include_uuids: list[ProjectID], -) -> AsyncIterator[ProjectAtDB]: - """ - - NOTE that it lists ONLY validated projects in 'project_uuids' - """ - async for row in conn.execute( - sa.select(projects).where( - projects.c.uuid.in_(f"{pid}" for pid in include_uuids) - ) - ): - with suppress(ValidationError): - yield ProjectAtDB.model_validate(row) - - -async def project_exists( - conn: SAConnection, - project_uuid: ProjectID, -) -> bool: - return bool( - await conn.scalar( - sa.select(sa.func.count()) - .select_from(projects) - .where(projects.c.uuid == f"{project_uuid}") - ) - == 1 - ) diff --git a/services/storage/src/simcore_service_storage/db_tokens.py b/services/storage/src/simcore_service_storage/db_tokens.py deleted file mode 100644 index 445a7c220d1..00000000000 --- a/services/storage/src/simcore_service_storage/db_tokens.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging -from typing import Any - -import sqlalchemy as sa -from aiohttp import web -from aiopg.sa.engine import Engine -from models_library.users import UserID -from simcore_postgres_database.storage_models import tokens - -from .constants import APP_AIOPG_ENGINE_KEY, APP_CONFIG_KEY - -log = logging.getLogger(__name__) - - -async def _get_tokens_from_db(engine: Engine, user_id: UserID) -> dict[str, Any]: - async with engine.acquire() as conn: - result = await conn.execute( - sa.select( - tokens, - ).where(tokens.c.user_id == user_id) - ) - row = await result.first() - return dict(row) if row else {} - - -async def get_api_token_and_secret( - app: web.Application, user_id: UserID -) -> tuple[str, str]: - # from the client side together with the userid? - engine = app[APP_AIOPG_ENGINE_KEY] - - # defaults from config if any, othewise None - api_token = app[APP_CONFIG_KEY].BF_API_KEY - api_secret = app[APP_CONFIG_KEY].BF_API_SECRET - - data = await _get_tokens_from_db(engine, user_id) - - data = data.get("token_data", {}) - api_token = data.get("token_key", api_token) - api_secret = data.get("token_secret", api_secret) - - return api_token, api_secret diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 4d69f32bd99..64d72a29404 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -1,17 +1,18 @@ import logging +from typing import cast -from aiohttp import web +from fastapi import FastAPI -from .constants import APP_DSM_KEY from .datcore_dsm import DatCoreDataManager, create_datcore_data_manager from .dsm_factory import DataManagerProvider +from .exceptions.errors import ConfigurationError from .simcore_s3_dsm import SimcoreS3DataManager, create_simcore_s3_data_manager logger = logging.getLogger(__name__) -def setup_dsm(app: web.Application): - async def _cleanup_context(app: web.Application): +def setup_dsm(app: FastAPI) -> None: + async def _on_startup() -> None: dsm_provider = DataManagerProvider(app) dsm_provider.register_builder( SimcoreS3DataManager.get_location_id(), @@ -23,17 +24,22 @@ async def _cleanup_context(app: web.Application): create_datcore_data_manager, DatCoreDataManager, ) - app[APP_DSM_KEY] = dsm_provider + app.state.dsm_provider = dsm_provider - yield - - logger.info("Shuting down %s", f"{dsm_provider=}") + async def _on_shutdown() -> None: + if app.state.dsm_provider: + # nothing to do + ... # ------ - app.cleanup_ctx.append(_cleanup_context) + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) -def get_dsm_provider(app: web.Application) -> DataManagerProvider: - dsm_provider: DataManagerProvider = app[APP_DSM_KEY] - return dsm_provider +def get_dsm_provider(app: FastAPI) -> DataManagerProvider: + if not app.state.dsm_provider: + raise ConfigurationError( + msg="DSM provider not available. Please check the configuration." + ) + return cast(DataManagerProvider, app.state.dsm_provider) diff --git a/services/storage/src/simcore_service_storage/dsm_cleaner.py b/services/storage/src/simcore_service_storage/dsm_cleaner.py index fe3fcf897ea..d09c83e4f5d 100644 --- a/services/storage/src/simcore_service_storage/dsm_cleaner.py +++ b/services/storage/src/simcore_service_storage/dsm_cleaner.py @@ -23,15 +23,14 @@ from datetime import timedelta from typing import cast -from aiohttp import web +from fastapi import FastAPI from servicelib.async_utils import cancel_wait_task from servicelib.background_task_utils import exclusive_periodic -from servicelib.logging_utils import log_catch, log_context +from servicelib.logging_utils import log_context -from .constants import APP_CONFIG_KEY, APP_DSM_KEY -from .dsm_factory import DataManagerProvider -from .redis import get_redis_client -from .settings import Settings +from .core.settings import get_application_settings +from .dsm import get_dsm_provider +from .modules.redis import get_redis_client from .simcore_s3_dsm import SimcoreS3DataManager _logger = logging.getLogger(__name__) @@ -39,37 +38,35 @@ _TASK_NAME_PERIODICALY_CLEAN_DSM = "periodic_cleanup_of_dsm" -async def dsm_cleaner_task(app: web.Application) -> None: - _logger.info("starting dsm cleaner task...") - dsm: DataManagerProvider = app[APP_DSM_KEY] - simcore_s3_dsm: SimcoreS3DataManager = cast( - SimcoreS3DataManager, dsm.get(SimcoreS3DataManager.get_location_id()) - ) - await simcore_s3_dsm.clean_expired_uploads() +async def dsm_cleaner_task(app: FastAPI) -> None: + with log_context(_logger, logging.INFO, "dsm cleaner task"): + dsm = get_dsm_provider(app) + simcore_s3_dsm: SimcoreS3DataManager = cast( + SimcoreS3DataManager, dsm.get(SimcoreS3DataManager.get_location_id()) + ) + await simcore_s3_dsm.clean_expired_uploads() -def setup_dsm_cleaner(app: web.Application): - async def _setup(app: web.Application): - with ( - log_context(_logger, logging.INFO, msg="setup dsm cleaner"), - log_catch(_logger, reraise=False), - ): - cfg: Settings = app[APP_CONFIG_KEY] - assert cfg.STORAGE_CLEANER_INTERVAL_S # nosec +def setup_dsm_cleaner(app: FastAPI) -> None: + async def _on_startup() -> None: + cfg = get_application_settings(app) + assert cfg.STORAGE_CLEANER_INTERVAL_S # nosec - @exclusive_periodic( - get_redis_client(app), - task_interval=timedelta(seconds=cfg.STORAGE_CLEANER_INTERVAL_S), - retry_after=timedelta(minutes=5), - ) - async def _periodic_dsm_clean() -> None: - await dsm_cleaner_task(app) + @exclusive_periodic( + get_redis_client(app), + task_interval=timedelta(seconds=cfg.STORAGE_CLEANER_INTERVAL_S), + retry_after=timedelta(minutes=5), + ) + async def _periodic_dsm_clean() -> None: + await dsm_cleaner_task(app) - storage_background_task = asyncio.create_task( - _periodic_dsm_clean(), name=_TASK_NAME_PERIODICALY_CLEAN_DSM - ) - yield + app.state.dsm_cleaner_task = asyncio.create_task( + _periodic_dsm_clean(), name=_TASK_NAME_PERIODICALY_CLEAN_DSM + ) - await cancel_wait_task(storage_background_task) + async def _on_shutdown() -> None: + assert isinstance(app.state.dsm_cleaner_task, asyncio.Task) # nosec + await cancel_wait_task(app.state.dsm_cleaner_task) - app.cleanup_ctx.append(_setup) + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/storage/src/simcore_service_storage/dsm_factory.py b/services/storage/src/simcore_service_storage/dsm_factory.py index c84ca4f15db..749bbf9a5e6 100644 --- a/services/storage/src/simcore_service_storage/dsm_factory.py +++ b/services/storage/src/simcore_service_storage/dsm_factory.py @@ -1,16 +1,24 @@ from abc import ABC, abstractmethod from collections.abc import Callable from dataclasses import dataclass, field +from pathlib import Path -from aiohttp import web -from models_library.api_schemas_storage import LinkType, UploadedPart +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import LinkType, UploadedPart from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize +from pydantic import AnyUrl, ByteSize, NonNegativeInt -from .models import DatasetMetaData, FileMetaData, UploadLinks +from .models import ( + DatasetMetaData, + FileMetaData, + GenericCursor, + PathMetaData, + TotalNumber, + UploadLinks, +) class BaseDataManager(ABC): @@ -61,6 +69,21 @@ async def list_files( """returns all the file meta data a user has access to (uuid_filter and or project_id may be used)""" # NOTE: expand_dirs will be replaced by pagination in the future + @abstractmethod + async def list_paths( + self, + user_id: UserID, + *, + file_filter: Path | None, + cursor: GenericCursor | None, + limit: NonNegativeInt, + ) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber | None]: + """returns a page of the file meta data a user has access to""" + + @abstractmethod + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + @abstractmethod async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaData: """returns the file meta data of file_id if user_id has the rights to""" @@ -105,17 +128,17 @@ async def delete_file(self, user_id: UserID, file_id: StorageFileID) -> None: @dataclass class DataManagerProvider: - app: web.Application + app: FastAPI _builders: dict[ LocationID, - tuple[Callable[[web.Application], BaseDataManager], type[BaseDataManager]], + tuple[Callable[[FastAPI], BaseDataManager], type[BaseDataManager]], ] = field(default_factory=dict) _services: list[BaseDataManager] = field(default_factory=list) def register_builder( self, location_id: LocationID, - builder: Callable[[web.Application], BaseDataManager], + builder: Callable[[FastAPI], BaseDataManager], dsm_type: type[BaseDataManager], ): self._builders[location_id] = (builder, dsm_type) diff --git a/services/storage/src/simcore_service_storage/exceptions/__init__.py b/services/storage/src/simcore_service_storage/exceptions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/exceptions.py b/services/storage/src/simcore_service_storage/exceptions/errors.py similarity index 65% rename from services/storage/src/simcore_service_storage/exceptions.py rename to services/storage/src/simcore_service_storage/exceptions/errors.py index 937a3afdd06..5856a2fec5b 100644 --- a/services/storage/src/simcore_service_storage/exceptions.py +++ b/services/storage/src/simcore_service_storage/exceptions/errors.py @@ -5,6 +5,10 @@ class StorageRuntimeError(OsparcErrorMixin, RuntimeError): ... +class ConfigurationError(StorageRuntimeError): + msg_template: str = "Application misconfiguration: {msg}" + + class DatabaseAccessError(StorageRuntimeError): msg_template: str = "Unexpected error while accessing database backend" @@ -29,3 +33,15 @@ class ProjectNotFoundError(DatabaseAccessError): class LinkAlreadyExistsError(DatabaseAccessError): msg_template: str = "The link {file_id} already exists" + + +class AccessLayerError(StorageRuntimeError): + msg_template: str = "Database access layer error" + + +class InvalidFileIdentifierError(AccessLayerError): + msg_template: str = "Error in {identifier}: {details}" + + +class DatCoreCredentialsMissingError(StorageRuntimeError): + msg_template: str = "DatCore credentials are incomplete. TIP: Check your settings" diff --git a/services/storage/src/simcore_service_storage/exceptions/handlers.py b/services/storage/src/simcore_service_storage/exceptions/handlers.py new file mode 100644 index 00000000000..78a93d8f46c --- /dev/null +++ b/services/storage/src/simcore_service_storage/exceptions/handlers.py @@ -0,0 +1,98 @@ +import logging + +from asyncpg.exceptions import PostgresError +from aws_library.s3 import S3AccessError, S3KeyNotFoundError +from fastapi import FastAPI, status +from servicelib.fastapi.http_error import ( + make_http_error_handler_for_exception, + set_app_default_http_error_handlers, +) + +from ..modules.datcore_adapter.datcore_adapter_exceptions import ( + DatcoreAdapterFileNotFoundError, + DatcoreAdapterTimeoutError, +) +from .errors import ( + DatCoreCredentialsMissingError, + FileAccessRightError, + FileMetaDataNotFoundError, + InvalidFileIdentifierError, + LinkAlreadyExistsError, + ProjectAccessRightError, + ProjectNotFoundError, +) + +_logger = logging.getLogger(__name__) + + +def set_exception_handlers(app: FastAPI) -> None: + set_app_default_http_error_handlers(app) + + # + # add custom exception handlers + # + app.add_exception_handler( + InvalidFileIdentifierError, + make_http_error_handler_for_exception( + status.HTTP_422_UNPROCESSABLE_ENTITY, + InvalidFileIdentifierError, + envelope_error=True, + ), + ) + for exc_not_found in ( + FileMetaDataNotFoundError, + S3KeyNotFoundError, + ProjectNotFoundError, + DatcoreAdapterFileNotFoundError, + ): + app.add_exception_handler( + exc_not_found, + make_http_error_handler_for_exception( + status.HTTP_404_NOT_FOUND, exc_not_found, envelope_error=True + ), + ) + for exc_access in ( + FileAccessRightError, + ProjectAccessRightError, + ): + app.add_exception_handler( + exc_access, + make_http_error_handler_for_exception( + status.HTTP_403_FORBIDDEN, exc_access, envelope_error=True + ), + ) + app.add_exception_handler( + LinkAlreadyExistsError, + make_http_error_handler_for_exception( + status.HTTP_422_UNPROCESSABLE_ENTITY, + LinkAlreadyExistsError, + envelope_error=True, + ), + ) + for exc_3rd_party in ( + PostgresError, + S3AccessError, + ): + app.add_exception_handler( + exc_3rd_party, + make_http_error_handler_for_exception( + status.HTTP_503_SERVICE_UNAVAILABLE, exc_3rd_party, envelope_error=True + ), + ) + + app.add_exception_handler( + DatcoreAdapterTimeoutError, + make_http_error_handler_for_exception( + status.HTTP_504_GATEWAY_TIMEOUT, + DatcoreAdapterTimeoutError, + envelope_error=True, + ), + ) + app.add_exception_handler( + DatCoreCredentialsMissingError, + make_http_error_handler_for_exception( + status.HTTP_401_UNAUTHORIZED, + DatCoreCredentialsMissingError, + envelope_error=True, + ), + ) diff --git a/services/storage/src/simcore_service_storage/handlers_datasets.py b/services/storage/src/simcore_service_storage/handlers_datasets.py deleted file mode 100644 index 2cd510bcf79..00000000000 --- a/services/storage/src/simcore_service_storage/handlers_datasets.py +++ /dev/null @@ -1,74 +0,0 @@ -import logging - -from aiohttp import web -from aiohttp.web import RouteTableDef -from common_library.json_serialization import json_dumps -from models_library.api_schemas_storage import FileMetaDataGet -from models_library.utils.fastapi_encoders import jsonable_encoder -from servicelib.aiohttp.requests_validation import ( - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) - -# Exclusive for simcore-s3 storage ----------------------- -from ._meta import API_VTAG -from .dsm import get_dsm_provider -from .models import ( - FileMetaData, - FilesMetadataDatasetPathParams, - FilesMetadataDatasetQueryParams, - LocationPathParams, - StorageQueryParamsBase, -) - -log = logging.getLogger(__name__) - -routes = RouteTableDef() - -UPLOAD_TASKS_KEY = f"{__name__}.upload_tasks" - - -@routes.get( - f"/{API_VTAG}/locations/{{location_id}}/datasets", name="get_datasets_metadata" -) -async def get_datasets_metadata(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(LocationPathParams, request) - log.debug( - "received call to get_datasets_metadata with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - return web.json_response( - {"data": await dsm.list_datasets(query_params.user_id)}, dumps=json_dumps - ) - - -@routes.get( - f"/{API_VTAG}/locations/{{location_id}}/datasets/{{dataset_id}}/metadata", - name="get_files_metadata_dataset", -) -async def get_files_metadata_dataset(request: web.Request) -> web.Response: - query_params: FilesMetadataDatasetQueryParams = parse_request_query_parameters_as( - FilesMetadataDatasetQueryParams, request - ) - path_params = parse_request_path_parameters_as( - FilesMetadataDatasetPathParams, request - ) - log.debug( - "received call to get_files_metadata_dataset with %s", - f"{path_params=}, {query_params=}", - ) - dsm = get_dsm_provider(request.app).get(path_params.location_id) - data: list[FileMetaData] = await dsm.list_files_in_dataset( - user_id=query_params.user_id, - dataset_id=path_params.dataset_id, - expand_dirs=query_params.expand_dirs, - ) - return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, - dumps=json_dumps, - ) diff --git a/services/storage/src/simcore_service_storage/handlers_files.py b/services/storage/src/simcore_service_storage/handlers_files.py deleted file mode 100644 index 0f78fdaeea5..00000000000 --- a/services/storage/src/simcore_service_storage/handlers_files.py +++ /dev/null @@ -1,414 +0,0 @@ -import asyncio -import logging -import urllib.parse -from typing import cast - -from aiohttp import web -from aiohttp.web import RouteTableDef -from common_library.json_serialization import json_dumps -from models_library.api_schemas_storage import ( - FileMetaDataGet, - FileUploadCompleteFutureResponse, - FileUploadCompleteLinks, - FileUploadCompleteResponse, - FileUploadCompleteState, - FileUploadCompletionBody, - FileUploadLinks, - FileUploadSchema, - SoftCopyBody, -) -from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyUrl, ByteSize, TypeAdapter -from servicelib.aiohttp import status -from servicelib.aiohttp.requests_validation import ( - parse_request_body_as, - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) - -from ._meta import API_VTAG -from .dsm import get_dsm_provider -from .exceptions import FileMetaDataNotFoundError -from .models import ( - CopyAsSoftLinkParams, - FileDownloadQueryParams, - FileMetaData, - FilePathIsUploadCompletedParams, - FilePathParams, - FilesMetadataQueryParams, - FileUploadQueryParams, - LocationPathParams, - StorageQueryParamsBase, - UploadLinks, -) -from .simcore_s3_dsm import SimcoreS3DataManager -from .utils import create_upload_completion_task_name - -log = logging.getLogger(__name__) - -routes = RouteTableDef() - -UPLOAD_TASKS_KEY = f"{__name__}.upload_tasks" - - -@routes.get( - f"/{API_VTAG}/locations/{{location_id}}/files/metadata", name="get_files_metadata" -) -async def get_files_metadata(request: web.Request) -> web.Response: - query_params: FilesMetadataQueryParams = parse_request_query_parameters_as( - FilesMetadataQueryParams, request - ) - path_params = parse_request_path_parameters_as(LocationPathParams, request) - log.debug( - "received call to get_files_metadata with %s", - f"{path_params=}, {query_params=}", - ) - dsm = get_dsm_provider(request.app).get(path_params.location_id) - data: list[FileMetaData] = await dsm.list_files( - user_id=query_params.user_id, - expand_dirs=query_params.expand_dirs, - uuid_filter=query_params.uuid_filter - or f"{query_params.project_id or ''}", # NOTE: https://github.com/ITISFoundation/osparc-issues/issues/1593 - project_id=query_params.project_id, - ) - return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, - dumps=json_dumps, - ) - - -@routes.get( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}/metadata", - name="get_file_metadata", -) -async def get_file_metadata(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - log.debug( - "received call to get_file_metadata_dataset with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - try: - data = await dsm.get_file( - user_id=query_params.user_id, - file_id=path_params.file_id, - ) - except FileMetaDataNotFoundError: - # NOTE: LEGACY compatibility - # This is what happens Larry... data must be an empty {} or else some old dynamic services will FAIL (sic) - # Cannot remove until we retire all legacy services - # https://github.com/ITISFoundation/osparc-simcore/issues/5676 - # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_enveloped.py#L34 - - return web.json_response( - {"error": "No result found", "data": {}}, dumps=json_dumps - ) - - if request.headers.get("User-Agent") == "OpenAPI-Generator/0.1.0/python": - # NOTE: LEGACY compatiblity with API v0.1.0 - # SEE models used in sdk in: - # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_enveloped.py#L34 - # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/services/storage/client-sdk/python/simcore_service_storage_sdk/models/file_meta_data_type.py#L34 - return web.json_response( - { - "data": { - "file_uuid": data.file_uuid, - "location_id": data.location_id, - "location": data.location, - "bucket_name": data.bucket_name, - "object_name": data.object_name, - "project_id": data.project_id, - "project_name": data.project_name, - "node_id": data.node_id, - "node_name": data.node_name, - "file_name": data.file_name, - "user_id": data.user_id, - "user_name": None, - }, - "error": None, - }, - dumps=json_dumps, - ) - - return web.json_response( - {"data": jsonable_encoder(FileMetaDataGet(**data.model_dump()))}, - dumps=json_dumps, - ) - - -@routes.get( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}", name="download_file" -) -async def download_file(request: web.Request) -> web.Response: - query_params: FileDownloadQueryParams = parse_request_query_parameters_as( - FileDownloadQueryParams, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - log.debug( - "received call to download_file with %s", - f"{path_params=}, {query_params=}", - ) - dsm = get_dsm_provider(request.app).get(path_params.location_id) - link = await dsm.create_file_download_link( - query_params.user_id, path_params.file_id, query_params.link_type - ) - return web.json_response({"data": {"link": link}}, dumps=json_dumps) - - -@routes.put( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}", name="upload_file" -) -async def upload_file(request: web.Request) -> web.Response: - """creates upload file links: - - This function covers v1 and v2 versions of the handler. - Note: calling this entrypoint on an already existing file will overwrite that file. That file will be deleted - before the upload takes place. - - v1 rationale: - - client calls this handler, which returns a single link (either direct S3 or presigned) to the S3 backend - - client uploads the file - - storage relies on lazy update to find if the file is finished uploaded (when client calls get_file_meta_data, or if the dsm_cleaner goes over it after the upload time is expired) - - v2 rationale: - - client calls this handler, which returns a FileUploadSchema object containing 1 or more links (either S3/presigned links) - - client uploads the file (by chunking it if there are more than 1 presigned link) - - client calls complete_upload handle which will reconstruct the file on S3 backend - - client waits for completion to finish and then the file is accessible on S3 backend - - - Use-case v1: query.file_size is not defined, returns a PresignedLink model (backward compatibility) - Use-case v1.1: if query.link_type=presigned or None, returns a presigned link (limited to a single 5GB file) - Use-case v1.2: if query.link_type=s3, returns a s3 direct link (limited to a single 5TB file) - - User-case v2: query.is_directory is True (query.file_size is forced to -1), returns an s3 path where to upload all the content of the directory - User-case v2: if query.file_size is defined, returns a FileUploadSchema model, expects client to call "complete_upload" when the file is finished uploading - Use-case v2.1: if query.file_size == 0 and query.link_type=presigned or None, returns a single presigned link inside FileUploadSchema (limited to a single 5Gb file) - Use-case v2.2: if query.file_size > 0 and query.link_type=presigned or None, returns 1 or more presigned links depending on the file size (limited to a single 5TB file) - Use-case v2.3: if query.link_type=s3 and query.file_size>=0, returns a single s3 direct link (limited to a single 5TB file) - """ - query_params: FileUploadQueryParams = parse_request_query_parameters_as( - FileUploadQueryParams, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - log.debug( - "received call to upload_file with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - links: UploadLinks = await dsm.create_file_upload_links( - user_id=query_params.user_id, - file_id=path_params.file_id, - link_type=query_params.link_type, - file_size_bytes=query_params.file_size or ByteSize(0), - is_directory=query_params.is_directory, - sha256_checksum=query_params.sha256_checksum, - ) - if query_params.is_v1_upload: - # return v1 response - assert len(links.urls) == 1 # nosec - response = { - "data": {"link": jsonable_encoder(f"{links.urls[0]}", by_alias=True)} - } - log.debug("Returning v1 response: %s", response) - return web.json_response(response, dumps=json_dumps) - - # v2 response - abort_url = request.url.join( - request.app.router["abort_upload_file"] - .url_for( - location_id=f"{path_params.location_id}", - file_id=urllib.parse.quote(path_params.file_id, safe=""), - ) - .with_query(user_id=query_params.user_id) - ) - complete_url = request.url.join( - request.app.router["complete_upload_file"] - .url_for( - location_id=f"{path_params.location_id}", - file_id=urllib.parse.quote(path_params.file_id, safe=""), - ) - .with_query(user_id=query_params.user_id) - ) - v2_response = FileUploadSchema( - chunk_size=links.chunk_size, - urls=links.urls, - links=FileUploadLinks( - abort_upload=TypeAdapter(AnyUrl).validate_python(f"{abort_url}"), - complete_upload=TypeAdapter(AnyUrl).validate_python(f"{complete_url}"), - ), - ) - log.debug("returning v2 response: %s", v2_response) - return jsonable_encoder(v2_response, by_alias=True) # type: ignore[no-any-return] # middleware takes care of enveloping - - -@routes.post( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}:abort", - name="abort_upload_file", -) -async def abort_upload_file(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - log.debug( - "received call to abort_upload_file with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - await dsm.abort_file_upload(query_params.user_id, path_params.file_id) - return web.json_response(status=status.HTTP_204_NO_CONTENT) - - -@routes.post( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}:complete", - name="complete_upload_file", -) -async def complete_upload_file(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - body = await parse_request_body_as(FileUploadCompletionBody, request) - log.debug( - "received call to complete_upload_file with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - # NOTE: completing a multipart upload on AWS can take up to several minutes - # therefore we wait a bit to see if it completes fast and return a 204 - # if it returns slow we return a 202 - Accepted, the client will have to check later - # for completeness - task = asyncio.create_task( - dsm.complete_file_upload(path_params.file_id, query_params.user_id, body.parts), - name=create_upload_completion_task_name( - query_params.user_id, path_params.file_id - ), - ) - request.app[UPLOAD_TASKS_KEY][task.get_name()] = task - assert request.transport # nosec - ip_addr, port = request.transport.get_extra_info( - "sockname" - ) # https://docs.python.org/3/library/asyncio-protocol.html#asyncio.BaseTransport.get_extra_info - route = ( - request.app.router["is_completed_upload_file"] - .url_for( - location_id=f"{path_params.location_id}", - file_id=urllib.parse.quote(path_params.file_id, safe=""), - future_id=task.get_name(), - ) - .with_query(user_id=query_params.user_id) - ) - complete_task_state_url = f"{request.url.scheme}://{ip_addr}:{port}{route}" - response = FileUploadCompleteResponse( - links=FileUploadCompleteLinks( - state=TypeAdapter(AnyUrl).validate_python(complete_task_state_url) - ) - ) - return web.json_response( - status=status.HTTP_202_ACCEPTED, - data={"data": jsonable_encoder(response, by_alias=True)}, - dumps=json_dumps, - ) - - -@routes.post( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}:complete/futures/{{future_id}}", - name="is_completed_upload_file", -) -async def is_completed_upload_file(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as( - FilePathIsUploadCompletedParams, request - ) - log.debug( - "received call to is completed upload file with %s", - f"{path_params=}, {query_params=}", - ) - - # NOTE: completing a multipart upload on AWS can take up to several minutes - # therefore we wait a bit to see if it completes fast and return a 204 - # if it returns slow we return a 202 - Accepted, the client will have to check later - # for completeness - task_name = create_upload_completion_task_name( - query_params.user_id, path_params.file_id - ) - assert task_name == path_params.future_id # nosec - # first check if the task is in the app - if task := request.app[UPLOAD_TASKS_KEY].get(task_name): - if task.done(): - new_fmd: FileMetaData = task.result() - request.app[UPLOAD_TASKS_KEY].pop(task_name) - response = FileUploadCompleteFutureResponse( - state=FileUploadCompleteState.OK, e_tag=new_fmd.entity_tag - ) - else: - # the task is still running - response = FileUploadCompleteFutureResponse( - state=FileUploadCompleteState.NOK - ) - return jsonable_encoder(response, by_alias=True) # type: ignore[no-any-return] # middleware takes care of enveloping - # there is no task, either wrong call or storage was restarted - # we try to get the file to see if it exists in S3 - dsm = get_dsm_provider(request.app).get(path_params.location_id) - if fmd := await dsm.get_file( - user_id=query_params.user_id, - file_id=path_params.file_id, - ): - response = FileUploadCompleteFutureResponse( - state=FileUploadCompleteState.OK, e_tag=fmd.entity_tag - ) - return jsonable_encoder(response, by_alias=True) # type: ignore[no-any-return] # middleware takes care of enveloping - raise web.HTTPNotFound( - reason="Not found. Upload could not be completed. Please try again and contact support if it fails again." - ) - - -@routes.delete( - f"/{API_VTAG}/locations/{{location_id}}/files/{{file_id}}", name="delete_file" -) -async def delete_file(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(FilePathParams, request) - log.debug( - "received call to delete_file with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = get_dsm_provider(request.app).get(path_params.location_id) - await dsm.delete_file(query_params.user_id, path_params.file_id) - return web.json_response(status=status.HTTP_204_NO_CONTENT) - - -@routes.post(f"/{API_VTAG}/files/{{file_id}}:soft-copy", name="copy_as_soft_link") -async def copy_as_soft_link(request: web.Request): - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - path_params = parse_request_path_parameters_as(CopyAsSoftLinkParams, request) - body = await parse_request_body_as(SoftCopyBody, request) - log.debug( - "received call to copy_as_soft_link with %s", - f"{path_params=}, {query_params=}, {body=}", - ) - - dsm = cast( - SimcoreS3DataManager, - get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), - ) - file_link: FileMetaData = await dsm.create_soft_link( - query_params.user_id, path_params.file_id, body.link_id - ) - - return jsonable_encoder(FileMetaDataGet(**file_link.model_dump())) diff --git a/services/storage/src/simcore_service_storage/handlers_health.py b/services/storage/src/simcore_service_storage/handlers_health.py deleted file mode 100644 index eb97bafe1ba..00000000000 --- a/services/storage/src/simcore_service_storage/handlers_health.py +++ /dev/null @@ -1,87 +0,0 @@ -""" - - - Checks connectivity with other services in the backend - -""" - -import logging - -from aiohttp import web -from aws_library.s3 import S3AccessError -from common_library.json_serialization import json_dumps -from models_library.api_schemas_storage import HealthCheck, S3BucketName -from models_library.app_diagnostics import AppStatusCheck -from pydantic import TypeAdapter -from servicelib.rest_constants import RESPONSE_MODEL_POLICY - -from ._meta import API_VERSION, API_VTAG, PROJECT_NAME, VERSION -from .constants import APP_CONFIG_KEY -from .db import get_engine_state -from .db import is_service_responsive as is_pg_responsive -from .s3 import get_s3_client -from .settings import Settings - -log = logging.getLogger(__name__) - -routes = web.RouteTableDef() - - -@routes.get(f"/{API_VTAG}/", name="health_check") -async def get_health(request: web.Request) -> web.Response: - assert request # nosec - return web.json_response( - { - "data": HealthCheck( - name=PROJECT_NAME, - version=f"{VERSION}", - api_version=API_VERSION, - status=None, - ).model_dump(**RESPONSE_MODEL_POLICY) - }, - dumps=json_dumps, - ) - - -@routes.get(f"/{API_VTAG}/status", name="get_status") -async def get_status(request: web.Request) -> web.Response: - # NOTE: all calls here must NOT raise - assert request.app # nosec - app_settings: Settings = request.app[APP_CONFIG_KEY] - s3_state = "disabled" - if app_settings.STORAGE_S3: - try: - s3_state = ( - "connected" - if await get_s3_client(request.app).bucket_exists( - bucket=TypeAdapter(S3BucketName).validate_python( - app_settings.STORAGE_S3.S3_BUCKET_NAME - ) - ) - else "no access to S3 bucket" - ) - except S3AccessError: - s3_state = "failed" - - postgres_state = "disabled" - if app_settings.STORAGE_POSTGRES: - postgres_state = ( - "connected" if await is_pg_responsive(request.app) else "failed" - ) - - status = AppStatusCheck.model_validate( - { - "app_name": PROJECT_NAME, - "version": f"{VERSION}", - "services": { - "postgres": { - "healthy": postgres_state, - "pool": get_engine_state(request.app), - }, - "s3": {"healthy": s3_state}, - }, - } - ) - - return web.json_response( - {"data": status.model_dump(exclude_unset=True)}, dumps=json_dumps - ) diff --git a/services/storage/src/simcore_service_storage/handlers_locations.py b/services/storage/src/simcore_service_storage/handlers_locations.py deleted file mode 100644 index 494c1821f15..00000000000 --- a/services/storage/src/simcore_service_storage/handlers_locations.py +++ /dev/null @@ -1,105 +0,0 @@ -import asyncio -import logging -from typing import cast - -from aiohttp import web -from aiohttp.web import RouteTableDef -from common_library.json_serialization import json_dumps -from models_library.api_schemas_storage import FileLocation -from models_library.projects_nodes_io import StorageFileID -from servicelib.aiohttp.application_keys import ( - APP_CONFIG_KEY, - APP_FIRE_AND_FORGET_TASKS_KEY, -) -from servicelib.aiohttp.requests_validation import ( - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) -from servicelib.utils import fire_and_forget_task - -# Exclusive for simcore-s3 storage ----------------------- -from ._meta import API_VTAG -from .dsm import get_dsm_provider -from .models import LocationPathParams, StorageQueryParamsBase, SyncMetadataQueryParams -from .settings import Settings -from .simcore_s3_dsm import SimcoreS3DataManager - -log = logging.getLogger(__name__) - -routes = RouteTableDef() - - -# HANDLERS --------------------------------------------------- -@routes.get(f"/{API_VTAG}/locations", name="get_storage_locations") -async def get_storage_locations(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - log.debug( - "received call to get_storage_locations with %s", - f"{query_params=}", - ) - dsm_provider = get_dsm_provider(request.app) - location_ids = dsm_provider.locations() - locs: list[FileLocation] = [] - for loc_id in location_ids: - dsm = dsm_provider.get(loc_id) - if await dsm.authorized(query_params.user_id): - locs.append(FileLocation(name=dsm.location_name, id=dsm.location_id)) - - return web.json_response({"error": None, "data": locs}, dumps=json_dumps) - - -@routes.post( - f"/{API_VTAG}/locations/{{location_id}}:sync", name="synchronise_meta_data_table" -) -async def synchronise_meta_data_table(request: web.Request) -> web.Response: - query_params: SyncMetadataQueryParams = parse_request_query_parameters_as( - SyncMetadataQueryParams, request - ) - path_params = parse_request_path_parameters_as(LocationPathParams, request) - log.debug( - "received call to synchronise_meta_data_table with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = cast( - SimcoreS3DataManager, - get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), - ) - sync_results: list[StorageFileID] = [] - sync_coro = dsm.synchronise_meta_data_table(dry_run=query_params.dry_run) - - if query_params.fire_and_forget: - settings: Settings = request.app[APP_CONFIG_KEY] - - async def _go(): - timeout = settings.STORAGE_SYNC_METADATA_TIMEOUT - try: - result = await asyncio.wait_for(sync_coro, timeout=timeout) - log.info( - "Sync metadata table completed: %d entries removed", - len(result), - ) - except asyncio.TimeoutError: - log.exception("Sync metadata table timed out (%s seconds)", timeout) - - fire_and_forget_task( - _go(), - task_suffix_name="synchronise_meta_data_table", - fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], - ) - else: - sync_results = await sync_coro - - return web.json_response( - { - "error": None, - "data": { - "removed": sync_results, - "fire_and_forget": query_params.fire_and_forget, - "dry_run": query_params.dry_run, - }, - }, - dumps=json_dumps, - ) diff --git a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py deleted file mode 100644 index bfbe41e1806..00000000000 --- a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py +++ /dev/null @@ -1,166 +0,0 @@ -import logging -from typing import cast - -from aiohttp import web -from aiohttp.web import RouteTableDef -from common_library.json_serialization import json_dumps -from models_library.api_schemas_storage import FileMetaDataGet, FoldersBody -from models_library.projects import ProjectID -from models_library.utils.fastapi_encoders import jsonable_encoder -from servicelib.aiohttp import status -from servicelib.aiohttp.long_running_tasks.server import ( - TaskProgress, - start_long_running_task, -) -from servicelib.aiohttp.requests_validation import ( - parse_request_body_as, - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) -from servicelib.logging_utils import log_context -from settings_library.s3 import S3Settings - -from . import sts -from ._meta import API_VTAG -from .dsm import get_dsm_provider -from .models import ( - DeleteFolderQueryParams, - FileMetaData, - SearchFilesQueryParams, - SimcoreS3FoldersParams, - StorageQueryParamsBase, -) -from .simcore_s3_dsm import SimcoreS3DataManager - -_logger = logging.getLogger(__name__) - -routes = RouteTableDef() - - -@routes.post(f"/{API_VTAG}/simcore-s3:access", name="get_or_create_temporary_s3_access") -async def get_or_create_temporary_s3_access(request: web.Request) -> web.Response: - # NOTE: the name of the method is not accurate, these are not temporary at all - # it returns the credentials of the s3 backend! - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - _logger.debug( - "received call to get_or_create_temporary_s3_access with %s", - f"{query_params=}", - ) - - s3_settings: S3Settings = await sts.get_or_create_temporary_token_for_user( - request.app, query_params.user_id - ) - return web.json_response({"data": s3_settings.model_dump()}, dumps=json_dumps) - - -async def _copy_folders_from_project( - task_progress: TaskProgress, - app: web.Application, - query_params: StorageQueryParamsBase, - body: FoldersBody, -) -> web.Response: - dsm = cast( - SimcoreS3DataManager, - get_dsm_provider(app).get(SimcoreS3DataManager.get_location_id()), - ) - with log_context( - _logger, - logging.INFO, - msg=f"copying {body.source['uuid']} -> {body.destination['uuid']}", - ): - await dsm.deep_copy_project_simcore_s3( - query_params.user_id, - body.source, - body.destination, - body.nodes_map, - task_progress=task_progress, - ) - - return web.json_response( - {"data": jsonable_encoder(body.destination)}, - status=status.HTTP_201_CREATED, - dumps=json_dumps, - ) - - -@routes.post(f"/{API_VTAG}/simcore-s3/folders", name="copy_folders_from_project") -async def copy_folders_from_project(request: web.Request) -> web.Response: - query_params: StorageQueryParamsBase = parse_request_query_parameters_as( - StorageQueryParamsBase, request - ) - body = await parse_request_body_as(FoldersBody, request) - _logger.debug( - "received call to create_folders_from_project with %s", - f"{body=}, {query_params=}", - ) - return await start_long_running_task( - request, - _copy_folders_from_project, # type: ignore[arg-type] - task_context={}, - app=request.app, - query_params=query_params, - body=body, - ) - - -@routes.delete( - f"/{API_VTAG}/simcore-s3/folders/{{folder_id}}", name="delete_folders_of_project" -) -async def delete_folders_of_project(request: web.Request) -> web.Response: - query_params: DeleteFolderQueryParams = parse_request_query_parameters_as( - DeleteFolderQueryParams, request - ) - path_params = parse_request_path_parameters_as(SimcoreS3FoldersParams, request) - _logger.debug( - "received call to delete_folders_of_project with %s", - f"{path_params=}, {query_params=}", - ) - - dsm = cast( - SimcoreS3DataManager, - get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), - ) - await dsm.delete_project_simcore_s3( - query_params.user_id, - ProjectID(path_params.folder_id), - query_params.node_id, - ) - - return web.json_response(status=status.HTTP_204_NO_CONTENT) - - -@routes.post(f"/{API_VTAG}/simcore-s3/files/metadata:search", name="search_files") -async def search_files(request: web.Request) -> web.Response: - query_params: SearchFilesQueryParams = parse_request_query_parameters_as( - SearchFilesQueryParams, request - ) - - _logger.debug( - "received call to search_files with %s", - f"{query_params=}", - ) - - dsm = cast( - SimcoreS3DataManager, - get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()), - ) - - data: list[FileMetaData] = await dsm.search_owned_files( - user_id=query_params.user_id, - file_id_prefix=query_params.startswith, - sha256_checksum=query_params.sha256_checksum, - limit=query_params.limit, - offset=query_params.offset, - ) - _logger.debug( - "Found %d files starting with '%s'", - len(data), - f"{query_params.startswith=}, {query_params.sha256_checksum=}", - ) - - return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, - dumps=json_dumps, - ) diff --git a/services/storage/src/simcore_service_storage/long_running_tasks.py b/services/storage/src/simcore_service_storage/long_running_tasks.py deleted file mode 100644 index 6f0f43c9a8a..00000000000 --- a/services/storage/src/simcore_service_storage/long_running_tasks.py +++ /dev/null @@ -1,11 +0,0 @@ -from aiohttp import web -from servicelib.aiohttp.long_running_tasks.server import setup - -from ._meta import API_VTAG - - -def setup_rest_api_long_running_tasks(app: web.Application) -> None: - setup( - app, - router_prefix=f"/{API_VTAG}/futures", - ) diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py new file mode 100644 index 00000000000..abf94338662 --- /dev/null +++ b/services/storage/src/simcore_service_storage/main.py @@ -0,0 +1,22 @@ +"""Main application to be deployed in for example uvicorn.""" + +import logging + +from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.core.application import create_app +from simcore_service_storage.core.settings import ApplicationSettings + +_settings = ApplicationSettings.create_from_envs() + +# SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 +logging.basicConfig(level=_settings.log_level) # NOSONAR +logging.root.setLevel(_settings.log_level) +config_all_loggers( + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=_settings.STORAGE_TRACING, +) + +_logger = logging.getLogger(__name__) + +app = create_app(_settings) diff --git a/services/storage/src/simcore_service_storage/models.py b/services/storage/src/simcore_service_storage/models.py index 672694b4fc7..1e4166c89c9 100644 --- a/services/storage/src/simcore_service_storage/models.py +++ b/services/storage/src/simcore_service_storage/models.py @@ -1,18 +1,21 @@ import datetime import urllib.parse from dataclasses import dataclass -from typing import Any, Literal, NamedTuple +from pathlib import Path +from typing import Annotated, Any, Literal, NamedTuple, TypeAlias from uuid import UUID import arrow from aws_library.s3 import UploadID -from models_library.api_schemas_storage import ( +from aws_library.s3._models import S3DirectoryMetaData, S3MetaData +from models_library.api_schemas_storage.storage_schemas import ( UNDEFINED_SIZE, UNDEFINED_SIZE_TYPE, DatasetMetaDataGet, ETag, FileMetaDataGet, LinkType, + PathMetaDataGet, S3BucketName, ) from models_library.basic_types import SHA256Str @@ -36,6 +39,8 @@ ByteSize, ConfigDict, Field, + NonNegativeInt, + PlainSerializer, TypeAdapter, field_validator, model_validator, @@ -56,17 +61,31 @@ def is_uuid(value: str) -> bool: class FileMetaDataAtDB(BaseModel): - location_id: LocationID + location_id: Annotated[ + LocationID, PlainSerializer(lambda x: f"{x}", return_type=str) + ] location: LocationName bucket_name: S3BucketName object_name: SimcoreS3FileID - project_id: ProjectID | None = None - node_id: NodeID | None = None - user_id: UserID - created_at: datetime.datetime + project_id: Annotated[ + ProjectID | None, + PlainSerializer( + lambda x: f"{x}" if x is not None else None, return_type=str | None + ), + ] = None + node_id: Annotated[ + NodeID | None, + PlainSerializer( + lambda x: f"{x}" if x is not None else None, return_type=str | None + ), + ] = None + user_id: Annotated[UserID, PlainSerializer(lambda x: f"{x}", return_type=str)] + created_at: Annotated[datetime.datetime, PlainSerializer(lambda x: x.isoformat())] file_id: SimcoreS3FileID file_size: UNDEFINED_SIZE_TYPE | ByteSize - last_modified: datetime.datetime + last_modified: Annotated[ + datetime.datetime, PlainSerializer(lambda x: x.isoformat()) + ] entity_tag: ETag | None = None is_soft_link: bool upload_id: UploadID | None = None @@ -89,6 +108,16 @@ class FileMetaData(FileMetaDataGet): user_id: UserID | None sha256_checksum: SHA256Str | None + def update_display_fields(self, id_name_mapping: dict[str, str]) -> None: + if self.project_id: + # NOTE: this is disabled because the project_name is defined in FileMetaDataGet + # pylint: disable=attribute-defined-outside-init + self.project_name = id_name_mapping.get(f"{self.project_id}") + if self.node_id: + # NOTE: this is disabled because the node_name is defined in FileMetaDataGet + # pylint: disable=attribute-defined-outside-init + self.node_name = id_name_mapping.get(f"{self.node_id}") + @classmethod @validate_call def from_simcore_node( @@ -135,6 +164,30 @@ def from_simcore_node( fmd_kwargs.update(**file_meta_data_kwargs) return cls.model_validate(fmd_kwargs) + @classmethod + def from_db_model(cls, x: FileMetaDataAtDB) -> "FileMetaData": + return cls.model_validate( + x.model_dump() + | {"file_uuid": x.file_id, "file_name": x.file_id.split("/")[-1]} + ) + + @classmethod + def from_s3_object_in_dir( + cls, x: S3MetaData, dir_fmd: "FileMetaData" + ) -> "FileMetaData": + return dir_fmd.model_copy( + update={ + "object_name": x.object_key, + "file_id": x.object_key, + "file_size": x.size, + "entity_tag": x.e_tag, + "sha256_checksum": x.sha256_checksum, + "is_directory": False, + "created_at": x.last_modified, + "last_modified": x.last_modified, + } + ) + @dataclass class UploadLinks: @@ -144,14 +197,18 @@ class UploadLinks: class StorageQueryParamsBase(BaseModel): user_id: UserID - model_config = ConfigDict(populate_by_name=True, extra="forbid") + model_config = ConfigDict(populate_by_name=True) + + +class ListPathsQueryParams(StorageQueryParamsBase): + file_filter: Path | None = None class FilesMetadataDatasetQueryParams(StorageQueryParamsBase): expand_dirs: bool = True -class FilesMetadataQueryParams(StorageQueryParamsBase): +class FileMetadataListQueryParams(StorageQueryParamsBase): project_id: ProjectID | None = None uuid_filter: str = "" expand_dirs: bool = True @@ -162,15 +219,18 @@ class SyncMetadataQueryParams(BaseModel): fire_and_forget: bool = False +class SyncMetadataResponse(BaseModel): + removed: list[StorageFileID] + fire_and_forget: bool + dry_run: bool + + class FileDownloadQueryParams(StorageQueryParamsBase): link_type: LinkType = LinkType.PRESIGNED - @field_validator("link_type", mode="before") - @classmethod - def convert_from_lower_case(cls, v: str) -> str: - if v is not None: - return f"{v}".upper() - return v + +class FileDownloadResponse(BaseModel): + link: AnyUrl class FileUploadQueryParams(StorageQueryParamsBase): @@ -179,13 +239,6 @@ class FileUploadQueryParams(StorageQueryParamsBase): is_directory: bool = False sha256_checksum: SHA256Str | None = None - @field_validator("link_type", mode="before") - @classmethod - def convert_from_lower_case(cls, v: str) -> str: - if v is not None: - return f"{v}".upper() - return v - @model_validator(mode="before") @classmethod def when_directory_force_link_type_and_file_size(cls, data: Any) -> Any: @@ -210,6 +263,10 @@ def is_v1_upload(self) -> bool: return self.file_size is None and self.is_directory is False +class FileUploadResponseV1(BaseModel): + link: AnyUrl + + class DeleteFolderQueryParams(StorageQueryParamsBase): node_id: NodeID | None = None @@ -275,13 +332,77 @@ class UserOrProjectFilter(NamedTuple): project_ids: list[ProjectID] -__all__ = ( - "ETag", - "FileMetaData", - "FileMetaDataAtDB", - "S3BucketName", - "SimcoreS3FileID", - "StorageFileID", - "UploadID", - "UploadLinks", -) +@dataclass(frozen=True) +class AccessRights: + read: bool + write: bool + delete: bool + + @classmethod + def all(cls) -> "AccessRights": + return cls(read=True, write=True, delete=True) + + @classmethod + def none(cls) -> "AccessRights": + return cls(read=False, write=False, delete=False) + + +TotalNumber: TypeAlias = NonNegativeInt +GenericCursor: TypeAlias = str | bytes + + +class PathMetaData(BaseModel): + path: Path + display_path: Annotated[ + Path, + Field( + description="Path with names instead of IDs (URL Encoded by parts as names may contain '/')" + ), + ] + location_id: LocationID + location: LocationName + bucket_name: str + + project_id: ProjectID | None + node_id: NodeID | None + user_id: UserID | None + created_at: datetime.datetime + last_modified: datetime.datetime + + file_meta_data: FileMetaData | None + + def update_display_fields(self, id_name_mapping: dict[str, str]) -> None: + display_path = f"{self.path}" + for old, new in id_name_mapping.items(): + display_path = display_path.replace(old, urllib.parse.quote(new, safe="")) + self.display_path = Path(display_path) + + if self.file_meta_data: + self.file_meta_data.update_display_fields(id_name_mapping) + + @classmethod + def from_s3_object_in_dir( + cls, s3_object: S3MetaData | S3DirectoryMetaData, dir_fmd: FileMetaData + ) -> "PathMetaData": + return cls( + path=s3_object.as_path(), + display_path=s3_object.as_path(), + location_id=dir_fmd.location_id, + location=dir_fmd.location, + bucket_name=dir_fmd.bucket_name, + user_id=dir_fmd.user_id, + project_id=dir_fmd.project_id, + node_id=dir_fmd.node_id, + created_at=dir_fmd.created_at, + last_modified=dir_fmd.last_modified, + file_meta_data=None + if isinstance(s3_object, S3DirectoryMetaData) + else FileMetaData.from_s3_object_in_dir(s3_object, dir_fmd), + ) + + def to_api_model(self) -> PathMetaDataGet: + return PathMetaDataGet.model_construct( + path=self.path, + display_path=self.display_path, + file_meta_data=self.file_meta_data, + ) diff --git a/services/storage/src/simcore_service_storage/modules/__init__.py b/services/storage/src/simcore_service_storage/modules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/modules/celery/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/__init__.py new file mode 100644 index 00000000000..fc6ed86c7b5 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/__init__.py @@ -0,0 +1,40 @@ +import logging +from asyncio import AbstractEventLoop + +from fastapi import FastAPI + +from ...core.settings import get_application_settings +from ._celery_types import register_celery_types +from ._common import create_app +from .client import CeleryTaskQueueClient + +_logger = logging.getLogger(__name__) + + +def setup_celery_client(app: FastAPI) -> None: + async def on_startup() -> None: + celery_settings = get_application_settings(app).STORAGE_CELERY + assert celery_settings # nosec + celery_app = create_app(celery_settings) + app.state.celery_client = CeleryTaskQueueClient(celery_app) + + register_celery_types() + + app.add_event_handler("startup", on_startup) + + +def get_celery_client(app: FastAPI) -> CeleryTaskQueueClient: + assert hasattr(app.state, "celery_client") # nosec + celery_client = app.state.celery_client + assert isinstance(celery_client, CeleryTaskQueueClient) + return celery_client + + +def get_event_loop(app: FastAPI) -> AbstractEventLoop: + event_loop = app.state.event_loop + assert isinstance(event_loop, AbstractEventLoop) + return event_loop + + +def set_event_loop(app: FastAPI, event_loop: AbstractEventLoop) -> None: + app.state.event_loop = event_loop diff --git a/services/storage/src/simcore_service_storage/modules/celery/_celery_types.py b/services/storage/src/simcore_service_storage/modules/celery/_celery_types.py new file mode 100644 index 00000000000..7fb44e087d5 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/_celery_types.py @@ -0,0 +1,61 @@ +from functools import partial +from pathlib import Path +from typing import Any + +from kombu.utils.json import register_type # type: ignore[import-untyped] +from models_library.api_schemas_storage.storage_schemas import ( + FileUploadCompletionBody, + FoldersBody, +) +from pydantic import BaseModel + +from ...models import FileMetaData +from ...modules.celery.models import TaskError + + +def _path_encoder(obj): + if isinstance(obj, Path): + return {"__path__": True, "path": str(obj)} + return obj + + +# Define how Path objects are deserialized +def _path_decoder(obj): + if "__path__" in obj: + return Path(obj["path"]) + return obj + + +def _class_full_name(clz: type) -> str: + return ".".join([clz.__module__, clz.__qualname__]) + + +def _encoder(obj: BaseModel, *args, **kwargs) -> dict[str, Any]: + return obj.model_dump(*args, **kwargs, mode="json") + + +def _decoder(clz: type[BaseModel], data: dict[str, Any]) -> BaseModel: + return clz(**data) + + +def _register_pydantic_types(*models: type[BaseModel]) -> None: + for model in models: + register_type( + model, + _class_full_name(model), + encoder=_encoder, + decoder=partial(_decoder, model), + ) + + +def register_celery_types() -> None: + register_type( + Path, + _class_full_name(Path), + _path_encoder, + _path_decoder, + ) + _register_pydantic_types(FileUploadCompletionBody) + _register_pydantic_types(FileMetaData) + _register_pydantic_types(FoldersBody) + _register_pydantic_types(TaskError) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py new file mode 100644 index 00000000000..b479408053f --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -0,0 +1,31 @@ +import logging +import ssl + +from celery import Celery # type: ignore[import-untyped] +from settings_library.celery import CelerySettings +from settings_library.redis import RedisDatabase + +_logger = logging.getLogger(__name__) + + +def create_app(celery_settings: CelerySettings) -> Celery: + assert celery_settings + + app = Celery( + broker=celery_settings.CELERY_RABBIT_BROKER.dsn, + backend=celery_settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ), + ) + app.conf.broker_connection_retry_on_startup = True + # NOTE: disable SSL cert validation (https://github.com/ITISFoundation/osparc-simcore/pull/7407) + if celery_settings.CELERY_REDIS_RESULT_BACKEND.REDIS_SECURE: + app.conf.redis_backend_use_ssl = {"ssl_cert_reqs": ssl.CERT_NONE} + app.conf.result_expires = celery_settings.CELERY_RESULT_EXPIRES + app.conf.result_extended = True # original args are included in the results + app.conf.result_serializer = "json" + app.conf.task_send_sent_event = True + app.conf.task_track_started = True + app.conf.worker_send_task_events = True # enable tasks monitoring + + return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/_task.py b/services/storage/src/simcore_service_storage/modules/celery/_task.py new file mode 100644 index 00000000000..6e735a8be81 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/_task.py @@ -0,0 +1,117 @@ +import asyncio +import inspect +import logging +import traceback +from collections.abc import Callable, Coroutine +from functools import wraps +from typing import Any, Concatenate, ParamSpec, TypeVar, overload + +from celery import Celery # type: ignore[import-untyped] +from celery.contrib.abortable import AbortableTask # type: ignore[import-untyped] +from celery.exceptions import Ignore # type: ignore[import-untyped] + +from . import get_event_loop +from .models import TaskError, TaskId, TaskState +from .utils import get_fastapi_app + +_logger = logging.getLogger(__name__) + + +def error_handling(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(task: AbortableTask, *args: Any, **kwargs: Any) -> Any: + try: + return func(task, *args, **kwargs) + except Exception as exc: + exc_type = type(exc).__name__ + exc_message = f"{exc}" + exc_traceback = traceback.format_exc().split("\n") + + _logger.exception( + "Task %s failed with exception: %s:%s", + task.request.id, + exc_type, + exc_message, + ) + + task.update_state( + state=TaskState.ERROR.upper(), + meta=TaskError( + exc_type=exc_type, + exc_msg=exc_message, + ).model_dump(mode="json"), + traceback=exc_traceback, + ) + raise Ignore from exc # ignore doing state updates + + return wrapper + + +T = TypeVar("T") +P = ParamSpec("P") +R = TypeVar("R") + + +def _async_task_wrapper( + app: Celery, +) -> Callable[ + [Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]]], + Callable[Concatenate[AbortableTask, P], R], +]: + def decorator( + coro: Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]], + ) -> Callable[Concatenate[AbortableTask, P], R]: + @wraps(coro) + def wrapper(task: AbortableTask, *args: P.args, **kwargs: P.kwargs) -> R: + fastapi_app = get_fastapi_app(app) + _logger.debug("task id: %s", task.request.id) + # NOTE: task.request is a thread local object, so we need to pass the id explicitly + assert task.request.id is not None # nosec + return asyncio.run_coroutine_threadsafe( + coro(task, task.request.id, *args, **kwargs), + get_event_loop(fastapi_app), + ).result() + + return wrapper + + return decorator + + +@overload +def define_task( + app: Celery, + fn: Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]], + task_name: str | None = None, +) -> None: ... + + +@overload +def define_task( + app: Celery, + fn: Callable[Concatenate[AbortableTask, P], R], + task_name: str | None = None, +) -> None: ... + + +def define_task( # type: ignore[misc] + app: Celery, + fn: ( + Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]] + | Callable[Concatenate[AbortableTask, P], R] + ), + task_name: str | None = None, +) -> None: + """Decorator to define a celery task with error handling and abortable support""" + wrapped_fn: Callable[Concatenate[AbortableTask, P], R] + if asyncio.iscoroutinefunction(fn): + wrapped_fn = _async_task_wrapper(app)(fn) + else: + assert inspect.isfunction(fn) # nosec + wrapped_fn = fn + + wrapped_fn = error_handling(wrapped_fn) + app.task( + name=task_name or fn.__name__, + bind=True, + base=AbortableTask, + )(wrapped_fn) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py new file mode 100644 index 00000000000..d5b1d1b88af --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -0,0 +1,176 @@ +import contextlib +import logging +from dataclasses import dataclass +from typing import Any, Final +from uuid import uuid4 + +from celery import Celery # type: ignore[import-untyped] +from celery.contrib.abortable import ( # type: ignore[import-untyped] + AbortableAsyncResult, +) +from common_library.async_tools import make_async +from models_library.progress_bar import ProgressReport +from pydantic import ValidationError +from servicelib.logging_utils import log_context + +from ...exceptions.errors import ConfigurationError +from .models import TaskContext, TaskID, TaskState, TaskStatus, TaskUUID + +_logger = logging.getLogger(__name__) + +_CELERY_INSPECT_TASK_STATUSES: Final[tuple[str, ...]] = ( + "active", + "scheduled", + "revoked", +) +_CELERY_TASK_META_PREFIX: Final[str] = "celery-task-meta-" +_CELERY_STATES_MAPPING: Final[dict[str, TaskState]] = { + "PENDING": TaskState.PENDING, + "STARTED": TaskState.PENDING, + "RETRY": TaskState.PENDING, + "RUNNING": TaskState.RUNNING, + "SUCCESS": TaskState.SUCCESS, + "ABORTED": TaskState.ABORTED, + "FAILURE": TaskState.ERROR, + "ERROR": TaskState.ERROR, +} +_CELERY_TASK_ID_KEY_SEPARATOR: Final[str] = ":" +_CELERY_TASK_ID_KEY_ENCODING = "utf-8" + +_MIN_PROGRESS_VALUE = 0.0 +_MAX_PROGRESS_VALUE = 100.0 + + +def _build_context_prefix(task_context: TaskContext) -> list[str]: + return [f"{task_context[key]}" for key in sorted(task_context)] + + +def _build_task_id_prefix(task_context: TaskContext) -> str: + return _CELERY_TASK_ID_KEY_SEPARATOR.join(_build_context_prefix(task_context)) + + +def _build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: + return _CELERY_TASK_ID_KEY_SEPARATOR.join( + [_build_task_id_prefix(task_context), f"{task_uuid}"] + ) + + +@dataclass +class CeleryTaskQueueClient: + _celery_app: Celery + + @make_async() + def send_task( + self, task_name: str, *, task_context: TaskContext, **task_params + ) -> TaskUUID: + with log_context( + _logger, + logging.DEBUG, + msg=f"Submit {task_name=}: {task_context=} {task_params=}", + ): + task_uuid = uuid4() + task_id = _build_task_id(task_context, task_uuid) + self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) + return task_uuid + + @staticmethod + @make_async() + def abort_task(task_context: TaskContext, task_uuid: TaskUUID) -> None: + with log_context( + _logger, + logging.DEBUG, + msg=f"Abort task {task_uuid=}: {task_context=}", + ): + task_id = _build_task_id(task_context, task_uuid) + AbortableAsyncResult(task_id).abort() + + @make_async() + def get_task_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: + with log_context( + _logger, + logging.DEBUG, + msg=f"Get task {task_uuid=}: {task_context=} result", + ): + task_id = _build_task_id(task_context, task_uuid) + return self._celery_app.AsyncResult(task_id).result + + def _get_progress_report( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> ProgressReport: + task_id = _build_task_id(task_context, task_uuid) + result = self._celery_app.AsyncResult(task_id).result + state = self._get_state(task_context, task_uuid) + if result and state == TaskState.RUNNING: + with contextlib.suppress(ValidationError): + # avoids exception if result is not a ProgressReport (or overwritten by a Celery's state update) + return ProgressReport.model_validate(result) + if state in ( + TaskState.ABORTED, + TaskState.ERROR, + TaskState.SUCCESS, + ): + return ProgressReport(actual_value=_MAX_PROGRESS_VALUE) + return ProgressReport(actual_value=_MIN_PROGRESS_VALUE) + + def _get_state(self, task_context: TaskContext, task_uuid: TaskUUID) -> TaskState: + task_id = _build_task_id(task_context, task_uuid) + return _CELERY_STATES_MAPPING[self._celery_app.AsyncResult(task_id).state] + + @make_async() + def get_task_status( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> TaskStatus: + return TaskStatus( + task_uuid=task_uuid, + task_state=self._get_state(task_context, task_uuid), + progress_report=self._get_progress_report(task_context, task_uuid), + ) + + def _get_completed_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: + search_key = _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) + backend_client = self._celery_app.backend.client + if hasattr(backend_client, "keys") and ( + keys := backend_client.keys(f"{search_key}*") + ): + return { + TaskUUID( + f"{key.decode(_CELERY_TASK_ID_KEY_ENCODING).removeprefix(search_key + _CELERY_TASK_ID_KEY_SEPARATOR)}" + ) + for key in keys + } + if hasattr(backend_client, "cache"): + # NOTE: backend used in testing. It is a dict-like object + found_keys = set() + for key in backend_client.cache: + str_key = key.decode(_CELERY_TASK_ID_KEY_ENCODING) + if str_key.startswith(search_key): + found_keys.add( + TaskUUID( + f"{str_key.removeprefix(search_key + _CELERY_TASK_ID_KEY_SEPARATOR)}" + ) + ) + return found_keys + msg = f"Unsupported backend {self._celery_app.backend.__class__.__name__}" + raise ConfigurationError(msg=msg) + + @make_async() + def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: + task_uuids = self._get_completed_task_uuids(task_context) + + task_id_prefix = _build_task_id_prefix(task_context) + inspect = self._celery_app.control.inspect() + for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: + tasks = getattr(inspect, task_inspect_status)() or {} + + task_uuids.update( + TaskUUID( + task_info["id"].removeprefix( + task_id_prefix + _CELERY_TASK_ID_KEY_SEPARATOR + ) + ) + for tasks_per_worker in tasks.values() + for task_info in tasks_per_worker + if "id" in task_info + ) + + return task_uuids diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py new file mode 100644 index 00000000000..6b72a6e0019 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -0,0 +1,60 @@ +from enum import StrEnum, auto +from typing import Any, Final, Self, TypeAlias +from uuid import UUID + +from models_library.progress_bar import ProgressReport +from pydantic import BaseModel, model_validator + +TaskContext: TypeAlias = dict[str, Any] +TaskID: TypeAlias = str +TaskUUID: TypeAlias = UUID + +_MIN_PROGRESS: Final[float] = 0.0 +_MAX_PROGRESS: Final[float] = 100.0 + + +class TaskState(StrEnum): + PENDING = auto() + RUNNING = auto() + SUCCESS = auto() + ERROR = auto() + ABORTED = auto() + + +_TASK_DONE = {TaskState.SUCCESS, TaskState.ERROR, TaskState.ABORTED} + + +class TaskStatus(BaseModel): + task_uuid: TaskUUID + task_state: TaskState + progress_report: ProgressReport + + @property + def is_done(self) -> bool: + return self.task_state in _TASK_DONE + + @model_validator(mode="after") + def _check_consistency(self) -> Self: + value = self.progress_report.actual_value + + valid_states = { + TaskState.PENDING: value == _MIN_PROGRESS, + TaskState.RUNNING: _MIN_PROGRESS <= value <= _MAX_PROGRESS, + TaskState.SUCCESS: value == _MAX_PROGRESS, + TaskState.ABORTED: value == _MAX_PROGRESS, + TaskState.ERROR: value == _MAX_PROGRESS, + } + + if not valid_states.get(self.task_state, True): + msg = f"Inconsistent progress actual value for state={self.task_state}: {value}" + raise ValueError(msg) + + return self + + +class TaskError(BaseModel): + exc_type: str + exc_msg: str + + +TaskId: TypeAlias = str diff --git a/services/storage/src/simcore_service_storage/modules/celery/signals.py b/services/storage/src/simcore_service_storage/modules/celery/signals.py new file mode 100644 index 00000000000..7f3835286bc --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/signals.py @@ -0,0 +1,71 @@ +import asyncio +import logging +import threading +from typing import Final + +from asgi_lifespan import LifespanManager +from celery import Celery # type: ignore[import-untyped] +from fastapi import FastAPI +from servicelib.async_utils import cancel_wait_task + +from ...core.application import create_app +from ...core.settings import ApplicationSettings +from ...modules.celery import get_event_loop, set_event_loop +from ...modules.celery.utils import ( + get_fastapi_app, + set_celery_worker, + set_fastapi_app, +) +from ...modules.celery.worker import CeleryTaskQueueWorker + +_logger = logging.getLogger(__name__) + +_LIFESPAN_TIMEOUT: Final[int] = 10 + + +def on_worker_init(sender, **_kwargs) -> None: + def _init_fastapi() -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + shutdown_event = asyncio.Event() + + fastapi_app = create_app(ApplicationSettings.create_from_envs()) + + async def lifespan(): + async with LifespanManager( + fastapi_app, + startup_timeout=_LIFESPAN_TIMEOUT, + shutdown_timeout=_LIFESPAN_TIMEOUT, + ): + try: + await shutdown_event.wait() + except asyncio.CancelledError: + _logger.warning("Lifespan task cancelled") + + lifespan_task = loop.create_task(lifespan()) + fastapi_app.state.lifespan_task = lifespan_task + fastapi_app.state.shutdown_event = shutdown_event + set_event_loop(fastapi_app, loop) + + set_fastapi_app(sender.app, fastapi_app) + set_celery_worker(sender.app, CeleryTaskQueueWorker(sender.app)) + + loop.run_forever() + + thread = threading.Thread(target=_init_fastapi, daemon=True) + thread.start() + + +def on_worker_shutdown(sender, **_kwargs): + assert isinstance(sender.app, Celery) + + fastapi_app = get_fastapi_app(sender.app) + assert isinstance(fastapi_app, FastAPI) + event_loop = get_event_loop(fastapi_app) + + async def shutdown(): + fastapi_app.state.shutdown_event.set() + + await cancel_wait_task(fastapi_app.state.lifespan_task, max_delay=5) + + asyncio.run_coroutine_threadsafe(shutdown(), event_loop) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py new file mode 100644 index 00000000000..014151acd74 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -0,0 +1,29 @@ +import logging +import time + +from celery import Task # type: ignore[import-untyped] +from models_library.progress_bar import ProgressReport +from models_library.projects_nodes_io import StorageFileID +from servicelib.logging_utils import log_context + +from .utils import get_celery_worker + +_logger = logging.getLogger(__name__) + + +def export_data(task: Task, files: list[StorageFileID]): + _logger.info("Exporting files: %s", files) + for n, file in enumerate(files, start=1): + with log_context( + _logger, + logging.INFO, + msg=f"Exporting {file=} ({n}/{len(files)})", + ): + assert task.name + get_celery_worker(task.app).set_task_progress( + task_name=task.name, + task_id=task.request.id, + report=ProgressReport(actual_value=n / len(files) * 100), + ) + time.sleep(10) + return "done" diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py new file mode 100644 index 00000000000..88c346ea211 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -0,0 +1,27 @@ +from celery import Celery # type: ignore[import-untyped] +from fastapi import FastAPI + +from .worker import CeleryTaskQueueWorker + +_WORKER_KEY = "celery_worker" +_FASTAPI_APP_KEY = "fastapi_app" + + +def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: + worker = celery_app.conf[_WORKER_KEY] + assert isinstance(worker, CeleryTaskQueueWorker) + return worker + + +def get_fastapi_app(celery_app: Celery) -> FastAPI: + fastapi_app = celery_app.conf[_FASTAPI_APP_KEY] + assert isinstance(fastapi_app, FastAPI) + return fastapi_app + + +def set_celery_worker(celery_app: Celery, worker: CeleryTaskQueueWorker) -> None: + celery_app.conf[_WORKER_KEY] = worker + + +def set_fastapi_app(celery_app: Celery, fastapi_app: FastAPI) -> None: + celery_app.conf[_FASTAPI_APP_KEY] = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py new file mode 100644 index 00000000000..c0983734905 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -0,0 +1,28 @@ +import logging + +from celery import Celery # type: ignore[import-untyped] +from models_library.progress_bar import ProgressReport +from servicelib.logging_utils import log_context + +from .models import TaskID + +_logger = logging.getLogger(__name__) + + +class CeleryTaskQueueWorker: + def __init__(self, celery_app: Celery) -> None: + self.celery_app = celery_app + + def set_task_progress( + self, task_name: str, task_id: TaskID, report: ProgressReport + ) -> None: + with log_context( + _logger, + logging.DEBUG, + msg=f"Setting progress for {task_name}: {report.model_dump_json()}", + ): + self.celery_app.tasks[task_name].update_state( + task_id=task_id, + state="RUNNING", + meta=report.model_dump(mode="json"), + ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py new file mode 100644 index 00000000000..99b9a53676e --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -0,0 +1,34 @@ +"""Main application to be deployed in for example uvicorn.""" + +import logging + +from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] +from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.api._worker_tasks.tasks import setup_worker_tasks + +from ...core.settings import ApplicationSettings +from ._common import create_app as create_celery_app +from .signals import ( + on_worker_init, + on_worker_shutdown, +) + +_settings = ApplicationSettings.create_from_envs() + +logging.basicConfig(level=_settings.log_level) # NOSONAR +logging.root.setLevel(_settings.log_level) +config_all_loggers( + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=_settings.STORAGE_TRACING, +) + +_logger = logging.getLogger(__name__) + +assert _settings.STORAGE_CELERY +app = create_celery_app(_settings.STORAGE_CELERY) +worker_init.connect(on_worker_init) +worker_shutdown.connect(on_worker_shutdown) + + +setup_worker_tasks(app) diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/__init__.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py new file mode 100644 index 00000000000..0f5ec1dae12 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py @@ -0,0 +1,377 @@ +import logging +from typing import Any, TypeAlias, cast + +import httpx +from fastapi import FastAPI, status +from fastapi_pagination import Page +from models_library.api_schemas_datcore_adapter.datasets import ( + DatasetMetaData as DatCoreDatasetMetaData, +) +from models_library.api_schemas_datcore_adapter.datasets import ( + FileMetaData as DatCoreFileMetaData, +) +from models_library.api_schemas_datcore_adapter.datasets import ( + PackageMetaData, +) +from models_library.api_schemas_storage.storage_schemas import ( + DatCoreCollectionName, + DatCoreDatasetName, + DatCorePackageName, +) +from models_library.users import UserID +from pydantic import AnyUrl, BaseModel, ByteSize, NonNegativeInt, TypeAdapter +from servicelib.fastapi.client_session import get_client_session +from servicelib.utils import logged_gather + +from ...constants import DATCORE_ID, DATCORE_STR, MAX_CONCURRENT_REST_CALLS +from ...core.settings import get_application_settings +from ...models import ( + DatasetMetaData, + FileMetaData, + GenericCursor, + PathMetaData, + TotalNumber, +) +from .datcore_adapter_client_utils import request, retrieve_all_pages +from .datcore_adapter_exceptions import ( + DatcoreAdapterError, + DatcoreAdapterFileNotFoundError, + DatcoreAdapterResponseError, +) +from .utils import ( + create_path_meta_data_from_datcore_fmd, + create_path_meta_data_from_datcore_package, +) + +_logger = logging.getLogger(__file__) + + +async def check_service_health(app: FastAPI) -> bool: + datcore_adapter_settings = get_application_settings(app).DATCORE_ADAPTER + url = datcore_adapter_settings.endpoint + "/ready" + session = get_client_session(app) + try: + response = await session.get(url) + response.raise_for_status() + except (TimeoutError, httpx.HTTPStatusError): + return False + return True + + +async def check_user_can_connect(app: FastAPI, api_key: str, api_secret: str) -> bool: + if not api_key or not api_secret: + # no need to ask, datcore is an authenticated service + return False + + try: + await request(app, api_key, api_secret, "GET", "/user/profile") + return True + except DatcoreAdapterError: + return False + + +async def list_all_datasets_files_metadatas( + app: FastAPI, user_id: UserID, api_key: str, api_secret: str +) -> list[FileMetaData]: + all_datasets: list[DatasetMetaData] = await list_all_datasets( + app, api_key, api_secret + ) + results = await logged_gather( + *( + list_all_files_metadatas_in_dataset( + app, + user_id, + api_key, + api_secret, + cast(DatCoreDatasetName, d.dataset_id), + ) + for d in all_datasets + ), + log=_logger, + max_concurrency=MAX_CONCURRENT_REST_CALLS, + ) + all_files_of_all_datasets: list[FileMetaData] = [] + for data in results: + all_files_of_all_datasets += data + return all_files_of_all_datasets + + +_LIST_ALL_DATASETS_TIMEOUT_S = 60 + + +async def list_all_files_metadatas_in_dataset( + app: FastAPI, + user_id: UserID, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, +) -> list[FileMetaData]: + all_files: list[dict[str, Any]] = cast( + list[dict[str, Any]], + await request( + app, + api_key, + api_secret, + "GET", + f"/datasets/{dataset_id}/files_legacy", + timeout=_LIST_ALL_DATASETS_TIMEOUT_S, + ), + ) + return [ + FileMetaData.model_construct( + file_uuid=d["path"], + location_id=DATCORE_ID, + location=DATCORE_STR, + bucket_name=d["dataset_id"], + object_name=d["path"], + file_name=d["name"], + file_id=d["package_id"], + file_size=d["size"], + created_at=d["created_at"], + last_modified=d["last_modified_at"], + project_id=None, + node_id=None, + user_id=user_id, + is_soft_link=False, + ) + for d in all_files + ] + + +_Size: TypeAlias = NonNegativeInt +_Page: TypeAlias = NonNegativeInt + + +class CursorParameters(BaseModel): + next_page: _Page + size: _Size + + +def _init_pagination( + cursor: GenericCursor | None, limit: NonNegativeInt +) -> tuple[_Page, _Size]: + if cursor is not None: + cursor_params = CursorParameters.model_validate_json(cursor) + return cursor_params.next_page, cursor_params.size + return 1, limit + + +def _create_next_cursor( + total: TotalNumber, page: _Page, size: _Size +) -> GenericCursor | None: + if total > page * size: + return CursorParameters.model_validate( + {"next_page": page + 1, "size": size} + ).model_dump_json() + return None + + +async def _list_top_level_objects( + app: FastAPI, + *, + user_id: UserID, + api_key: str, + api_secret: str, + cursor: GenericCursor | None, + limit: NonNegativeInt, + request_path: str, +) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber]: + page, size = _init_pagination(cursor, limit) + response = await request( + app, + api_key, + api_secret, + "GET", + request_path, + params={"size": size, "page": page}, + ) + assert isinstance(response, dict) # nosec + file_metadata_page = Page[DatCoreFileMetaData](**response) + entries = file_metadata_page.items + total = file_metadata_page.total + assert isinstance(total, int) # nosec + next_cursor = _create_next_cursor(total, page, size) + + return ( + [create_path_meta_data_from_datcore_fmd(user_id, e) for e in entries], + next_cursor, + total, + ) + + +async def list_top_level_objects_in_dataset( + app: FastAPI, + *, + user_id: UserID, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, + cursor: GenericCursor | None, + limit: NonNegativeInt, +) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber]: + return await _list_top_level_objects( + app, + user_id=user_id, + api_key=api_key, + api_secret=api_secret, + cursor=cursor, + limit=limit, + request_path=f"/datasets/{dataset_id}/files", + ) + + +async def list_top_level_objects_in_collection( + app: FastAPI, + *, + user_id: UserID, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, + collection_id: DatCoreCollectionName, + cursor: GenericCursor | None, + limit: NonNegativeInt, +) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber]: + return await _list_top_level_objects( + app, + user_id=user_id, + api_key=api_key, + api_secret=api_secret, + cursor=cursor, + limit=limit, + request_path=f"/datasets/{dataset_id}/files/{collection_id}", + ) + + +async def get_package_file_as_path( + app: FastAPI, + *, + user_id: UserID, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, + package_id: DatCorePackageName, +) -> PathMetaData: + pck_files = await get_package_files( + app, + api_key=api_key, + api_secret=api_secret, + package_id=package_id, + ) + + assert len(pck_files) == 1 # nosec + return create_path_meta_data_from_datcore_package( + user_id, + dataset_id, + pck_files[0], + ) + + +async def list_all_datasets( + app: FastAPI, api_key: str, api_secret: str +) -> list[DatasetMetaData]: + all_datasets: list[DatasetMetaData] = await retrieve_all_pages( + app, + api_key, + api_secret, + "GET", + "/datasets", + lambda d: DatasetMetaData(dataset_id=d["id"], display_name=d["display_name"]), + ) + + return all_datasets + + +async def list_datasets( + app: FastAPI, + *, + api_key: str, + api_secret: str, + cursor: GenericCursor | None, + limit: NonNegativeInt, +) -> tuple[list[DatasetMetaData], GenericCursor | None, TotalNumber]: + page, size = _init_pagination(cursor, limit) + + response = await request( + app, + api_key, + api_secret, + "GET", + "/datasets", + params={"size": size, "page": page}, + ) + assert isinstance(response, dict) # nosec + datasets_page = Page[DatCoreDatasetMetaData](**response) + datasets = datasets_page.items + total = datasets_page.total + + assert isinstance(total, int) # nosec + next_cursor = _create_next_cursor(total, page, size) + + return ( + [ + DatasetMetaData(dataset_id=d.id, display_name=d.display_name) + for d in datasets + ], + next_cursor, + total, + ) + + +async def get_dataset( + app: FastAPI, + *, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, +) -> tuple[DatasetMetaData, ByteSize | None]: + response = await request( + app, + api_key, + api_secret, + "GET", + f"/datasets/{dataset_id}", + ) + assert isinstance(response, dict) # nosec + datcore_dataset = DatCoreDatasetMetaData(**response) + + return ( + DatasetMetaData( + dataset_id=datcore_dataset.id, display_name=datcore_dataset.display_name + ), + datcore_dataset.size, + ) + + +async def get_file_download_presigned_link( + app: FastAPI, api_key: str, api_secret: str, file_id: str +) -> AnyUrl: + try: + file_download_data = cast( + dict[str, Any], + await request(app, api_key, api_secret, "GET", f"/files/{file_id}"), + ) + return TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) + except DatcoreAdapterResponseError as exc: + if exc.status == status.HTTP_404_NOT_FOUND: + raise DatcoreAdapterFileNotFoundError(file_id=file_id) from exc + raise + + +async def get_package_files( + app: FastAPI, *, api_key: str, api_secret: str, package_id: str +) -> list[PackageMetaData]: + return TypeAdapter(list[PackageMetaData]).validate_python( + await request( + app, + api_key, + api_secret, + "GET", + f"/packages/{package_id}/files", + ) + ) + + +async def delete_file( + app: FastAPI, api_key: str, api_secret: str, file_id: str +) -> None: + await request(app, api_key, api_secret, "DELETE", f"/files/{file_id}") diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_client_utils.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_client_utils.py new file mode 100644 index 00000000000..3972d07d72d --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_client_utils.py @@ -0,0 +1,99 @@ +import logging +from collections.abc import Callable +from math import ceil +from typing import Any, TypeVar, cast + +import httpx +from fastapi import FastAPI +from servicelib.fastapi.client_session import get_client_session + +from ...core.settings import get_application_settings +from .datcore_adapter_exceptions import ( + DatcoreAdapterClientError, + DatcoreAdapterResponseError, + DatcoreAdapterTimeoutError, +) + +_logger = logging.getLogger(__file__) + + +async def request( + app: FastAPI, + api_key: str, + api_secret: str, + method: str, + path: str, + *, + json: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + **request_kwargs, +) -> dict[str, Any] | list[dict[str, Any]]: + datcore_adapter_settings = get_application_settings(app).DATCORE_ADAPTER + url = datcore_adapter_settings.endpoint + path + session = get_client_session(app) + + try: + if request_kwargs is None: + request_kwargs = {} + response = await session.request( + method.upper(), + url, + headers={ + "x-datcore-api-key": api_key, + "x-datcore-api-secret": api_secret, + }, + json=json, + params=params, + **request_kwargs, + ) + response.raise_for_status() + response_data = response.json() + assert isinstance(response_data, dict | list) # nosec + return response_data + + except httpx.HTTPStatusError as exc: + raise DatcoreAdapterResponseError( + status=exc.response.status_code, reason=f"{exc}" + ) from exc + + except TimeoutError as exc: + msg = f"datcore-adapter server timed-out: {exc}" + raise DatcoreAdapterTimeoutError(msg) from exc + + except httpx.RequestError as exc: + msg = f"unexpected request error: {exc}" + raise DatcoreAdapterClientError(msg) from exc + + +_T = TypeVar("_T") + + +async def retrieve_all_pages( + app: FastAPI, + api_key: str, + api_secret: str, + method: str, + path: str, + return_type_creator: Callable[..., _T], +) -> list[_T]: + page = 1 + objs = [] + while ( + response := cast( + dict[str, Any], + await request( + app, api_key, api_secret, method, path, params={"page": page} + ), + ) + ) and response.get("items"): + _logger.debug( + "called %s [%d/%d], received %d objects", + path, + page, + ceil(response.get("total", -1) / response.get("size", 1)), + len(response.get("items", [])), + ) + + objs += [return_type_creator(d) for d in response.get("items", [])] + page += 1 + return objs diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_exceptions.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py similarity index 63% rename from services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_exceptions.py rename to services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py index 03ba1aa603c..f4643380ab0 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_exceptions.py +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py @@ -33,3 +33,22 @@ class DatcoreAdapterMultipleFilesError(DatcoreAdapterError): def __init__(self, msg: str) -> None: super().__init__(msg=msg) + + +class DatcoreAdapterResponseError(DatcoreAdapterError): + """Basic exception for response errors""" + + def __init__(self, status: int, reason: str) -> None: + self.status = status + self.reason = reason + super().__init__( + msg=f"forwarded call failed with status {status}, reason {reason}" + ) + + +class DatcoreAdapterFileNotFoundError(DatcoreAdapterError): + """special error to check the assumption that /packages/{package_id}/files returns only one file""" + + def __init__(self, file_id: str) -> None: + self.file_id = file_id + super().__init__(msg=f"file {file_id} not found!") diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_settings.py similarity index 100% rename from services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py rename to services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_settings.py diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/utils.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/utils.py new file mode 100644 index 00000000000..34a1379f8c0 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/utils.py @@ -0,0 +1,97 @@ +from pathlib import Path + +from models_library.api_schemas_datcore_adapter.datasets import ( + DataType as DatCoreDataType, +) +from models_library.api_schemas_datcore_adapter.datasets import ( + FileMetaData as DatCoreFileMetaData, +) +from models_library.api_schemas_datcore_adapter.datasets import PackageMetaData +from models_library.api_schemas_storage.storage_schemas import DatCoreDatasetName +from models_library.users import UserID +from pydantic import ByteSize + +from ...constants import DATCORE_ID, DATCORE_STR +from ...models import FileMetaData, PathMetaData + + +def create_fmd_from_datcore_package( + user_id: UserID, pck_metadata: PackageMetaData +) -> FileMetaData: + return FileMetaData( + file_uuid=f"{pck_metadata.package_id}", + location_id=DATCORE_ID, + location=DATCORE_STR, + bucket_name=pck_metadata.s3_bucket, + object_name=f"{pck_metadata.package_id}", + file_name=pck_metadata.name, + file_id=pck_metadata.package_id, + file_size=ByteSize(pck_metadata.size), + created_at=pck_metadata.created_at, + last_modified=pck_metadata.updated_at, + project_id=None, + node_id=None, + user_id=user_id, + is_soft_link=False, + sha256_checksum=None, + ) + + +def create_fmd_from_datcore_fmd( + user_id: UserID, dat_core_fmd: DatCoreFileMetaData +) -> FileMetaData: + return FileMetaData( + file_uuid=f"{dat_core_fmd.path}", + location_id=DATCORE_ID, + location=DATCORE_STR, + bucket_name=dat_core_fmd.dataset_id, + object_name=f"{dat_core_fmd.package_id}", + file_name=dat_core_fmd.name, + file_id=dat_core_fmd.package_id, + file_size=ByteSize(dat_core_fmd.size), + created_at=dat_core_fmd.created_at, + last_modified=dat_core_fmd.last_modified_at, + project_id=None, + node_id=None, + user_id=user_id, + is_soft_link=False, + sha256_checksum=None, + ) + + +def create_path_meta_data_from_datcore_package( + user_id: UserID, dataset_id: DatCoreDatasetName, pck_metadata: PackageMetaData +) -> PathMetaData: + return PathMetaData( + path=Path(dataset_id) / pck_metadata.package_id, + display_path=pck_metadata.display_path, + location_id=DATCORE_ID, + location=DATCORE_STR, + bucket_name=pck_metadata.s3_bucket, + project_id=None, + node_id=None, + user_id=user_id, + created_at=pck_metadata.created_at, + last_modified=pck_metadata.updated_at, + file_meta_data=create_fmd_from_datcore_package(user_id, pck_metadata), + ) + + +def create_path_meta_data_from_datcore_fmd( + user_id: UserID, dat_core_fmd: DatCoreFileMetaData +) -> PathMetaData: + return PathMetaData( + path=Path(dat_core_fmd.dataset_id) / dat_core_fmd.id, + display_path=dat_core_fmd.path, + location_id=DATCORE_ID, + location=DATCORE_STR, + bucket_name=dat_core_fmd.dataset_id, + project_id=None, + node_id=None, + user_id=user_id, + created_at=dat_core_fmd.created_at, + last_modified=dat_core_fmd.last_modified_at, + file_meta_data=None + if dat_core_fmd.data_type == DatCoreDataType.FOLDER + else create_fmd_from_datcore_fmd(user_id, dat_core_fmd), + ) diff --git a/services/storage/src/simcore_service_storage/modules/db/__init__.py b/services/storage/src/simcore_service_storage/modules/db/__init__.py new file mode 100644 index 00000000000..41372f5a2c2 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/db/__init__.py @@ -0,0 +1,31 @@ +import logging + +from fastapi import FastAPI +from servicelib.db_async_engine import close_db_connection +from servicelib.fastapi.db_asyncpg_engine import connect_to_db +from servicelib.retry_policies import PostgresRetryPolicyUponInitialization +from sqlalchemy.ext.asyncio import AsyncEngine +from tenacity import retry + +from ...core.settings import get_application_settings + +_logger = logging.getLogger(__name__) + + +def setup_db(app: FastAPI) -> None: + @retry(**PostgresRetryPolicyUponInitialization(_logger).kwargs) + async def _on_startup() -> None: + app_settings = get_application_settings(app) + assert app_settings.STORAGE_POSTGRES is not None # nosec + await connect_to_db(app, app_settings.STORAGE_POSTGRES) + + async def _on_shutdown() -> None: + await close_db_connection(app) + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + +def get_db_engine(app: FastAPI) -> AsyncEngine: + assert isinstance(app.state.engine, AsyncEngine) # nosec + return app.state.engine diff --git a/services/storage/src/simcore_service_storage/modules/db/_base.py b/services/storage/src/simcore_service_storage/modules/db/_base.py new file mode 100644 index 00000000000..5bacb6def41 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/db/_base.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from typing import TypeVar + +from sqlalchemy.ext.asyncio import AsyncEngine + +RepositoryType = TypeVar("RepositoryType", bound="BaseRepository") + + +@dataclass +class BaseRepository: + db_engine: AsyncEngine + + @classmethod + def instance(cls: type[RepositoryType], db_engine: AsyncEngine) -> RepositoryType: + return cls(db_engine=db_engine) diff --git a/services/storage/src/simcore_service_storage/db_access_layer.py b/services/storage/src/simcore_service_storage/modules/db/access_layer.py similarity index 52% rename from services/storage/src/simcore_service_storage/db_access_layer.py rename to services/storage/src/simcore_service_storage/modules/db/access_layer.py index 27f9dfb9214..0f70f0e959a 100644 --- a/services/storage/src/simcore_service_storage/db_access_layer.py +++ b/services/storage/src/simcore_service_storage/modules/db/access_layer.py @@ -1,4 +1,4 @@ -""" Helper functions to determin access-rights on stored data +"""Helper functions to determin access-rights on stored data # DRAFT Rationale: @@ -37,11 +37,8 @@ """ import logging -from dataclasses import dataclass import sqlalchemy as sa -from aiopg.sa.connection import SAConnection -from aiopg.sa.result import ResultProxy, RowProxy from models_library.groups import GroupID from models_library.projects import ProjectID from models_library.projects_nodes_io import StorageFileID @@ -52,49 +49,22 @@ workspaces_access_rights, ) from simcore_postgres_database.storage_models import file_meta_data, user_to_groups +from simcore_postgres_database.utils_repos import pass_or_acquire_connection from simcore_postgres_database.utils_sql import assemble_array_groups +from sqlalchemy.ext.asyncio import AsyncConnection -logger = logging.getLogger(__name__) +from ...exceptions.errors import InvalidFileIdentifierError +from ...models import AccessRights +from ._base import BaseRepository +_logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class AccessRights: - read: bool - write: bool - delete: bool - @classmethod - def all(cls) -> "AccessRights": - return cls(read=True, write=True, delete=True) - - @classmethod - def none(cls) -> "AccessRights": - return cls(read=False, write=False, delete=False) - - -class AccessLayerError(Exception): - """Base class for access-layer related errors""" - - -class InvalidFileIdentifierError(AccessLayerError): - """Identifier does not follow the criteria to - be a file identifier (see naming criteria below) - """ - - def __init__(self, identifier, reason=None, details=None): - self.identifier = identifier - self.reason = reason or "Invalid file identifier" - self.details = details - - super().__init__(self.reason, self.details) - - def __str__(self): - return f"Error in {self.identifier}: {self.reason} [{self.details}]" - - -async def _get_user_groups_ids(conn: SAConnection, user_id: UserID) -> list[GroupID]: +async def _get_user_groups_ids( + connection: AsyncConnection, user_id: UserID +) -> list[GroupID]: stmt = sa.select(user_to_groups.c.gid).where(user_to_groups.c.uid == user_id) - rows = await (await conn.execute(stmt)).fetchall() + rows = (await connection.execute(stmt)).fetchall() assert rows is not None # nosec return [g.gid for g in rows] @@ -112,7 +82,7 @@ def _aggregate_access_rights( return AccessRights(**prj_access) except KeyError: # NOTE: database does NOT include schema for json access_rights column! - logger.warning( + _logger.warning( "Invalid entry in projects.access_rights. Revoking all rights [%s]", access_rights, ) @@ -159,14 +129,14 @@ def _aggregate_access_rights( ).subquery("workspace_access_rights_subquery") -async def list_projects_access_rights( - conn: SAConnection, user_id: UserID +async def _list_projects_access_rights( + connection: AsyncConnection, user_id: UserID ) -> dict[ProjectID, AccessRights]: """ Returns access-rights of user (user_id) over all OWNED or SHARED projects """ - user_group_ids: list[GroupID] = await _get_user_groups_ids(conn, user_id) + user_group_ids: list[GroupID] = await _get_user_groups_ids(connection, user_id) private_workspace_query = ( sa.select( @@ -211,12 +181,12 @@ async def list_projects_access_rights( projects_access_rights = {} - async for row in conn.execute(combined_query): + async for row in await connection.stream(combined_query): assert isinstance(row.access_rights, dict) # nosec assert isinstance(row.uuid, str) # nosec if row.access_rights: - # TODO: access_rights should be direclty filtered from result in stm instead calling again user_group_ids + # NOTE: access_rights should be direclty filtered from result in stm instead calling again user_group_ids projects_access_rights[ProjectID(row.uuid)] = _aggregate_access_rights( row.access_rights, user_group_ids ) @@ -229,156 +199,164 @@ async def list_projects_access_rights( return projects_access_rights -async def get_project_access_rights( - conn: SAConnection, user_id: UserID, project_id: ProjectID -) -> AccessRights: - """ - Returns access-rights of user (user_id) over a project resource (project_id) - """ - user_group_ids: list[GroupID] = await _get_user_groups_ids(conn, user_id) - - private_workspace_query = ( - sa.select( - projects.c.prj_owner, - access_rights_subquery.c.access_rights, - ) - .select_from(projects.join(access_rights_subquery, isouter=True)) - .where( - (projects.c.uuid == f"{project_id}") - & ( - (projects.c.prj_owner == user_id) - | sa.text( - f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" +class AccessLayerRepository(BaseRepository): + async def get_project_access_rights( + self, + *, + connection: AsyncConnection | None = None, + user_id: UserID, + project_id: ProjectID, + ) -> AccessRights: + """ + Returns access-rights of user (user_id) over a project resource (project_id) + """ + + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + user_group_ids = await _get_user_groups_ids(conn, user_id) + + private_workspace_query = ( + sa.select( + projects.c.prj_owner, + access_rights_subquery.c.access_rights, + ) + .select_from(projects.join(access_rights_subquery, isouter=True)) + .where( + (projects.c.uuid == f"{project_id}") + & ( + (projects.c.prj_owner == user_id) + | sa.text( + f"jsonb_exists_any(access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" + ) + ) + & (projects.c.workspace_id.is_(None)) ) ) - & (projects.c.workspace_id.is_(None)) - ) - ) - shared_workspace_query = ( - sa.select( - projects.c.prj_owner, - workspace_access_rights_subquery.c.access_rights, - ) - .select_from( - projects.join( - workspace_access_rights_subquery, - projects.c.workspace_id - == workspace_access_rights_subquery.c.workspace_id, - ) - ) - .where( - (projects.c.uuid == f"{project_id}") - & ( - sa.text( - f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" + shared_workspace_query = ( + sa.select( + projects.c.prj_owner, + workspace_access_rights_subquery.c.access_rights, + ) + .select_from( + projects.join( + workspace_access_rights_subquery, + projects.c.workspace_id + == workspace_access_rights_subquery.c.workspace_id, + ) + ) + .where( + (projects.c.uuid == f"{project_id}") + & ( + sa.text( + f"jsonb_exists_any(workspace_access_rights_subquery.access_rights, {assemble_array_groups(user_group_ids)})" + ) + ) + & (projects.c.workspace_id.is_not(None)) ) ) - & (projects.c.workspace_id.is_not(None)) - ) - ) - - combined_query = sa.union_all(private_workspace_query, shared_workspace_query) - - result: ResultProxy = await conn.execute(combined_query) - row: RowProxy | None = await result.first() - if not row: - # Either project does not exists OR user_id has NO access - return AccessRights.none() - - assert row.prj_owner is None or isinstance(row.prj_owner, int) # nosec - assert isinstance(row.access_rights, dict) # nosec - - if row.prj_owner == user_id: - return AccessRights.all() - - # determine user's access rights by aggregating AR of all groups - return _aggregate_access_rights(row.access_rights, user_group_ids) - - -async def get_file_access_rights( - conn: SAConnection, user_id: UserID, file_id: StorageFileID -) -> AccessRights: - """ - Returns access-rights of user (user_id) over data file resource (file_id) + combined_query = sa.union_all( + private_workspace_query, shared_workspace_query + ) + result = await conn.execute(combined_query) + row = result.one_or_none() - raises InvalidFileIdentifier - """ + if not row: + # Either project does not exists OR user_id has NO access + return AccessRights.none() - # - # 1. file registered in file_meta_data table - # - stmt = sa.select(file_meta_data.c.project_id, file_meta_data.c.user_id).where( - file_meta_data.c.file_id == f"{file_id}" - ) - result: ResultProxy = await conn.execute(stmt) - row: RowProxy | None = await result.first() + assert row.prj_owner is None or isinstance(row.prj_owner, int) # nosec + assert isinstance(row.access_rights, dict) # nosec - if row: - if int(row.user_id) == user_id: - # is owner + if row.prj_owner == user_id: return AccessRights.all() - if not row.project_id: - # not owner and not shared via project - return AccessRights.none() + # determine user's access rights by aggregating AR of all groups + return _aggregate_access_rights(row.access_rights, user_group_ids) - # has associated project - access_rights = await get_project_access_rights( - conn, user_id, project_id=row.project_id - ) - if not access_rights: - logger.warning( - "File %s references a project %s that does not exists in db." - "TIP: Audit sync between files_meta_data and projects tables", - file_id, - row.project_id, - ) - return AccessRights.none() + async def get_file_access_rights( + self, + *, + connection: AsyncConnection | None = None, + user_id: UserID, + file_id: StorageFileID, + ) -> AccessRights: + """ + Returns access-rights of user (user_id) over data file resource (file_id) + + raises InvalidFileIdentifier + """ - else: - # - # 2. file is NOT registered in meta-data table e.g. it is about to be uploaded or it was deleted - # We rely on the assumption that file_id is formatted either as # - # - project's data: {project_id}/{node_id}/{filename/with/possible/folders} - # - API data: api/{file_id}/{filename/with/possible/folders} + # 1. file registered in file_meta_data table # - try: - parent, _, _ = file_id.split("/", maxsplit=2) + stmt = sa.select(file_meta_data.c.project_id, file_meta_data.c.user_id).where( + file_meta_data.c.file_id == f"{file_id}" + ) + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + result = await conn.execute(stmt) + row = result.one_or_none() - if parent == "api": - # FIXME: this is wrong, all api data must be registered and OWNED - # ownership still not defined, so we assume it is user_id + if row: + if int(row.user_id) == user_id: + # is owner return AccessRights.all() - # otherwise assert 'parent' string corresponds to a valid UUID - access_rights = await get_project_access_rights( - conn, user_id, project_id=ProjectID(parent) + if not row.project_id: + # not owner and not shared via project + return AccessRights.none() + + # has associated project + access_rights = await self.get_project_access_rights( + user_id=user_id, project_id=row.project_id ) if not access_rights: - logger.warning( - "File %s references a project that does not exists in db", + _logger.warning( + "File %s references a project %s that does not exists in db." + "TIP: Audit sync between files_meta_data and projects tables", file_id, + row.project_id, ) return AccessRights.none() - except (ValueError, AttributeError) as err: - raise InvalidFileIdentifierError( - identifier=file_id, - details=str(err), - ) from err - - return access_rights - - -# HELPERS ----------------------------------------------- - - -async def get_readable_project_ids( - conn: SAConnection, user_id: UserID -) -> list[ProjectID]: - """Returns a list of projects where user has granted read-access""" - projects_access_rights = await list_projects_access_rights(conn, user_id) - return [pid for pid, access in projects_access_rights.items() if access.read] + else: + # + # 2. file is NOT registered in meta-data table e.g. it is about to be uploaded or it was deleted + # We rely on the assumption that file_id is formatted either as + # + # - project's data: {project_id}/{node_id}/{filename/with/possible/folders} + # - API data: api/{file_id}/{filename/with/possible/folders} + # + try: + parent, _, _ = file_id.split("/", maxsplit=2) + + if parent == "api": + # ownership still not defined, so we assume it is user_id + return AccessRights.all() + + # otherwise assert 'parent' string corresponds to a valid UUID + access_rights = await self.get_project_access_rights( + user_id=user_id, project_id=ProjectID(parent) + ) + if not access_rights: + _logger.warning( + "File %s references a project that does not exists in db", + file_id, + ) + return AccessRights.none() + + except (ValueError, AttributeError) as err: + raise InvalidFileIdentifierError( + identifier=file_id, + details=str(err), + ) from err + + return access_rights + + async def get_readable_project_ids( + self, *, connection: AsyncConnection | None = None, user_id: UserID + ) -> list[ProjectID]: + """Returns a list of projects where user has granted read-access""" + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + projects_access_rights = await _list_projects_access_rights(conn, user_id) + return [pid for pid, access in projects_access_rights.items() if access.read] diff --git a/services/storage/src/simcore_service_storage/modules/db/file_meta_data.py b/services/storage/src/simcore_service_storage/modules/db/file_meta_data.py new file mode 100644 index 00000000000..1c942b4f436 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/db/file_meta_data.py @@ -0,0 +1,442 @@ +import contextlib +import datetime +from collections.abc import AsyncGenerator +from pathlib import Path +from typing import TypeAlias + +import sqlalchemy as sa +from models_library.basic_types import SHA256Str +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID, SimcoreS3FileID +from models_library.users import UserID +from models_library.utils.fastapi_encoders import jsonable_encoder +from pydantic import BaseModel +from simcore_postgres_database.storage_models import file_meta_data +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy import and_, literal_column +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import MultipleResultsFound +from sqlalchemy.ext.asyncio import AsyncConnection + +from ...exceptions.errors import FileMetaDataNotFoundError +from ...models import ( + FileMetaData, + FileMetaDataAtDB, + GenericCursor, + PathMetaData, + UserOrProjectFilter, +) +from ._base import BaseRepository + +TotalChildren: TypeAlias = int + + +class _PathsCursorParameters(BaseModel): + offset: int + file_prefix: Path | None + project_ids: list[ProjectID] | None + partial: bool + + +def _init_pagination( + cursor: GenericCursor | None, + *, + filter_by_project_ids: list[ProjectID] | None, + filter_by_file_prefix: Path | None, + is_partial_prefix: bool, +) -> _PathsCursorParameters: + if cursor: + return _PathsCursorParameters.model_validate_json(cursor) + return _PathsCursorParameters( + offset=0, + file_prefix=filter_by_file_prefix, + project_ids=filter_by_project_ids, + partial=is_partial_prefix, + ) + + +def _create_next_cursor( + total_count: TotalChildren, limit: int, cursor_params: _PathsCursorParameters +) -> GenericCursor | None: + if cursor_params.offset + limit < total_count: + return cursor_params.model_copy( + update={"offset": cursor_params.offset + limit} + ).model_dump_json() + return None + + +def _list_filter_with_partial_file_id_stmt( + *, + user_or_project_filter: UserOrProjectFilter, + file_id_prefix: str | None, + partial_file_id: str | None, + sha256_checksum: SHA256Str | None, + is_directory: bool | None, + limit: int | None = None, + offset: int | None = None, +): + conditions: list = [] + + # Checks access rights (project can be owned or shared) + user_id = user_or_project_filter.user_id + if user_id is not None: + project_ids = user_or_project_filter.project_ids + conditions.append( + sa.or_( + file_meta_data.c.user_id == f"{user_id}", + ( + file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) + if project_ids + else False + ), + ) + ) + + # Optional filters + if file_id_prefix: + conditions.append(file_meta_data.c.file_id.startswith(file_id_prefix)) + if partial_file_id: + conditions.append(file_meta_data.c.file_id.ilike(f"%{partial_file_id}%")) + if is_directory is not None: + conditions.append(file_meta_data.c.is_directory.is_(is_directory)) + if sha256_checksum: + conditions.append(file_meta_data.c.sha256_checksum == sha256_checksum) + + return ( + sa.select(file_meta_data) + .where(sa.and_(*conditions)) + .order_by(file_meta_data.c.created_at.asc()) # sorted as oldest first + .offset(offset) + .limit(limit) + ) + + +class FileMetaDataRepository(BaseRepository): + async def exists( + self, *, connection: AsyncConnection | None = None, file_id: SimcoreS3FileID + ) -> bool: + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + return bool( + await conn.scalar( + sa.select(sa.func.count()) + .select_from(file_meta_data) + .where(file_meta_data.c.file_id == file_id) + ) + == 1 + ) + + async def upsert( + self, + *, + connection: AsyncConnection | None = None, + fmd: FileMetaData | FileMetaDataAtDB, + ) -> FileMetaDataAtDB: + # NOTE: upsert file_meta_data, if the file already exists, we update the whole row + # so we get the correct time stamps + fmd_db = ( + FileMetaDataAtDB.model_validate(fmd) + if isinstance(fmd, FileMetaData) + else fmd + ) + insert_statement = pg_insert(file_meta_data).values(**fmd_db.model_dump()) + on_update_statement = insert_statement.on_conflict_do_update( + index_elements=[file_meta_data.c.file_id], set_=fmd_db.model_dump() + ).returning(literal_column("*")) + async with transaction_context(self.db_engine, connection) as conn: + result = await conn.execute(on_update_statement) + row = result.one() + return FileMetaDataAtDB.model_validate(row) + + async def insert( + self, *, connection: AsyncConnection | None = None, fmd: FileMetaData + ) -> FileMetaDataAtDB: + fmd_db = FileMetaDataAtDB.model_validate(fmd) + async with transaction_context(self.db_engine, connection) as conn: + result = await conn.execute( + file_meta_data.insert() + .values(jsonable_encoder(fmd_db)) + .returning(literal_column("*")) + ) + row = result.one() + return FileMetaDataAtDB.model_validate(row) + + async def get( + self, *, connection: AsyncConnection | None = None, file_id: SimcoreS3FileID + ) -> FileMetaDataAtDB: + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + result = await conn.execute( + sa.select(file_meta_data).where(file_meta_data.c.file_id == file_id) + ) + if row := result.one_or_none(): + return FileMetaDataAtDB.model_validate(row) + raise FileMetaDataNotFoundError(file_id=file_id) + + async def list_filter_with_partial_file_id( + self, + *, + connection: AsyncConnection | None = None, + user_or_project_filter: UserOrProjectFilter, + file_id_prefix: str | None, + partial_file_id: str | None, + sha256_checksum: SHA256Str | None, + is_directory: bool | None, + limit: int | None = None, + offset: int | None = None, + ) -> list[FileMetaDataAtDB]: + stmt = _list_filter_with_partial_file_id_stmt( + user_or_project_filter=user_or_project_filter, + file_id_prefix=file_id_prefix, + partial_file_id=partial_file_id, + sha256_checksum=sha256_checksum, + is_directory=is_directory, + limit=limit, + offset=offset, + ) + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + return [ + FileMetaDataAtDB.model_validate(row) + async for row in await conn.stream(stmt) + ] + + async def try_get_directory( + self, *, connection: AsyncConnection | None = None, file_filter: Path + ) -> FileMetaData | None: + """Check if the given file filter is a directory or is inside a directory.""" + # we might be exactly on a directory or inside it + potential_directories = (file_filter, *file_filter.parents) + with contextlib.suppress(MultipleResultsFound): + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + for file_id in potential_directories: + # there should be only 1 entry if this is a directory + result = await conn.execute( + sa.select(file_meta_data).where( + file_meta_data.c.file_id == f"{file_id}" + ) + ) + if row := result.one_or_none(): + fmd = FileMetaDataAtDB.model_validate(row) + if fmd.is_directory: + return FileMetaData.from_db_model(fmd) + return None + return None + + async def list_child_paths( + self, + *, + connection: AsyncConnection | None = None, + filter_by_project_ids: list[ProjectID] | None, + filter_by_file_prefix: Path | None, + cursor: GenericCursor | None, + limit: int, + is_partial_prefix: bool, + ) -> tuple[list[PathMetaData], GenericCursor | None, TotalChildren]: + """returns a list of FileMetaDataAtDB that are one level deep. + e.g. when no filter is used, these are top level objects + """ + + cursor_params = _init_pagination( + cursor, + filter_by_project_ids=filter_by_project_ids, + filter_by_file_prefix=filter_by_file_prefix, + is_partial_prefix=is_partial_prefix, + ) + + if cursor_params.file_prefix: + prefix_levels = len(cursor_params.file_prefix.parts) - 1 + search_prefix = ( + f"{cursor_params.file_prefix}%" + if cursor_params.partial + else f"{cursor_params.file_prefix / '%'}" + ) + search_regex = rf"^[^/]+(?:/[^/]+){{{prefix_levels}}}{'' if cursor_params.partial else '/[^/]+'}" + ranked_files = ( + sa.select( + file_meta_data.c.file_id, + sa.func.substring(file_meta_data.c.file_id, search_regex).label( + "path" + ), + sa.func.row_number() + .over( + partition_by=sa.func.substring( + file_meta_data.c.file_id, search_regex + ), + order_by=(file_meta_data.c.file_id.asc(),), + ) + .label("row_num"), + ) + .where( + and_( + file_meta_data.c.file_id.like(search_prefix), + ( + file_meta_data.c.project_id.in_( + [f"{_}" for _ in cursor_params.project_ids] + ) + if cursor_params.project_ids + else True + ), + ) + ) + .cte("ranked_files") + ) + else: + ranked_files = ( + sa.select( + file_meta_data.c.file_id, + sa.func.split_part(file_meta_data.c.file_id, "/", 1).label("path"), + sa.func.row_number() + .over( + partition_by=sa.func.split_part( + file_meta_data.c.file_id, "/", 1 + ), + order_by=(file_meta_data.c.file_id.asc(),), + ) + .label("row_num"), + ) + .where( + file_meta_data.c.project_id.in_( + [f"{_}" for _ in cursor_params.project_ids] + ) + if cursor_params.project_ids + else True + ) + .cte("ranked_files") + ) + + files_query = ( + ( + sa.select(ranked_files, file_meta_data) + .where( + and_( + ranked_files.c.row_num == 1, + ranked_files.c.file_id == file_meta_data.c.file_id, + ) + ) + .order_by(file_meta_data.c.file_id.asc()) + ) + .limit(limit) + .offset(cursor_params.offset) + ) + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + total_count = await conn.scalar( + sa.select(sa.func.count()) + .select_from(ranked_files) + .where(ranked_files.c.row_num == 1) + ) + + items = [ + PathMetaData( + path=row.path + or row.file_id, # NOTE: if path_prefix is partial then path is None + display_path=row.path or row.file_id, + location_id=row.location_id, + location=row.location, + bucket_name=row.bucket_name, + project_id=row.project_id, + node_id=row.node_id, + user_id=row.user_id, + created_at=row.created_at, + last_modified=row.last_modified, + file_meta_data=( + FileMetaData.from_db_model(FileMetaDataAtDB.model_validate(row)) + if row.file_id == row.path and not row.is_directory + else None + ), + ) + async for row in await conn.stream(files_query) + ] + + return ( + items, + _create_next_cursor(total_count, limit, cursor_params), + total_count, + ) + + async def list_fmds( + self, + *, + connection: AsyncConnection | None = None, + user_id: UserID | None = None, + project_ids: list[ProjectID] | None = None, + file_ids: list[SimcoreS3FileID] | None = None, + expired_after: datetime.datetime | None = None, + ) -> list[FileMetaDataAtDB]: + stmt = sa.select(file_meta_data).where( + and_( + (file_meta_data.c.user_id == f"{user_id}") if user_id else True, + ( + (file_meta_data.c.project_id.in_([f"{p}" for p in project_ids])) + if project_ids + else True + ), + (file_meta_data.c.file_id.in_(file_ids)) if file_ids else True, + ( + (file_meta_data.c.upload_expires_at < expired_after) + if expired_after + else True + ), + ) + ) + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + return [ + FileMetaDataAtDB.model_validate(row) + async for row in await conn.stream(stmt) + ] + + async def total(self, *, connection: AsyncConnection | None = None) -> int: + """returns the number of uploaded file entries""" + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + return ( + await conn.scalar( + sa.select(sa.func.count()).select_from(file_meta_data) + ) + or 0 + ) + + async def list_valid_uploads( + self, + *, + connection: AsyncConnection | None = None, + ) -> AsyncGenerator[FileMetaDataAtDB, None]: + """returns all the theoretically valid fmds (e.g. upload_expires_at column is null)""" + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + async for row in await conn.stream( + sa.select(file_meta_data).where( + file_meta_data.c.upload_expires_at.is_( + None + ) # lgtm [py/test-equals-none] + ) + ): + fmd_at_db = FileMetaDataAtDB.model_validate(row) + yield fmd_at_db + + async def delete( + self, + *, + connection: AsyncConnection | None = None, + file_ids: list[SimcoreS3FileID], + ) -> None: + async with transaction_context(self.db_engine, connection) as conn: + await conn.execute( + file_meta_data.delete().where(file_meta_data.c.file_id.in_(file_ids)) + ) + + async def delete_all_from_project( + self, *, connection: AsyncConnection | None = None, project_id: ProjectID + ) -> None: + async with transaction_context(self.db_engine, connection) as conn: + await conn.execute( + file_meta_data.delete().where( + file_meta_data.c.project_id == f"{project_id}" + ) + ) + + async def delete_all_from_node( + self, *, connection: AsyncConnection | None = None, node_id: NodeID + ) -> None: + async with transaction_context(self.db_engine, connection) as conn: + await conn.execute( + file_meta_data.delete().where(file_meta_data.c.node_id == f"{node_id}") + ) diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py new file mode 100644 index 00000000000..765430a6dd1 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -0,0 +1,69 @@ +from collections.abc import AsyncIterator +from contextlib import suppress + +import sqlalchemy as sa +from models_library.projects import ProjectAtDB, ProjectID, ProjectIDStr +from models_library.projects_nodes_io import NodeIDStr +from pydantic import ValidationError +from simcore_postgres_database.storage_models import projects +from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from sqlalchemy.ext.asyncio import AsyncConnection + +from ._base import BaseRepository + + +class ProjectRepository(BaseRepository): + async def list_valid_projects_in( + self, + *, + connection: AsyncConnection | None = None, + include_uuids: list[ProjectID], + ) -> AsyncIterator[ProjectAtDB]: + """ + + NOTE that it lists ONLY validated projects in 'project_uuids' + """ + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + async for row in await conn.stream( + sa.select(projects).where( + projects.c.uuid.in_(f"{pid}" for pid in include_uuids) + ) + ): + with suppress(ValidationError): + yield ProjectAtDB.model_validate(row) + + async def project_exists( + self, + *, + connection: AsyncConnection | None = None, + project_uuid: ProjectID, + ) -> bool: + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + return bool( + await conn.scalar( + sa.select(sa.func.count()) + .select_from(projects) + .where(projects.c.uuid == f"{project_uuid}") + ) + == 1 + ) + + async def get_project_id_and_node_id_to_names_map( + self, + *, + connection: AsyncConnection | None = None, + project_uuids: list[ProjectID], + ) -> dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]]: + mapping = {} + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + async for row in await conn.stream( + sa.select(projects.c.uuid, projects.c.name, projects.c.workbench).where( + projects.c.uuid.in_(f"{pid}" for pid in project_uuids) + ) + ): + mapping[ProjectID(f"{row.uuid}")] = {f"{row.uuid}": row.name} | { + f"{node_id}": node["label"] + for node_id, node in row.workbench.items() + } + + return mapping diff --git a/services/storage/src/simcore_service_storage/modules/db/tokens.py b/services/storage/src/simcore_service_storage/modules/db/tokens.py new file mode 100644 index 00000000000..cf331ea3ca6 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/db/tokens.py @@ -0,0 +1,31 @@ +import logging + +import sqlalchemy as sa +from models_library.users import UserID +from simcore_postgres_database.storage_models import tokens +from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from sqlalchemy.ext.asyncio import AsyncConnection + +from ._base import BaseRepository + +_logger = logging.getLogger(__name__) + + +class TokenRepository(BaseRepository): + async def get_api_token_and_secret( + self, *, connection: AsyncConnection | None = None, user_id: UserID + ) -> tuple[str | None, str | None]: + async with pass_or_acquire_connection(self.db_engine, connection) as conn: + result = await conn.execute( + sa.select( + tokens, + ).where(tokens.c.user_id == user_id) + ) + row = result.one_or_none() + data = row._asdict() if row else {} + + data = data.get("token_data", {}) + api_token = data.get("token_key") + api_secret = data.get("token_secret") + + return api_token, api_secret diff --git a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py new file mode 100644 index 00000000000..d0c929f7adc --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py @@ -0,0 +1,20 @@ +import asyncio + +from fastapi import FastAPI +from servicelib.fastapi.long_running_tasks._server import setup + +from .._meta import API_VTAG + + +def setup_rest_api_long_running_tasks_for_uploads(app: FastAPI) -> None: + setup( + app, + router_prefix=f"/{API_VTAG}/futures", + ) + + app.state.completed_upload_tasks = {} + + +def get_completed_upload_tasks(app: FastAPI) -> dict[str, asyncio.Task]: + assert isinstance(app.state.completed_upload_tasks, dict) # nosec + return app.state.completed_upload_tasks diff --git a/services/storage/src/simcore_service_storage/modules/rabbitmq.py b/services/storage/src/simcore_service_storage/modules/rabbitmq.py new file mode 100644 index 00000000000..27b2f291630 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/rabbitmq.py @@ -0,0 +1,49 @@ +import logging +from typing import cast + +from fastapi import FastAPI +from servicelib.logging_utils import log_context +from servicelib.rabbitmq import ( + RabbitMQRPCClient, + wait_till_rabbitmq_responsive, +) +from settings_library.rabbit import RabbitSettings + +from ..exceptions.errors import ConfigurationError + +_logger = logging.getLogger(__name__) + + +def setup(app: FastAPI) -> None: + async def on_startup() -> None: + with log_context( + _logger, + logging.INFO, + msg="Storage startup Rabbitmq", + ): + rabbit_settings: RabbitSettings | None = app.state.settings.STORAGE_RABBITMQ + if not rabbit_settings: + raise ConfigurationError( + msg="RabbitMQ client is de-activated in the settings" + ) + await wait_till_rabbitmq_responsive(rabbit_settings.dsn) + app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name="storage_rpc_server", settings=rabbit_settings + ) + + async def on_shutdown() -> None: + with log_context( + _logger, + logging.INFO, + msg="Storage shutdown Rabbitmq", + ): + if app.state.rabbitmq_rpc_server: + await app.state.rabbitmq_rpc_server.close() + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + +def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_server # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) diff --git a/services/storage/src/simcore_service_storage/modules/redis.py b/services/storage/src/simcore_service_storage/modules/redis.py new file mode 100644 index 00000000000..6b2c15476ec --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/redis.py @@ -0,0 +1,34 @@ +import logging +from typing import cast + +from fastapi import FastAPI +from servicelib.redis import RedisClientSDK +from settings_library.redis import RedisDatabase + +from .._meta import APP_NAME +from ..core.settings import get_application_settings + +_logger = logging.getLogger(__name__) + + +def setup(app: FastAPI) -> None: + async def on_startup() -> None: + app.state.redis_client_sdk = None + redis_settings = get_application_settings(app).STORAGE_REDIS + assert redis_settings # nosec + redis_locks_dsn = redis_settings.build_redis_dsn(RedisDatabase.LOCKS) + app.state.redis_client_sdk = RedisClientSDK( + redis_locks_dsn, client_name=APP_NAME + ) + + async def on_shutdown() -> None: + redis_client_sdk = app.state.redis_client_sdk + if redis_client_sdk: + await cast(RedisClientSDK, app.state.redis_client_sdk).shutdown() + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + +def get_redis_client(app: FastAPI) -> RedisClientSDK: + return cast(RedisClientSDK, app.state.redis_client_sdk) diff --git a/services/storage/src/simcore_service_storage/modules/s3.py b/services/storage/src/simcore_service_storage/modules/s3.py new file mode 100644 index 00000000000..7fb81b29973 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/s3.py @@ -0,0 +1,69 @@ +"""Module to access s3 service""" + +import logging +from typing import Literal, cast + +from aws_library.s3 import SimcoreS3API +from common_library.json_serialization import json_dumps +from fastapi import FastAPI +from pydantic import TypeAdapter +from servicelib.logging_utils import log_context +from tenacity.asyncio import AsyncRetrying +from tenacity.before_sleep import before_sleep_log +from tenacity.wait import wait_fixed +from types_aiobotocore_s3.literals import BucketLocationConstraintType + +from ..constants import RETRY_WAIT_SECS +from ..core.settings import get_application_settings +from ..exceptions.errors import ConfigurationError + +_logger = logging.getLogger(__name__) + + +def setup_s3(app: FastAPI) -> None: + async def _on_startup() -> None: + app.state.s3_client = None + settings = get_application_settings(app) + + async for attempt in AsyncRetrying( + wait=wait_fixed(RETRY_WAIT_SECS), + before_sleep=before_sleep_log(_logger, logging.WARNING), + reraise=True, + ): + with attempt: + assert settings.STORAGE_S3 # nosec + client = await SimcoreS3API.create( + settings.STORAGE_S3, + settings.STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY, + ) + _logger.info( + "S3 client %s successfully created [%s]", + f"{client=}", + json_dumps(attempt.retry_state.retry_object.statistics), + ) + assert client # nosec + app.state.s3_client = client + + with log_context(_logger, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"): + assert settings.STORAGE_S3 # nosec + await client.create_bucket( + bucket=settings.STORAGE_S3.S3_BUCKET_NAME, + region=TypeAdapter( + BucketLocationConstraintType | Literal["us-east-1"] + ).validate_python(settings.STORAGE_S3.S3_REGION), + ) + + async def _on_shutdown() -> None: + if app.state.s3_client: + await cast(SimcoreS3API, app.state.s3_client).close() + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + +def get_s3_client(app: FastAPI) -> SimcoreS3API: + if not app.state.s3_client: + raise ConfigurationError( + msg="S3 client is not available. Please check the configuration." + ) + return cast(SimcoreS3API, app.state.s3_client) diff --git a/services/storage/src/simcore_service_storage/sts.py b/services/storage/src/simcore_service_storage/modules/sts.py similarity index 62% rename from services/storage/src/simcore_service_storage/sts.py rename to services/storage/src/simcore_service_storage/modules/sts.py index 85b9ffb4ecc..3bd5dcd2d27 100644 --- a/services/storage/src/simcore_service_storage/sts.py +++ b/services/storage/src/simcore_service_storage/modules/sts.py @@ -3,16 +3,16 @@ https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html """ -from aiohttp import web +from fastapi import FastAPI from models_library.users import UserID -from servicelib.aiohttp.application_keys import APP_CONFIG_KEY from settings_library.s3 import S3Settings -from .settings import Settings +from ..core.settings import get_application_settings async def get_or_create_temporary_token_for_user( - app: web.Application, _user_id: UserID + app: FastAPI, _user_id: UserID ) -> S3Settings: - app_settings: Settings = app[APP_CONFIG_KEY] + app_settings = get_application_settings(app) + assert app_settings.STORAGE_S3 # nosec return app_settings.STORAGE_S3 diff --git a/services/storage/src/simcore_service_storage/redis.py b/services/storage/src/simcore_service_storage/redis.py deleted file mode 100644 index 9bf600e3907..00000000000 --- a/services/storage/src/simcore_service_storage/redis.py +++ /dev/null @@ -1,37 +0,0 @@ -import logging -from typing import cast - -from aiohttp import web -from servicelib.redis import RedisClientSDK -from settings_library.redis import RedisDatabase, RedisSettings - -from ._meta import APP_NAME -from .constants import APP_CONFIG_KEY -from .settings import Settings - -_logger = logging.getLogger(__name__) - -_APP_REDIS_KEY = "APP_REDIS_KEY" - - -def setup_redis(app: web.Application): - async def _setup(app: web.Application): - app[_APP_REDIS_KEY] = None - settings: Settings = app[APP_CONFIG_KEY] - assert settings.STORAGE_REDIS # nosec - redis_settings: RedisSettings = settings.STORAGE_REDIS - redis_locks_dsn = redis_settings.build_redis_dsn(RedisDatabase.LOCKS) - app[_APP_REDIS_KEY] = client = RedisClientSDK( - redis_locks_dsn, client_name=APP_NAME - ) - - yield - - if client: - await client.shutdown() - - app.cleanup_ctx.append(_setup) - - -def get_redis_client(app: web.Application) -> RedisClientSDK: - return cast(RedisClientSDK, app[_APP_REDIS_KEY]) diff --git a/services/storage/src/simcore_service_storage/rest.py b/services/storage/src/simcore_service_storage/rest.py deleted file mode 100644 index 8fc8ca74986..00000000000 --- a/services/storage/src/simcore_service_storage/rest.py +++ /dev/null @@ -1,70 +0,0 @@ -""" RESTful API for simcore_service_storage - -""" - -import logging -from pathlib import Path - -from aiohttp import web -from aiohttp_swagger import setup_swagger # type: ignore[import-untyped] -from servicelib.aiohttp.rest_middlewares import append_rest_middlewares -from servicelib.aiohttp.rest_utils import ( - get_named_routes_as_message, - set_default_route_names, -) - -from . import ( - handlers_datasets, - handlers_files, - handlers_health, - handlers_locations, - handlers_simcore_s3, -) -from ._meta import API_VTAG -from .handlers_files import UPLOAD_TASKS_KEY -from .resources import storage_resources - -_logger = logging.getLogger(__name__) - - -def setup_rest(app: web.Application): - """Setup the rest API module in the application in aiohttp fashion. - - - loads and validate openapi specs from a remote (e.g. apihub) or local location - - connects openapi specs paths to handlers (see rest_routes.py) - - enables error, validation and envelope middlewares on API routes - - - IMPORTANT: this is a critical subsystem. Any failure should stop - the system startup. It CANNOT be simply disabled & continue - """ - _logger.debug("Setting up %s ...", __name__) - - spec_path: Path = storage_resources.get_path("api/v0/openapi.yaml") - - # Connects handlers - for routes in [ - handlers_health.routes, - handlers_locations.routes, - handlers_datasets.routes, - handlers_files.routes, - handlers_simcore_s3.routes, - ]: - set_default_route_names(routes) - app.router.add_routes(routes) - - _logger.debug("routes: %s", get_named_routes_as_message(app)) - - # prepare container for upload tasks - app[UPLOAD_TASKS_KEY] = {} - - # Enable error, validation and envelop middleware on API routes - append_rest_middlewares(app, api_version=f"/{API_VTAG}") - - # Adds swagger doc UI - setup_swagger( - app, - swagger_url="/dev/doc", - swagger_from_file=f"{spec_path}", - ui_version=3, - ) diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py deleted file mode 100644 index e96782b54ca..00000000000 --- a/services/storage/src/simcore_service_storage/s3.py +++ /dev/null @@ -1,77 +0,0 @@ -""" Module to access s3 service - -""" - -import logging -from collections.abc import AsyncGenerator -from typing import cast - -from aiohttp import web -from aws_library.s3 import SimcoreS3API -from common_library.json_serialization import json_dumps -from servicelib.logging_utils import log_context -from tenacity.asyncio import AsyncRetrying -from tenacity.before_sleep import before_sleep_log -from tenacity.wait import wait_fixed - -from .constants import APP_CONFIG_KEY, APP_S3_KEY, RETRY_WAIT_SECS -from .settings import Settings - -log = logging.getLogger(__name__) - - -async def setup_s3_client(app) -> AsyncGenerator[None, None]: - client = None - - with log_context(log, logging.DEBUG, msg="setup.s3_client.cleanup_ctx"): - storage_settings: Settings = app[APP_CONFIG_KEY] - storage_s3_settings = storage_settings.STORAGE_S3 - assert storage_s3_settings # nosec - - async for attempt in AsyncRetrying( - wait=wait_fixed(RETRY_WAIT_SECS), - before_sleep=before_sleep_log(log, logging.WARNING), - reraise=True, - ): - with attempt: - client = await SimcoreS3API.create( - storage_s3_settings, - storage_settings.STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY, - ) - log.info( - "S3 client %s successfully created [%s]", - f"{client=}", - json_dumps(attempt.retry_state.retry_object.statistics), - ) - assert client # nosec - app[APP_S3_KEY] = client - - yield - - with log_context(log, logging.DEBUG, msg="teardown.s3_client.cleanup_ctx"): - if client: - await client.close() - - -async def setup_s3_bucket(app: web.Application): - with log_context(log, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"): - storage_s3_settings = app[APP_CONFIG_KEY].STORAGE_S3 - client = get_s3_client(app) - await client.create_bucket( - bucket=storage_s3_settings.S3_BUCKET_NAME, - region=storage_s3_settings.S3_REGION, - ) - yield - - -def setup_s3(app: web.Application): - if setup_s3_client not in app.cleanup_ctx: - app.cleanup_ctx.append(setup_s3_client) - if setup_s3_bucket not in app.cleanup_ctx: - app.cleanup_ctx.append(setup_s3_bucket) - - -def get_s3_client(app: web.Application) -> SimcoreS3API: - assert app[APP_S3_KEY] # nosec - assert isinstance(app[APP_S3_KEY], SimcoreS3API) # nosec - return cast(SimcoreS3API, app[APP_S3_KEY]) diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py deleted file mode 100644 index 75d25311fcd..00000000000 --- a/services/storage/src/simcore_service_storage/settings.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import Annotated, Self - -from pydantic import ( - AliasChoices, - Field, - PositiveInt, - TypeAdapter, - field_validator, - model_validator, -) -from servicelib.logging_utils_filtering import LoggerName, MessageSubstring -from settings_library.base import BaseCustomSettings -from settings_library.basic_types import LogLevel, PortInt -from settings_library.postgres import PostgresSettings -from settings_library.redis import RedisSettings -from settings_library.s3 import S3Settings -from settings_library.tracing import TracingSettings -from settings_library.utils_logging import MixinLoggingSettings - -from .datcore_adapter.datcore_adapter_settings import DatcoreAdapterSettings - - -class Settings(BaseCustomSettings, MixinLoggingSettings): - STORAGE_HOST: str = "0.0.0.0" # nosec - STORAGE_PORT: PortInt = TypeAdapter(PortInt).validate_python(8080) - - LOG_LEVEL: Annotated[ - LogLevel, - Field( - validation_alias=AliasChoices("STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), - ), - ] = LogLevel.INFO - - STORAGE_MAX_WORKERS: PositiveInt = Field( - 8, - description="Number of workers for the thead executor pool used in DatcoreWrapper", - ) - - STORAGE_MONITORING_ENABLED: bool = False - STORAGE_PROFILING: bool = False - - BF_API_KEY: str | None = Field( - None, description="Pennsieve API key ONLY for testing purposes" - ) - BF_API_SECRET: str | None = Field( - None, description="Pennsieve API secret ONLY for testing purposes" - ) - - STORAGE_POSTGRES: PostgresSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_REDIS: RedisSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_S3: S3Settings = Field(json_schema_extra={"auto_default_from_env": True}) - - STORAGE_TRACING: TracingSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - DATCORE_ADAPTER: DatcoreAdapterSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_SYNC_METADATA_TIMEOUT: PositiveInt = Field( - 180, description="Timeout (seconds) for metadata sync task" - ) - - STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS: int = Field( - 3600, description="Default expiration time in seconds for presigned links" - ) - - STORAGE_CLEANER_INTERVAL_S: int | None = Field( - 30, - description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", - ) - - STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: int = Field( - 4, - description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", - ) - - STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - default=False, - validation_alias=AliasChoices( - "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", - "LOG_FORMAT_LOCAL_DEV_ENABLED", - ), - description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", - ) - STORAGE_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( - default_factory=dict, - validation_alias=AliasChoices( - "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" - ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", - ) - - @field_validator("LOG_LEVEL", mode="before") - @classmethod - def _validate_loglevel(cls, value: str) -> str: - log_level: str = cls.validate_log_level(value) - return log_level - - @model_validator(mode="after") - def _ensure_settings_consistency(self) -> Self: - if self.STORAGE_CLEANER_INTERVAL_S is not None and not self.STORAGE_REDIS: - msg = ( - "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! " - "Please correct settings." - ) - raise ValueError(msg) - return self diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index d41630b5230..6448cc0cc59 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -9,18 +9,16 @@ from pathlib import Path from typing import Any, Final, cast -import arrow -from aiohttp import web -from aiopg.sa import Engine -from aiopg.sa.connection import SAConnection from aws_library.s3 import ( CopiedBytesTransferredCallback, S3DirectoryMetaData, S3KeyNotFoundError, S3MetaData, UploadedBytesTransferredCallback, + UploadID, ) -from models_library.api_schemas_storage import ( +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( UNDEFINED_SIZE, UNDEFINED_SIZE_TYPE, LinkType, @@ -36,16 +34,15 @@ StorageFileID, ) from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter -from servicelib.aiohttp.client_session import get_client_session +from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter, ValidationError from servicelib.aiohttp.long_running_tasks.server import TaskProgress +from servicelib.fastapi.client_session import get_client_session from servicelib.logging_utils import log_context from servicelib.utils import ensure_ends_with, limited_gather +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.ext.asyncio import AsyncEngine -from . import db_file_meta_data, db_projects, db_tokens from .constants import ( - APP_AIOPG_ENGINE_KEY, - APP_CONFIG_KEY, DATCORE_ID, EXPAND_DIR_MAX_ITEM_COUNT, MAX_CONCURRENT_S3_TASKS, @@ -54,15 +51,9 @@ SIMCORE_S3_ID, SIMCORE_S3_STR, ) -from .datcore_adapter import datcore_adapter -from .db_access_layer import ( - AccessRights, - get_file_access_rights, - get_project_access_rights, - get_readable_project_ids, -) +from .core.settings import get_application_settings from .dsm_factory import BaseDataManager -from .exceptions import ( +from .exceptions.errors import ( FileAccessRightError, FileMetaDataNotFoundError, LinkAlreadyExistsError, @@ -73,19 +64,29 @@ DatasetMetaData, FileMetaData, FileMetaDataAtDB, - UploadID, + GenericCursor, + PathMetaData, + TotalNumber, UploadLinks, UserOrProjectFilter, ) -from .s3 import get_s3_client -from .s3_utils import S3TransferDataCB, update_task_progress -from .settings import Settings -from .simcore_s3_dsm_utils import ( +from .modules.datcore_adapter import datcore_adapter +from .modules.db import get_db_engine +from .modules.db.access_layer import AccessLayerRepository +from .modules.db.file_meta_data import FileMetaDataRepository +from .modules.db.projects import ProjectRepository +from .modules.db.tokens import TokenRepository +from .modules.s3 import get_s3_client +from .utils.s3_utils import S3TransferDataCB, update_task_progress +from .utils.simcore_s3_dsm_utils import ( compute_file_id_prefix, expand_directory, + get_accessible_project_ids, get_directory_file_id, + list_child_paths_from_repository, + list_child_paths_from_s3, ) -from .utils import ( +from .utils.utils import ( convert_db_to_model, download_to_file_or_raise, is_file_entry_valid, @@ -98,12 +99,45 @@ _logger = logging.getLogger(__name__) +async def _add_frontend_needed_data( + engine: AsyncEngine, + *, + project_ids: list[ProjectID], + data: list[FileMetaData], +) -> list[FileMetaData]: + # artifically fills ['project_name', 'node_name', 'file_id', 'raw_file_path', 'display_file_path'] + # with information from the projects table! + # NOTE: This part with the projects, should be done in the client code not here! + + prj_names_mapping: dict[ProjectID | NodeID, str] = {} + + async for proj_data in ProjectRepository.instance(engine).list_valid_projects_in( + include_uuids=project_ids + ): + prj_names_mapping |= {proj_data.uuid: proj_data.name} | { + NodeID(node_id): node_data.label + for node_id, node_data in proj_data.workbench.items() + } + + clean_data: list[FileMetaData] = [] + for d in data: + if d.project_id not in prj_names_mapping: + continue + assert d.project_id # nosec + d.project_name = prj_names_mapping[d.project_id] + if d.node_id in prj_names_mapping: + assert d.node_id # nosec + d.node_name = prj_names_mapping[d.node_id] + if d.node_name and d.project_name: + clean_data.append(d) + + return clean_data + + @dataclass class SimcoreS3DataManager(BaseDataManager): - engine: Engine simcore_bucket_name: S3BucketName - app: web.Application - settings: Settings + app: FastAPI @classmethod def get_location_id(cls) -> LocationID: @@ -117,17 +151,19 @@ async def authorized(self, _user_id: UserID) -> bool: return True # always true for now async def list_datasets(self, user_id: UserID) -> list[DatasetMetaData]: - async with self.engine.acquire() as conn: - readable_projects_ids = await get_readable_project_ids(conn, user_id) - return [ - DatasetMetaData( - dataset_id=prj_data.uuid, - display_name=prj_data.name, - ) - async for prj_data in db_projects.list_valid_projects_in( - conn, readable_projects_ids - ) - ] + readable_projects_ids = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_readable_project_ids(user_id=user_id) + + return [ + DatasetMetaData( + dataset_id=prj_data.uuid, + display_name=prj_data.name, + ) + async for prj_data in ProjectRepository.instance( + get_db_engine(self.app) + ).list_valid_projects_in(include_uuids=readable_projects_ids) + ] async def list_files_in_dataset( self, user_id: UserID, dataset_id: str, *, expand_dirs: bool @@ -141,7 +177,143 @@ async def list_files_in_dataset( ) return data - async def list_files( # noqa C901 + async def list_paths( + self, + user_id: UserID, + *, + file_filter: Path | None, + cursor: GenericCursor | None, + limit: NonNegativeInt, + ) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber | None]: + """returns a page of the file meta data a user has access to""" + + next_cursor: GenericCursor | None = None + total: TotalNumber | None = None + # if we have a file_filter, that means that we have potentially a project ID + project_id = None + with contextlib.suppress(ValueError): + # NOTE: we currently do not support anything else than project_id/node_id/file_path here, sorry chap + project_id = ProjectID(file_filter.parts[0]) if file_filter else None + + accessible_projects_ids = await get_accessible_project_ids( + get_db_engine(self.app), user_id=user_id, project_id=project_id + ) + + # check if the file_filter is a directory or inside one + dir_fmd = None + if file_filter: + dir_fmd = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).try_get_directory(file_filter=file_filter) + + if dir_fmd: + # NOTE: files are not listed in the DB but in S3 only + assert file_filter # nosec + assert project_id # nosec + (paths_metadata, next_cursor) = await list_child_paths_from_s3( + get_s3_client(self.app), + dir_fmd=dir_fmd, + bucket=self.simcore_bucket_name, + file_filter=file_filter, + limit=limit, + cursor=cursor, + ) + else: + # NOTE: files are DB-based + ( + paths_metadata, + next_cursor, + total, + ) = await list_child_paths_from_repository( + get_db_engine(self.app), + filter_by_project_ids=accessible_projects_ids, + filter_by_file_prefix=file_filter, + limit=limit, + cursor=cursor, + ) + + # extract the returned project_ids + project_ids = list( + {path.project_id for path in paths_metadata if path.project_id is not None} + ) + + ids_names_map = await ProjectRepository.instance( + get_db_engine(self.app) + ).get_project_id_and_node_id_to_names_map(project_uuids=project_ids) + + for path in paths_metadata: + if path.project_id is not None: + id_name_map = ids_names_map.get(path.project_id, {}) + path.update_display_fields(id_name_map) + + return paths_metadata, next_cursor, total + + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + # check access rights first + project_id = None + with contextlib.suppress(ValueError): + # NOTE: we currently do not support anything else than project_id/node_id/file_path here, sorry chap + project_id = ProjectID(path.parts[0]) + + accessible_projects_ids = await get_accessible_project_ids( + get_db_engine(self.app), user_id=user_id, project_id=project_id + ) + + # use-cases: + # 1. path is not a valid StorageFileID (e.g. a project or project/node) --> all entries are in the DB (files and folder) + # 2. path is valid StorageFileID and not in the DB --> entries are only in S3 + # 3. path is valid StorageFileID and in the DB --> return directly from the DB + + use_db_data = True + with contextlib.suppress(ValidationError): + file_id: StorageFileID = TypeAdapter(StorageFileID).validate_python( + f"{path}" + ) + # path is a valid StorageFileID + + if ( + dir_fmd := await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).try_get_directory(file_filter=path) + ) and dir_fmd.file_id != file_id: + # this is pure S3 aka use-case 2 + use_db_data = False + + if not use_db_data: + assert file_id # nosec + s3_metadata = await get_s3_client(self.app).get_directory_metadata( + bucket=self.simcore_bucket_name, prefix=file_id + ) + assert s3_metadata.size # nosec + return s3_metadata.size + + # all other use-cases are in the DB + fmds = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).list_filter_with_partial_file_id( + user_or_project_filter=UserOrProjectFilter( + user_id=user_id, project_ids=accessible_projects_ids + ), + file_id_prefix=f"{path}", + partial_file_id=None, + sha256_checksum=None, + is_directory=None, + ) + + # ensure file sizes are uptodate + updated_fmds = [] + for metadata in fmds: + if is_file_entry_valid(metadata): + updated_fmds.append(convert_db_to_model(metadata)) + continue + updated_fmds.append( + convert_db_to_model(await self._update_database_from_storage(metadata)) + ) + + return ByteSize(sum(fmd.file_size for fmd in updated_fmds)) + + async def list_files( self, user_id: UserID, *, @@ -163,32 +335,33 @@ async def list_files( # noqa C901 data: list[FileMetaData] = [] accessible_projects_ids = [] uid: UserID | None = None - async with self.engine.acquire() as conn: - if project_id is not None: - project_access_rights = await get_project_access_rights( - conn=conn, user_id=user_id, project_id=project_id + access_layer_repo = AccessLayerRepository.instance(get_db_engine(self.app)) + if project_id is not None: + project_access_rights = await access_layer_repo.get_project_access_rights( + user_id=user_id, project_id=project_id + ) + if not project_access_rights.read: + raise ProjectAccessRightError( + access_right="read", project_id=project_id ) - if not project_access_rights.read: - raise ProjectAccessRightError( - access_right="read", project_id=project_id - ) - accessible_projects_ids = [project_id] - uid = None - else: - accessible_projects_ids = await get_readable_project_ids(conn, user_id) - uid = user_id - file_and_directory_meta_data: list[ - FileMetaDataAtDB - ] = await db_file_meta_data.list_filter_with_partial_file_id( - conn, - user_or_project_filter=UserOrProjectFilter( - user_id=uid, project_ids=accessible_projects_ids - ), - file_id_prefix=None, - is_directory=None, - partial_file_id=uuid_filter, - sha256_checksum=None, + accessible_projects_ids = [project_id] + uid = None + else: + accessible_projects_ids = await access_layer_repo.get_readable_project_ids( + user_id=user_id ) + uid = user_id + file_and_directory_meta_data = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).list_filter_with_partial_file_id( + user_or_project_filter=UserOrProjectFilter( + user_id=uid, project_ids=accessible_projects_ids + ), + file_id_prefix=None, + is_directory=None, + partial_file_id=uuid_filter, + sha256_checksum=None, + ) # add all the entries from file_meta_data without for metadata in file_and_directory_meta_data: @@ -204,17 +377,6 @@ async def list_files( # noqa C901 updated_fmd = await self._update_database_from_storage(metadata) data.append(convert_db_to_model(updated_fmd)) - # now parse the project to search for node/project names - async with self.engine.acquire() as conn: - prj_names_mapping: dict[ProjectID | NodeID, str] = {} - async for proj_data in db_projects.list_valid_projects_in( - conn, accessible_projects_ids - ): - prj_names_mapping |= {proj_data.uuid: proj_data.name} | { - NodeID(node_id): node_data.label - for node_id, node_data in proj_data.workbench.items() - } - # expand directories until the max number of files to return is reached directory_expands: list[Coroutine] = [] for metadata in file_and_directory_meta_data: @@ -237,39 +399,33 @@ async def list_files( # noqa C901 ): data.extend(files_in_directory) - # artifically fills ['project_name', 'node_name', 'file_id', 'raw_file_path', 'display_file_path'] - # with information from the projects table! - # NOTE: This part with the projects, should be done in the client code not here! - clean_data: list[FileMetaData] = [] - for d in data: - if d.project_id not in prj_names_mapping: - continue - d.project_name = prj_names_mapping[d.project_id] - if d.node_id in prj_names_mapping: - d.node_name = prj_names_mapping[d.node_id] - if d.node_name and d.project_name: - clean_data.append(d) - - data = clean_data - return data + return await _add_frontend_needed_data( + get_db_engine(self.app), project_ids=accessible_projects_ids, data=data + ) async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaData: - async with self.engine.acquire() as conn: - can: AccessRights = await get_file_access_rights( - conn, int(user_id), file_id - ) - if not can.read: - raise FileAccessRightError(access_right="read", file_id=file_id) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.read: + raise FileAccessRightError(access_right="read", file_id=file_id) - fmd = await db_file_meta_data.get( - conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) - ) + fmd = await FileMetaDataRepository.instance(get_db_engine(self.app)).get( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id) + ) if is_file_entry_valid(fmd): return convert_db_to_model(fmd) # get file from storage if available fmd = await self._update_database_from_storage(fmd) return convert_db_to_model(fmd) + async def can_read_file(self, user_id: UserID, file_id: StorageFileID): + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.read: + raise FileAccessRightError(access_right="read", file_id=file_id) + async def create_file_upload_links( self, user_id: UserID, @@ -280,17 +436,18 @@ async def create_file_upload_links( sha256_checksum: SHA256Str | None, is_directory: bool, ) -> UploadLinks: - async with self.engine.acquire() as conn: - can: AccessRights = await get_file_access_rights(conn, user_id, file_id) - if not can.write: - raise FileAccessRightError(access_right="write", file_id=file_id) - - # NOTE: if this gets called successively with the same file_id, and - # there was a multipart upload in progress beforehand, it MUST be - # cancelled to prevent unwanted costs in AWS - await self._clean_pending_upload( - conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) - ) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.write: + raise FileAccessRightError(access_right="write", file_id=file_id) + + # NOTE: if this gets called successively with the same file_id, and + # there was a multipart upload in progress beforehand, it MUST be + # cancelled to prevent unwanted costs in AWS + await self._clean_pending_upload( + TypeAdapter(SimcoreS3FileID).validate_python(file_id) + ) if ( not is_directory @@ -305,23 +462,21 @@ async def create_file_upload_links( # SEE https://github.com/ITISFoundation/osparc-simcore/issues/5159 enforce_access_rights=False, ) - async with self.engine.acquire() as conn: - # initiate the file meta data table - fmd = await self._create_fmd_for_upload( - conn, - user_id, - file_id, - upload_id=( - S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID - if ( - get_s3_client(self.app).is_multipart(file_size_bytes) - or link_type == LinkType.S3 - ) - else None - ), - is_directory=is_directory, - sha256_checksum=sha256_checksum, - ) + # initiate the file meta data table + fmd = await self._create_fmd_for_upload( + user_id, + file_id, + upload_id=( + S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID + if ( + get_s3_client(self.app).is_multipart(file_size_bytes) + or link_type == LinkType.S3 + ) + else None + ), + is_directory=is_directory, + sha256_checksum=sha256_checksum, + ) if link_type == LinkType.PRESIGNED and get_s3_client(self.app).is_multipart( file_size_bytes @@ -334,13 +489,16 @@ async def create_file_upload_links( bucket=fmd.bucket_name, object_key=fmd.file_id, file_size=file_size_bytes, - expiration_secs=self.settings.STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, + expiration_secs=get_application_settings( + self.app + ).STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, sha256_checksum=fmd.sha256_checksum, ) # update the database so we keep the upload id fmd.upload_id = multipart_presigned_links.upload_id - async with self.engine.acquire() as conn: - await db_file_meta_data.upsert(conn, fmd) + await FileMetaDataRepository.instance(get_db_engine(self.app)).upsert( + fmd=fmd + ) return UploadLinks( multipart_presigned_links.urls, multipart_presigned_links.chunk_size, @@ -352,7 +510,9 @@ async def create_file_upload_links( ).create_single_presigned_upload_link( bucket=self.simcore_bucket_name, object_key=fmd.file_id, - expiration_secs=self.settings.STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, + expiration_secs=get_application_settings( + self.app + ).STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, ) return UploadLinks( [single_presigned_link], @@ -373,16 +533,15 @@ async def abort_file_upload( user_id: UserID, file_id: StorageFileID, ) -> None: - async with self.engine.acquire() as conn: - can: AccessRights = await get_file_access_rights( - conn, int(user_id), file_id - ) - if not can.delete or not can.write: - raise FileAccessRightError(access_right="write/delete", file_id=file_id) - - fmd: FileMetaDataAtDB = await db_file_meta_data.get( - conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) - ) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.delete or not can.write: + raise FileAccessRightError(access_right="write/delete", file_id=file_id) + + fmd = await FileMetaDataRepository.instance(get_db_engine(self.app)).get( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id) + ) if is_valid_managed_multipart_upload(fmd.upload_id): assert fmd.upload_id # nosec await get_s3_client(self.app).abort_multipart_upload( @@ -401,8 +560,9 @@ async def abort_file_upload( await self._update_database_from_storage(fmd) except S3KeyNotFoundError: # the file does not exist, so we delete the entry in the db - async with self.engine.acquire() as conn: - await db_file_meta_data.delete(conn, [fmd.file_id]) + await FileMetaDataRepository.instance(get_db_engine(self.app)).delete( + file_ids=[fmd.file_id] + ) async def complete_file_upload( self, @@ -410,15 +570,14 @@ async def complete_file_upload( user_id: UserID, uploaded_parts: list[UploadedPart], ) -> FileMetaData: - async with self.engine.acquire() as conn: - can: AccessRights = await get_file_access_rights( - conn, int(user_id), file_id - ) - if not can.write: - raise FileAccessRightError(access_right="write", file_id=file_id) - fmd = await db_file_meta_data.get( - conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) - ) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.write: + raise FileAccessRightError(access_right="write", file_id=file_id) + fmd = await FileMetaDataRepository.instance(get_db_engine(self.app)).get( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id) + ) if is_valid_managed_multipart_upload(fmd.upload_id): # NOTE: Processing of a Complete Multipart Upload request @@ -452,44 +611,41 @@ async def create_file_download_link( 3. Raises FileNotFoundError if the file does not exist 4. Raises FileAccessRightError if the user does not have access to the file """ - async with self.engine.acquire() as conn: - directory_file_id: SimcoreS3FileID | None = await get_directory_file_id( - conn, cast(SimcoreS3FileID, file_id) - ) - await self.__ensure_read_access_rights( - conn, user_id, directory_file_id if directory_file_id else file_id + directory_file_id = await get_directory_file_id( + get_db_engine(self.app), cast(SimcoreS3FileID, file_id) + ) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights( + user_id=user_id, file_id=directory_file_id if directory_file_id else file_id + ) + if not can.read: + # NOTE: this is tricky. A user with read access can download and data! + # If write permission would be required, then shared projects as views cannot + # recover data in nodes (e.g. jupyter cannot pull work data) + # + raise FileAccessRightError( + access_right="read", + file_id=directory_file_id if directory_file_id else file_id, ) if directory_file_id: if not await get_s3_client(self.app).object_exists( bucket=self.simcore_bucket_name, object_key=f"{file_id}" ): raise S3KeyNotFoundError(key=file_id, bucket=self.simcore_bucket_name) - return await self.__get_link( + return await self._get_link( TypeAdapter(SimcoreS3FileID).validate_python(file_id), link_type ) # standard file link - async with self.engine.acquire() as conn: - fmd = await db_file_meta_data.get( - conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) - ) + fmd = await FileMetaDataRepository.instance(get_db_engine(self.app)).get( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id) + ) if not is_file_entry_valid(fmd): # try lazy update fmd = await self._update_database_from_storage(fmd) - return await self.__get_link(fmd.object_name, link_type) + return await self._get_link(fmd.object_name, link_type) - @staticmethod - async def __ensure_read_access_rights( - conn: SAConnection, user_id: UserID, storage_file_id: StorageFileID - ) -> None: - can: AccessRights = await get_file_access_rights(conn, user_id, storage_file_id) - if not can.read: - # NOTE: this is tricky. A user with read access can download and data! - # If write permission would be required, then shared projects as views cannot - # recover data in nodes (e.g. jupyter cannot pull work data) - # - raise FileAccessRightError(access_right="read", file_id=storage_file_id) - - async def __get_link( + async def _get_link( self, s3_file_id: SimcoreS3FileID, link_type: LinkType ) -> AnyUrl: link: AnyUrl = TypeAdapter(AnyUrl).validate_python( @@ -499,7 +655,9 @@ async def __get_link( link = await get_s3_client(self.app).create_single_presigned_download_link( bucket=self.simcore_bucket_name, object_key=s3_file_id, - expiration_secs=self.settings.STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, + expiration_secs=get_application_settings( + self.app + ).STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS, ) return link @@ -522,11 +680,13 @@ async def delete_file( # Only use this in those circumstances where a collaborator requires to delete a file (the current # permissions model will not allow him to do so, even though this is a legitimate action) # SEE https://github.com/ITISFoundation/osparc-simcore/issues/5159 - async with self.engine.acquire() as conn: - if enforce_access_rights: - can: AccessRights = await get_file_access_rights(conn, user_id, file_id) - if not can.delete: - raise FileAccessRightError(access_right="delete", file_id=file_id) + + if enforce_access_rights: + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_file_access_rights(user_id=user_id, file_id=file_id) + if not can.delete: + raise FileAccessRightError(access_right="delete", file_id=file_id) try: await get_s3_client(self.app).delete_objects_recursively( @@ -538,11 +698,14 @@ async def delete_file( # we still need to clean up the database entry (it exists) # and to invalidate the size of the parent directory - async with self.engine.acquire() as conn: - await db_file_meta_data.delete(conn, [file_id]) + async with transaction_context(get_db_engine(self.app)) as connection: + file_meta_data_repo = FileMetaDataRepository.instance( + get_db_engine(self.app) + ) + await file_meta_data_repo.delete(connection=connection, file_ids=[file_id]) - if parent_dir_fmds := await db_file_meta_data.list_filter_with_partial_file_id( - conn, + if parent_dir_fmds := await file_meta_data_repo.list_filter_with_partial_file_id( + connection=connection, user_or_project_filter=UserOrProjectFilter( user_id=user_id, project_ids=[] ), @@ -553,25 +716,27 @@ async def delete_file( ): parent_dir_fmd = max(parent_dir_fmds, key=lambda fmd: len(fmd.file_id)) parent_dir_fmd.file_size = UNDEFINED_SIZE - await db_file_meta_data.upsert(conn, parent_dir_fmd) + await file_meta_data_repo.upsert( + connection=connection, fmd=parent_dir_fmd + ) async def delete_project_simcore_s3( self, user_id: UserID, project_id: ProjectID, node_id: NodeID | None = None ) -> None: - async with self.engine.acquire() as conn: - can: AccessRights = await get_project_access_rights( - conn, user_id, project_id - ) - if not can.delete: - raise ProjectAccessRightError( - access_right="delete", project_id=project_id - ) - - # we can do it this way, since we are in a transaction, it will rollback in case of error - if not node_id: - await db_file_meta_data.delete_all_from_project(conn, project_id) - else: - await db_file_meta_data.delete_all_from_node(conn, node_id) + can = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_project_access_rights(user_id=user_id, project_id=project_id) + if not can.delete: + raise ProjectAccessRightError(access_right="delete", project_id=project_id) + + if not node_id: + await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).delete_all_from_project(project_id=project_id) + else: + await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).delete_all_from_node(node_id=node_id) await get_s3_client(self.app).delete_objects_recursively( bucket=self.simcore_bucket_name, @@ -597,16 +762,18 @@ async def deep_copy_project_simcore_s3( "Step 1: check access rights (read of src and write of dst)", ): update_task_progress(task_progress, "Checking study access rights...") - async with self.engine.acquire() as conn: - for prj_uuid in [src_project_uuid, dst_project_uuid]: - if not await db_projects.project_exists(conn, prj_uuid): - raise ProjectNotFoundError(project_id=prj_uuid) - source_access_rights = await get_project_access_rights( - conn, user_id, project_id=src_project_uuid - ) - dest_access_rights = await get_project_access_rights( - conn, user_id, project_id=dst_project_uuid - ) + + for prj_uuid in [src_project_uuid, dst_project_uuid]: + if not await ProjectRepository.instance( + get_db_engine(self.app) + ).project_exists(project_uuid=prj_uuid): + raise ProjectNotFoundError(project_id=prj_uuid) + source_access_rights = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_project_access_rights(user_id=user_id, project_id=src_project_uuid) + dest_access_rights = await AccessLayerRepository.instance( + get_db_engine(self.app) + ).get_project_access_rights(user_id=user_id, project_id=dst_project_uuid) if not source_access_rights.read: raise ProjectAccessRightError( access_right="read", project_id=src_project_uuid @@ -625,12 +792,10 @@ async def deep_copy_project_simcore_s3( update_task_progress( task_progress, f"Collecting files of '{src_project['name']}'..." ) - async with self.engine.acquire() as conn: - src_project_files: list[ - FileMetaDataAtDB - ] = await db_file_meta_data.list_fmds( - conn, project_ids=[src_project_uuid] - ) + + src_project_files = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).list_fmds(project_ids=[src_project_uuid]) with log_context( _logger, @@ -743,21 +908,17 @@ async def search_owned_files( limit: int | None = None, offset: int | None = None, ) -> list[FileMetaData]: - async with self.engine.acquire() as conn: - file_metadatas: list[ - FileMetaDataAtDB - ] = await db_file_meta_data.list_filter_with_partial_file_id( - conn, - user_or_project_filter=UserOrProjectFilter( - user_id=user_id, project_ids=[] - ), - file_id_prefix=file_id_prefix, - partial_file_id=None, - is_directory=False, - sha256_checksum=sha256_checksum, - limit=limit, - offset=offset, - ) + file_metadatas = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).list_filter_with_partial_file_id( + user_or_project_filter=UserOrProjectFilter(user_id=user_id, project_ids=[]), + file_id_prefix=file_id_prefix, + partial_file_id=None, + is_directory=False, + sha256_checksum=sha256_checksum, + limit=limit, + offset=offset, + ) resolved_fmds = [] for fmd in file_metadatas: if is_file_entry_valid(fmd): @@ -771,11 +932,11 @@ async def search_owned_files( async def create_soft_link( self, user_id: int, target_file_id: StorageFileID, link_file_id: StorageFileID ) -> FileMetaData: - async with self.engine.acquire() as conn: - if await db_file_meta_data.exists( - conn, TypeAdapter(SimcoreS3FileID).validate_python(link_file_id) - ): - raise LinkAlreadyExistsError(file_id=link_file_id) + file_meta_data_repo = FileMetaDataRepository.instance(get_db_engine(self.app)) + if await file_meta_data_repo.exists( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(link_file_id) + ): + raise LinkAlreadyExistsError(file_id=link_file_id) # validate target_uuid target = await self.get_file(user_id, target_file_id) # duplicate target and change the following columns: @@ -783,43 +944,13 @@ async def create_soft_link( target.file_id = link_file_id # NOTE: api-server relies on this id target.is_soft_link = True - async with self.engine.acquire() as conn: - return convert_db_to_model(await db_file_meta_data.insert(conn, target)) - - async def synchronise_meta_data_table( - self, *, dry_run: bool - ) -> list[StorageFileID]: + return convert_db_to_model(await file_meta_data_repo.insert(fmd=target)) - async with self.engine.acquire() as conn: - _logger.warning( - "Total number of entries to check %d", - await db_file_meta_data.total(conn), - ) - # iterate over all entries to check if there is a file in the S3 backend - file_ids_to_remove = [ - fmd.file_id - async for fmd in db_file_meta_data.list_valid_uploads(conn) - if not await get_s3_client(self.app).object_exists( - bucket=self.simcore_bucket_name, object_key=fmd.object_name - ) - ] - - if not dry_run: - await db_file_meta_data.delete(conn, file_ids_to_remove) - - _logger.info( - "%s %d entries ", - "Would delete" if dry_run else "Deleted", - len(file_ids_to_remove), - ) - - return cast(list[StorageFileID], file_ids_to_remove) - - async def _clean_pending_upload( - self, conn: SAConnection, file_id: SimcoreS3FileID - ) -> None: + async def _clean_pending_upload(self, file_id: SimcoreS3FileID) -> None: with suppress(FileMetaDataNotFoundError): - fmd = await db_file_meta_data.get(conn, file_id) + fmd = await FileMetaDataRepository.instance(get_db_engine(self.app)).get( + file_id=file_id + ) if is_valid_managed_multipart_upload(fmd.upload_id): assert fmd.upload_id # nosec await get_s3_client(self.app).abort_multipart_upload( @@ -834,11 +965,11 @@ async def _clean_expired_uploads(self) -> None: 1. will try to update the entry from S3 backend if exists 2. will delete the entry if nothing exists in S3 backend. """ - now = arrow.utcnow().datetime - async with self.engine.acquire() as conn: - list_of_expired_uploads = await db_file_meta_data.list_fmds( - conn, expired_after=now - ) + now = datetime.datetime.utcnow() + + list_of_expired_uploads = await FileMetaDataRepository.instance( + get_db_engine(self.app) + ).list_fmds(expired_after=now) if not list_of_expired_uploads: return @@ -916,7 +1047,10 @@ async def clean_expired_uploads(self) -> None: await self._clean_expired_uploads() async def _update_fmd_from_other( - self, conn: SAConnection, *, fmd: FileMetaDataAtDB, copy_from: FileMetaDataAtDB + self, + *, + fmd: FileMetaDataAtDB, + copy_from: FileMetaDataAtDB, ) -> FileMetaDataAtDB: if not fmd.is_directory: s3_metadata = await get_s3_client(self.app).get_object_metadata( @@ -931,10 +1065,10 @@ async def _update_fmd_from_other( fmd.upload_expires_at = None fmd.upload_id = None - updated_fmd: FileMetaDataAtDB = await db_file_meta_data.upsert( - conn, convert_db_to_model(fmd) + + return await FileMetaDataRepository.instance(get_db_engine(self.app)).upsert( + fmd=convert_db_to_model(fmd) ) - return updated_fmd async def _get_s3_metadata( self, fmd: FileMetaDataAtDB @@ -967,11 +1101,10 @@ async def _update_database_from_storage( fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.upload_expires_at = None fmd.upload_id = None - async with self.engine.acquire() as conn: - updated_fmd: FileMetaDataAtDB = await db_file_meta_data.upsert( - conn, convert_db_to_model(fmd) - ) - return updated_fmd + + return await FileMetaDataRepository.instance(get_db_engine(self.app)).upsert( + fmd=convert_db_to_model(fmd) + ) async def _copy_file_datcore_s3( self, @@ -984,9 +1117,11 @@ async def _copy_file_datcore_s3( ) -> FileMetaData: session = get_client_session(self.app) # 2 steps: Get download link for local copy, then upload to S3 - api_token, api_secret = await db_tokens.get_api_token_and_secret( - self.app, user_id - ) + api_token, api_secret = await TokenRepository.instance( + get_db_engine(self.app) + ).get_api_token_and_secret(user_id=user_id) + assert api_token # nosec + assert api_secret # nosec dc_link = await datcore_adapter.get_file_download_presigned_link( self.app, api_token, api_secret, source_uuid ) @@ -1003,15 +1138,13 @@ async def _copy_file_datcore_s3( await download_to_file_or_raise(session, f"{dc_link}", local_file_path) # copying will happen using aioboto3, therefore multipart might happen - async with self.engine.acquire() as conn: - new_fmd = await self._create_fmd_for_upload( - conn, - user_id, - dst_file_id, - upload_id=S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID, - is_directory=False, - sha256_checksum=None, - ) + new_fmd = await self._create_fmd_for_upload( + user_id, + dst_file_id, + upload_id=S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID, + is_directory=False, + sha256_checksum=None, + ) # Uploads local -> S3 await get_s3_client(self.app).upload_file( bucket=self.simcore_bucket_name, @@ -1042,15 +1175,13 @@ async def _copy_path_s3_s3( ): # copying will happen using aioboto3, therefore multipart might happen # NOTE: connection must be released to ensure database update - async with self.engine.acquire() as conn: - new_fmd = await self._create_fmd_for_upload( - conn, - user_id, - dst_file_id, - upload_id=S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID, - is_directory=src_fmd.is_directory, - sha256_checksum=src_fmd.sha256_checksum, - ) + new_fmd = await self._create_fmd_for_upload( + user_id, + dst_file_id, + upload_id=S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID, + is_directory=src_fmd.is_directory, + sha256_checksum=src_fmd.sha256_checksum, + ) s3_client = get_s3_client(self.app) @@ -1069,15 +1200,13 @@ async def _copy_path_s3_s3( bytes_transfered_cb=bytes_transfered_cb, ) # we are done, let's update the copy with the src - async with self.engine.acquire() as conn: - updated_fmd = await self._update_fmd_from_other( - conn, fmd=new_fmd, copy_from=src_fmd - ) + updated_fmd = await self._update_fmd_from_other( + fmd=new_fmd, copy_from=src_fmd + ) return convert_db_to_model(updated_fmd) async def _create_fmd_for_upload( self, - conn: SAConnection, user_id: UserID, file_id: StorageFileID, upload_id: UploadID | None, @@ -1085,9 +1214,11 @@ async def _create_fmd_for_upload( is_directory: bool, sha256_checksum: SHA256Str | None, ) -> FileMetaDataAtDB: - now = arrow.utcnow().datetime + now = datetime.datetime.utcnow() upload_expiration_date = now + datetime.timedelta( - seconds=self.settings.STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS + seconds=get_application_settings( + self.app + ).STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS ) fmd = FileMetaData.from_simcore_node( user_id=user_id, @@ -1100,17 +1231,18 @@ async def _create_fmd_for_upload( is_directory=is_directory, sha256_checksum=sha256_checksum, ) - return await db_file_meta_data.upsert(conn, fmd) + + return await FileMetaDataRepository.instance(get_db_engine(self.app)).upsert( + fmd=fmd + ) -def create_simcore_s3_data_manager(app: web.Application) -> SimcoreS3DataManager: - cfg: Settings = app[APP_CONFIG_KEY] +def create_simcore_s3_data_manager(app: FastAPI) -> SimcoreS3DataManager: + cfg = get_application_settings(app) assert cfg.STORAGE_S3 # nosec return SimcoreS3DataManager( - engine=app[APP_AIOPG_ENGINE_KEY], simcore_bucket_name=TypeAdapter(S3BucketName).validate_python( cfg.STORAGE_S3.S3_BUCKET_NAME ), app=app, - settings=cfg, ) diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py deleted file mode 100644 index eb5f2f1240c..00000000000 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py +++ /dev/null @@ -1,126 +0,0 @@ -from contextlib import suppress -from pathlib import Path - -from aiopg.sa.connection import SAConnection -from aws_library.s3 import S3MetaData, SimcoreS3API -from models_library.api_schemas_storage import S3BucketName -from models_library.projects_nodes_io import ( - SimcoreS3DirectoryID, - SimcoreS3FileID, - StorageFileID, -) -from pydantic import ByteSize, NonNegativeInt, TypeAdapter -from servicelib.utils import ensure_ends_with - -from . import db_file_meta_data -from .exceptions import FileMetaDataNotFoundError -from .models import FileMetaData, FileMetaDataAtDB -from .utils import convert_db_to_model - - -async def _list_all_files_in_folder( - *, - s3_client: SimcoreS3API, - bucket: S3BucketName, - prefix: str, - max_files_to_list: int, -) -> list[S3MetaData]: - async for s3_objects in s3_client.list_objects_paginated( - bucket, prefix, items_per_page=max_files_to_list - ): - # NOTE: stop immediately after listing after `max_files_to_list` - return s3_objects - return [] - - -async def expand_directory( - s3_client: SimcoreS3API, - simcore_bucket_name: S3BucketName, - fmd: FileMetaDataAtDB, - max_items_to_include: NonNegativeInt, -) -> list[FileMetaData]: - """ - Scans S3 backend and returns a list S3MetaData entries which get mapped - to FileMetaData entry. - """ - files_in_folder: list[S3MetaData] = await _list_all_files_in_folder( - s3_client=s3_client, - bucket=simcore_bucket_name, - prefix=ensure_ends_with(fmd.file_id, "/"), - max_files_to_list=max_items_to_include, - ) - result: list[FileMetaData] = [ - convert_db_to_model( - FileMetaDataAtDB( - location_id=fmd.location_id, - location=fmd.location, - bucket_name=fmd.bucket_name, - object_name=x.object_key, - user_id=fmd.user_id, - # NOTE: to ensure users have a consistent experience the - # `created_at` field is inherited from the last_modified - # coming from S3. This way if a file is created 1 month after the - # creation of the directory, the file's creation date - # will not be 1 month in the passed. - created_at=x.last_modified, - file_id=x.object_key, - file_size=TypeAdapter(ByteSize).validate_python(x.size), - last_modified=x.last_modified, - entity_tag=x.e_tag, - is_soft_link=False, - is_directory=False, - project_id=fmd.project_id, - node_id=fmd.node_id, - ) - ) - for x in files_in_folder - ] - return result - - -def get_simcore_directory(file_id: SimcoreS3FileID) -> str: - try: - directory_id = SimcoreS3DirectoryID.from_simcore_s3_object(file_id) - except ValueError: - return "" - return f"{Path(directory_id)}" - - -async def get_directory_file_id( - conn: SAConnection, file_id: SimcoreS3FileID -) -> SimcoreS3FileID | None: - """ - returns the containing file's `directory_file_id` if the entry exists - in the `file_meta_data` table - """ - - async def _get_fmd( - conn: SAConnection, s3_file_id: StorageFileID - ) -> FileMetaDataAtDB | None: - with suppress(FileMetaDataNotFoundError): - return await db_file_meta_data.get( - conn, TypeAdapter(SimcoreS3FileID).validate_python(s3_file_id) - ) - return None - - provided_file_id_fmd = await _get_fmd(conn, file_id) - if provided_file_id_fmd: - # file_meta_data exists it is not a directory - return None - - directory_file_id_str: str = get_simcore_directory(file_id) - if directory_file_id_str == "": - # could not extract a directory name from the provided path - return None - - directory_file_id = TypeAdapter(SimcoreS3FileID).validate_python( - directory_file_id_str - ) - directory_file_id_fmd = await _get_fmd(conn, directory_file_id) - - return directory_file_id if directory_file_id_fmd else None - - -def compute_file_id_prefix(file_id: str, levels: int): - components = file_id.strip("/").split("/") - return "/".join(components[:levels]) diff --git a/services/storage/src/simcore_service_storage/utils/__init__.py b/services/storage/src/simcore_service_storage/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/s3_utils.py b/services/storage/src/simcore_service_storage/utils/s3_utils.py similarity index 100% rename from services/storage/src/simcore_service_storage/s3_utils.py rename to services/storage/src/simcore_service_storage/utils/s3_utils.py diff --git a/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py b/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py new file mode 100644 index 00000000000..a50bdbff1ed --- /dev/null +++ b/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py @@ -0,0 +1,225 @@ +from contextlib import suppress +from pathlib import Path + +import orjson +from aws_library.s3 import S3MetaData, SimcoreS3API +from models_library.api_schemas_storage.storage_schemas import S3BucketName +from models_library.projects import ProjectID +from models_library.projects_nodes_io import ( + SimcoreS3DirectoryID, + SimcoreS3FileID, + StorageFileID, +) +from models_library.users import UserID +from pydantic import ByteSize, NonNegativeInt, TypeAdapter +from servicelib.utils import ensure_ends_with +from sqlalchemy.ext.asyncio import AsyncEngine + +from ..exceptions.errors import FileMetaDataNotFoundError, ProjectAccessRightError +from ..models import FileMetaData, FileMetaDataAtDB, GenericCursor, PathMetaData +from ..modules.db.access_layer import AccessLayerRepository +from ..modules.db.file_meta_data import FileMetaDataRepository, TotalChildren +from .utils import convert_db_to_model + + +async def _list_all_files_in_folder( + *, + s3_client: SimcoreS3API, + bucket: S3BucketName, + prefix: str, + max_files_to_list: int, +) -> list[S3MetaData]: + async for s3_objects in s3_client.list_objects_paginated( + bucket, prefix, items_per_page=max_files_to_list + ): + # NOTE: stop immediately after listing after `max_files_to_list` + return s3_objects + return [] + + +async def expand_directory( + s3_client: SimcoreS3API, + simcore_bucket_name: S3BucketName, + fmd: FileMetaDataAtDB, + max_items_to_include: NonNegativeInt, +) -> list[FileMetaData]: + """ + Scans S3 backend and returns a list S3MetaData entries which get mapped + to FileMetaData entry. + """ + files_in_folder: list[S3MetaData] = await _list_all_files_in_folder( + s3_client=s3_client, + bucket=simcore_bucket_name, + prefix=ensure_ends_with(fmd.file_id, "/"), + max_files_to_list=max_items_to_include, + ) + result: list[FileMetaData] = [ + convert_db_to_model( + FileMetaDataAtDB( + location_id=fmd.location_id, + location=fmd.location, + bucket_name=fmd.bucket_name, + object_name=x.object_key, + user_id=fmd.user_id, + # NOTE: to ensure users have a consistent experience the + # `created_at` field is inherited from the last_modified + # coming from S3. This way if a file is created 1 month after the + # creation of the directory, the file's creation date + # will not be 1 month in the passed. + created_at=x.last_modified, + file_id=x.object_key, + file_size=TypeAdapter(ByteSize).validate_python(x.size), + last_modified=x.last_modified, + entity_tag=x.e_tag, + is_soft_link=False, + is_directory=False, + project_id=fmd.project_id, + node_id=fmd.node_id, + ) + ) + for x in files_in_folder + ] + return result + + +def get_simcore_directory(file_id: SimcoreS3FileID) -> str: + try: + directory_id = SimcoreS3DirectoryID.from_simcore_s3_object(file_id) + except ValueError: + return "" + return f"{Path(directory_id)}" + + +async def _try_get_fmd( + db_engine: AsyncEngine, s3_file_id: StorageFileID +) -> FileMetaDataAtDB | None: + with suppress(FileMetaDataNotFoundError): + return await FileMetaDataRepository.instance(db_engine).get( + file_id=TypeAdapter(SimcoreS3FileID).validate_python(s3_file_id) + ) + return None + + +async def get_directory_file_id( + db_engine: AsyncEngine, file_id: SimcoreS3FileID +) -> SimcoreS3FileID | None: + """ + returns the containing file's `directory_file_id` if the entry exists + in the `file_meta_data` table + """ + + provided_file_id_fmd = await _try_get_fmd(db_engine, file_id) + if provided_file_id_fmd: + # file_meta_data exists it is not a directory + return None + + directory_file_id_str: str = get_simcore_directory(file_id) + if directory_file_id_str == "": + # could not extract a directory name from the provided path + return None + + directory_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + directory_file_id_str + ) + directory_file_id_fmd = await _try_get_fmd(db_engine, directory_file_id) + + return directory_file_id if directory_file_id_fmd else None + + +def compute_file_id_prefix(file_id: str, levels: int): + components = file_id.strip("/").split("/") + return "/".join(components[:levels]) + + +async def list_child_paths_from_s3( + s3_client: SimcoreS3API, + *, + dir_fmd: FileMetaData, + bucket: S3BucketName, + file_filter: Path, + limit: int, + cursor: GenericCursor | None, +) -> tuple[list[PathMetaData], GenericCursor | None]: + """list direct children given by `file_filter` of a directory. + Tries first using file_filter as a full path, if not results are found will + try using file_filter as a partial prefix. + """ + objects_cursor = None + if cursor is not None: + cursor_params = orjson.loads(cursor) + assert cursor_params["file_filter"] == f"{file_filter}" # nosec + objects_cursor = cursor_params["objects_next_cursor"] + list_s3_objects, objects_next_cursor = await s3_client.list_objects( + bucket=bucket, + prefix=file_filter, + start_after=None, + limit=limit, + next_cursor=objects_cursor, + is_partial_prefix=False, + ) + if not list_s3_objects: + list_s3_objects, objects_next_cursor = await s3_client.list_objects( + bucket=bucket, + prefix=file_filter, + start_after=None, + limit=limit, + next_cursor=objects_cursor, + is_partial_prefix=True, + ) + + paths_metadata = [ + PathMetaData.from_s3_object_in_dir(s3_object, dir_fmd) + for s3_object in list_s3_objects + ] + next_cursor = None + if objects_next_cursor: + next_cursor = orjson.dumps( + { + "file_filter": f"{file_filter}", + "objects_next_cursor": objects_next_cursor, + } + ) + + return paths_metadata, next_cursor + + +async def list_child_paths_from_repository( + db_engine: AsyncEngine, + *, + filter_by_project_ids: list[ProjectID] | None, + filter_by_file_prefix: Path | None, + cursor: GenericCursor | None, + limit: int, +) -> tuple[list[PathMetaData], GenericCursor | None, TotalChildren]: + file_meta_data_repo = FileMetaDataRepository.instance(db_engine) + paths_metadata, next_cursor, total = await file_meta_data_repo.list_child_paths( + filter_by_project_ids=filter_by_project_ids, + filter_by_file_prefix=filter_by_file_prefix, + limit=limit, + cursor=cursor, + is_partial_prefix=False, + ) + if not paths_metadata: + paths_metadata, next_cursor, total = await file_meta_data_repo.list_child_paths( + filter_by_project_ids=filter_by_project_ids, + filter_by_file_prefix=filter_by_file_prefix, + limit=limit, + cursor=cursor, + is_partial_prefix=True, + ) + + return paths_metadata, next_cursor, total + + +async def get_accessible_project_ids( + db_engine: AsyncEngine, *, user_id: UserID, project_id: ProjectID | None +) -> list[ProjectID]: + access_layer_repo = AccessLayerRepository.instance(db_engine) + if project_id: + project_access_rights = await access_layer_repo.get_project_access_rights( + user_id=user_id, project_id=project_id + ) + if not project_access_rights.read: + raise ProjectAccessRightError(access_right="read", project_id=project_id) + return [project_id] + return await access_layer_repo.get_readable_project_ids(user_id=user_id) diff --git a/services/storage/src/simcore_service_storage/utils.py b/services/storage/src/simcore_service_storage/utils/utils.py similarity index 73% rename from services/storage/src/simcore_service_storage/utils.py rename to services/storage/src/simcore_service_storage/utils/utils.py index 7abc18ed552..36fef50d268 100644 --- a/services/storage/src/simcore_service_storage/utils.py +++ b/services/storage/src/simcore_service_storage/utils/utils.py @@ -1,15 +1,16 @@ +import hashlib import logging -import urllib.parse from pathlib import Path import aiofiles -from aiohttp import ClientSession +import httpx from aiohttp.typedefs import StrOrURL +from aws_library.s3 import UploadID from models_library.projects_nodes_io import StorageFileID from models_library.users import UserID -from .constants import MAX_CHUNK_SIZE, S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID -from .models import FileMetaData, FileMetaDataAtDB, UploadID +from ..constants import MAX_CHUNK_SIZE, S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID +from ..models import FileMetaData, FileMetaDataAtDB logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def convert_db_to_model(x: FileMetaDataAtDB) -> FileMetaData: async def download_to_file_or_raise( - session: ClientSession, + session: httpx.AsyncClient, url: StrOrURL, destination_path: Path, *, @@ -46,12 +47,13 @@ async def download_to_file_or_raise( dest_file = Path(destination_path) total_size = 0 - async with session.get(url, raise_for_status=True) as response: - dest_file.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(dest_file, mode="wb") as fh: - async for chunk in response.content.iter_chunked(chunk_size): - await fh.write(chunk) - total_size += len(chunk) + response = await session.get(f"{url}") + response.raise_for_status() + dest_file.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(dest_file, mode="wb") as fh: + async for chunk in response.aiter_bytes(chunk_size): + await fh.write(chunk) + total_size += len(chunk) return total_size @@ -67,7 +69,8 @@ def is_file_entry_valid(file_metadata: FileMetaData | FileMetaDataAtDB) -> bool: def create_upload_completion_task_name(user_id: UserID, file_id: StorageFileID) -> str: - return f"upload_complete_task_{user_id}_{urllib.parse.quote(file_id, safe='')}" + the_hash = hashlib.sha256(f"{user_id}_{file_id}".encode()).hexdigest() + return f"upload_complete_task_{the_hash}" def is_valid_managed_multipart_upload(upload_id: UploadID | None) -> bool: diff --git a/services/storage/src/simcore_service_storage/utils_handlers.py b/services/storage/src/simcore_service_storage/utils_handlers.py deleted file mode 100644 index e0438cc0c92..00000000000 --- a/services/storage/src/simcore_service_storage/utils_handlers.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging - -from aiohttp import web -from aiohttp.typedefs import Handler -from aiohttp.web_request import Request -from aws_library.s3 import S3AccessError, S3KeyNotFoundError -from pydantic import ValidationError -from servicelib.aiohttp.aiopg_utils import DBAPIError - -from .datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterTimeoutError -from .db_access_layer import InvalidFileIdentifierError -from .exceptions import ( - FileAccessRightError, - FileMetaDataNotFoundError, - LinkAlreadyExistsError, - ProjectAccessRightError, - ProjectNotFoundError, -) - -_logger = logging.getLogger(__name__) - - -@web.middleware -async def dsm_exception_handler( - request: Request, handler: Handler -) -> web.StreamResponse: - try: - return await handler(request) - except InvalidFileIdentifierError as err: - raise web.HTTPUnprocessableEntity( - reason=f"{err} is an invalid file identifier" - ) from err - except (FileMetaDataNotFoundError, S3KeyNotFoundError, ProjectNotFoundError) as err: - raise web.HTTPNotFound(reason=f"{err}") from err - except (FileAccessRightError, ProjectAccessRightError) as err: - raise web.HTTPForbidden(reason=f"{err}") from err - except LinkAlreadyExistsError as err: - raise web.HTTPUnprocessableEntity(reason=f"{err}") from err - except ValidationError as err: - raise web.HTTPUnprocessableEntity(reason=f"{err}") from err - except DBAPIError as err: - _logger.exception("Unexpected error while accessing DB:") - raise web.HTTPServiceUnavailable( - reason=f"Unexpected error while accessing the database: {err}" - ) from err - except S3AccessError as err: - _logger.exception("Unexpected error while accessing S3:") - raise web.HTTPServiceUnavailable( - reason=f"Unexpected error while accessing S3 backend: {err}" - ) from err - except DatcoreAdapterTimeoutError as err: - _logger.exception("Unexpected error while accessing Datcore-Adapter:") - raise web.HTTPGatewayTimeout(reason=f"{err}") from err diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 8dc4a95b488..03c73f5bfd0 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -8,67 +8,82 @@ import asyncio import logging +import random import sys -import urllib.parse from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager from pathlib import Path -from typing import cast +from typing import Any, Final, cast -import aioresponses -import dotenv +import httpx import pytest +import respx import simcore_service_storage -from aiohttp.test_utils import TestClient -from aiopg.sa import Engine +from asgi_lifespan import LifespanManager from aws_library.s3 import SimcoreS3API from faker import Faker from fakeredis.aioredis import FakeRedis -from models_library.api_schemas_storage import ( +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( FileMetaDataGet, FileUploadCompleteFutureResponse, FileUploadCompleteResponse, FileUploadCompleteState, FileUploadCompletionBody, FileUploadSchema, + LinkType, UploadedPart, ) from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes import NodeID -from models_library.projects_nodes_io import LocationID, SimcoreS3FileID +from models_library.projects_nodes_io import LocationID, SimcoreS3FileID, StorageFileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context -from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.s3 import upload_file_to_presigned_link +from pytest_simcore.helpers.storage_utils import ( + FileIDDict, + ProjectWithFilesParams, + get_updated_project, +) +from pytest_simcore.helpers.storage_utils_file_meta_data import ( + assert_file_meta_data_in_db, +) from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status -from settings_library.s3 import S3Settings +from servicelib.utils import limited_gather +from settings_library.rabbit import RabbitSettings +from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users -from simcore_service_storage.application import create +from simcore_service_storage.core.application import create_app +from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.datcore_dsm import DatCoreDataManager from simcore_service_storage.dsm import get_dsm_provider -from simcore_service_storage.handlers_files import UPLOAD_TASKS_KEY -from simcore_service_storage.models import S3BucketName -from simcore_service_storage.s3 import get_s3_client -from simcore_service_storage.settings import Settings +from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB, S3BucketName +from simcore_service_storage.modules.long_running_tasks import ( + get_completed_upload_tasks, +) +from simcore_service_storage.modules.s3 import get_s3_client from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy import literal_column +from sqlalchemy.ext.asyncio import AsyncEngine from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db from types_aiobotocore_s3 import S3Client from yarl import URL pytest_plugins = [ - "pytest_simcore.aioresponses_mocker", "pytest_simcore.aws_s3_service", "pytest_simcore.aws_server", "pytest_simcore.cli_runner", + "pytest_simcore.disk_usage_monitoring", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", @@ -78,9 +93,11 @@ "pytest_simcore.openapi_specs", "pytest_simcore.postgres_service", "pytest_simcore.pytest_global_environs", + "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", - "tests.fixtures.data_models", - "tests.fixtures.datcore_adapter", + "pytest_simcore.simcore_storage_data_models", + "pytest_simcore.simcore_storage_datcore_adapter", + "pytest_simcore.simcore_storage_service", ] CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -109,82 +126,79 @@ def project_slug_dir(osparc_simcore_root_dir: Path) -> Path: return service_folder -@pytest.fixture(scope="session") -def project_env_devel_dict(project_slug_dir: Path) -> dict[str, str | None]: - env_devel_file = project_slug_dir / ".env-devel" - assert env_devel_file.exists() - return dotenv.dotenv_values(env_devel_file, verbose=True, interpolate=True) - - -@pytest.fixture -def project_env_devel_environment( - project_env_devel_dict: dict[str, str], monkeypatch: pytest.MonkeyPatch -) -> None: - for key, value in project_env_devel_dict.items(): - monkeypatch.setenv(key, value) - - -## FAKE DATA FIXTURES ---------------------------------------------- - - @pytest.fixture -async def cleanup_user_projects_file_metadata(aiopg_engine: Engine): +async def cleanup_user_projects_file_metadata(sqlalchemy_async_engine: AsyncEngine): yield # cleanup - async with aiopg_engine.acquire() as conn: + async with sqlalchemy_async_engine.begin() as conn: await conn.execute(file_meta_data.delete()) await conn.execute(projects.delete()) await conn.execute(users.delete()) @pytest.fixture -def simcore_s3_dsm(client: TestClient) -> SimcoreS3DataManager: - assert client.app +def simcore_s3_dsm(initialized_app: FastAPI) -> SimcoreS3DataManager: return cast( SimcoreS3DataManager, - get_dsm_provider(client.app).get(SimcoreS3DataManager.get_location_id()), + get_dsm_provider(initialized_app).get(SimcoreS3DataManager.get_location_id()), ) @pytest.fixture -async def storage_s3_client( - client: TestClient, -) -> SimcoreS3API: - assert client.app - return get_s3_client(client.app) +async def storage_s3_client(initialized_app: FastAPI) -> SimcoreS3API: + return get_s3_client(initialized_app) @pytest.fixture -async def storage_s3_bucket(app_settings: Settings) -> str: +async def storage_s3_bucket(app_settings: ApplicationSettings) -> str: assert app_settings.STORAGE_S3 return app_settings.STORAGE_S3.S3_BUCKET_NAME +@pytest.fixture +async def mock_rabbit_setup(mocker: MockerFixture) -> MockerFixture: + mocker.patch("simcore_service_storage.core.application.setup_rabbitmq") + mocker.patch("simcore_service_storage.core.application.setup_rpc_api_routes") + return mocker + + +@pytest.fixture +def app_environment( + mock_env_devel_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + external_envfile_dict: EnvVarsDict, +) -> EnvVarsDict: + if external_envfile_dict: + delenvs_from_dict(monkeypatch, mock_env_devel_environment, raising=False) + return setenvs_from_dict(monkeypatch, {**external_envfile_dict}) + + envs = setenvs_from_dict(monkeypatch, {}) + return mock_env_devel_environment | envs + + +@pytest.fixture +def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("STORAGE_RABBITMQ", "null") + + +@pytest.fixture +def enabled_rabbitmq( + app_environment: EnvVarsDict, rabbit_service: RabbitSettings +) -> RabbitSettings: + return rabbit_service + + @pytest.fixture def app_settings( - aiopg_engine: Engine, + app_environment: EnvVarsDict, + enabled_rabbitmq: RabbitSettings, + sqlalchemy_async_engine: AsyncEngine, postgres_host_config: dict[str, str], mocked_s3_server_envs: EnvVarsDict, - external_envfile_dict: EnvVarsDict, - datcore_adapter_service_mock: aioresponses.aioresponses, - monkeypatch: pytest.MonkeyPatch, -) -> Settings: - s3_settings_dict = {} - if external_envfile_dict: - s3_settings = S3Settings.create_from_envs(**external_envfile_dict) - if s3_settings.S3_ENDPOINT is None: - monkeypatch.delenv("S3_ENDPOINT") - s3_settings_dict = s3_settings.model_dump(exclude={"S3_ENDPOINT"}) - else: - s3_settings_dict = s3_settings.model_dump() - setenvs_from_dict( - monkeypatch, - { - **s3_settings_dict, - "STORAGE_TRACING": "null", - }, - ) - test_app_settings = Settings.create_from_envs() + datcore_adapter_service_mock: respx.MockRouter, + mocked_redis_server, +) -> ApplicationSettings: + test_app_settings = ApplicationSettings.create_from_envs() print(f"{test_app_settings.model_dump_json(indent=2)=}") return test_app_settings @@ -195,18 +209,30 @@ async def mocked_redis_server(mocker: MockerFixture) -> None: mocker.patch("redis.asyncio.from_url", return_value=mock_redis) +_LIFESPAN_TIMEOUT: Final[int] = 10 + + @pytest.fixture -def client( - event_loop: asyncio.AbstractEventLoop, - aiohttp_client: Callable, - unused_tcp_port_factory: Callable[..., int], - app_settings: Settings, - mocked_redis_server, -) -> TestClient: - app = create(app_settings) - return event_loop.run_until_complete( - aiohttp_client(app, server_kwargs={"port": unused_tcp_port_factory()}) - ) +async def initialized_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: + settings = ApplicationSettings.create_from_envs() + app = create_app(settings) + # NOTE: the timeout is sometime too small for CI machines, and even larger machines + async with LifespanManager( + app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT + ): + yield app + + +@pytest.fixture +async def client( + initialized_app: FastAPI, +) -> AsyncIterator[httpx.AsyncClient]: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=initialized_app), + base_url=f"http://{initialized_app.title}.testserver.io", + headers={"Content-Type": "application/json"}, + ) as client: + yield client @pytest.fixture @@ -233,11 +259,11 @@ def simcore_file_id( @pytest.fixture( params=[ SimcoreS3DataManager.get_location_id(), - # DatCoreDataManager.get_location_id(), + DatCoreDataManager.get_location_id(), ], ids=[ SimcoreS3DataManager.get_location_name(), - # DatCoreDataManager.get_location_name(), + DatCoreDataManager.get_location_name(), ], ) def location_id(request: pytest.FixtureRequest) -> LocationID: @@ -246,23 +272,25 @@ def location_id(request: pytest.FixtureRequest) -> LocationID: @pytest.fixture async def get_file_meta_data( - client: TestClient, user_id: UserID, location_id: LocationID + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, + location_id: LocationID, ) -> Callable[..., Awaitable[FileMetaDataGet]]: async def _getter(file_id: SimcoreS3FileID) -> FileMetaDataGet: - assert client.app - url = ( - client.app.router["get_file_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(file_id, safe=""), - ) - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "get_file_metadata", + location_id=f"{location_id}", + file_id=file_id, + ).with_query(user_id=user_id) + response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + received_fmd, error = assert_status( + response, status.HTTP_200_OK, FileMetaDataGet + ) assert not error - assert data - received_fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert received_fmd return received_fmd @@ -271,30 +299,31 @@ async def _getter(file_id: SimcoreS3FileID) -> FileMetaDataGet: @pytest.fixture async def create_upload_file_link_v2( - client: TestClient, user_id: UserID, location_id: LocationID + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, + location_id: LocationID, ) -> AsyncIterator[Callable[..., Awaitable[FileUploadSchema]]]: file_params: list[tuple[UserID, int, SimcoreS3FileID]] = [] async def _link_creator( file_id: SimcoreS3FileID, **query_kwargs ) -> FileUploadSchema: - assert client.app - url = ( - client.app.router["upload_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(file_id, safe=""), - ) - .with_query(**query_kwargs, user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "upload_file", + location_id=f"{location_id}", + file_id=file_id, + ).with_query(**query_kwargs, user_id=user_id) assert ( "file_size" in url.query ), "V2 call to upload file must contain file_size field!" response = await client.put(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + received_file_upload, error = assert_status( + response, status.HTTP_200_OK, FileUploadSchema + ) assert not error - assert data - received_file_upload = TypeAdapter(FileUploadSchema).validate_python(data) assert received_file_upload file_params.append((user_id, location_id, file_id)) return received_file_upload @@ -302,27 +331,26 @@ async def _link_creator( yield _link_creator # cleanup - assert client.app clean_tasks = [] for u_id, loc_id, file_id in file_params: - url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{loc_id}", - file_id=urllib.parse.quote(file_id, safe=""), - ) - .with_query(user_id=u_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "delete_file", + location_id=f"{loc_id}", + file_id=file_id, + ).with_query(user_id=u_id) clean_tasks.append(client.delete(f"{url}")) await asyncio.gather(*clean_tasks) @pytest.fixture def upload_file( - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, project_id: ProjectID, node_id: NodeID, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], @@ -338,7 +366,6 @@ async def _uploader( sha256_checksum: SHA256Str | None = None, project_id: ProjectID = project_id, ) -> tuple[Path, SimcoreS3FileID]: - assert client.app # create a file file = create_file_of_size(file_size, file_name) if not file_id: @@ -348,7 +375,10 @@ async def _uploader( if sha256_checksum: query_params["sha256_checksum"] = f"{sha256_checksum}" file_upload_link = await create_upload_file_link_v2( - file_id, link_type="presigned", file_size=file_size, **query_params + file_id, + link_type=LinkType.PRESIGNED.value, + file_size=file_size, + **query_params, ) # upload the file @@ -363,12 +393,11 @@ async def _uploader( json=jsonable_encoder(FileUploadCompletionBody(parts=part_to_etag)), ) response.raise_for_status() - data, error = await assert_status(response, status.HTTP_202_ACCEPTED) - assert not error - assert data - file_upload_complete_response = FileUploadCompleteResponse.model_validate( - data + file_upload_complete_response, error = assert_status( + response, status.HTTP_202_ACCEPTED, FileUploadCompleteResponse ) + assert not error + assert file_upload_complete_response state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None @@ -378,18 +407,22 @@ async def _uploader( stop=stop_after_delay(60), retry=retry_if_exception_type(ValueError), ): - with attempt, log_context( - logging.INFO, - f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", - ) as ctx: + with ( + attempt, + log_context( + logging.INFO, + f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", + ) as ctx, + ): response = await client.post(f"{state_url}") response.raise_for_status() - data, error = await assert_status(response, status.HTTP_200_OK) + future, error = assert_status( + response, status.HTTP_200_OK, FileUploadCompleteFutureResponse + ) assert not error - assert data - future = FileUploadCompleteFutureResponse.model_validate(data) + assert future if future.state == FileUploadCompleteState.NOK: - msg = f"{data=}" + msg = f"{future=}" raise ValueError(msg) assert future.state == FileUploadCompleteState.OK assert future.e_tag is not None @@ -401,7 +434,7 @@ async def _uploader( # check the entry in db now has the correct file size, and the upload id is gone await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=file_id, expected_entry_exists=True, expected_file_size=file_size, @@ -421,25 +454,6 @@ async def _uploader( return _uploader -@pytest.fixture -def create_simcore_file_id( - faker: Faker, -) -> Callable[[ProjectID, NodeID, str, Path | None], SimcoreS3FileID]: - def _creator( - project_id: ProjectID, - node_id: NodeID, - file_name: str, - file_base_path: Path | None = None, - ) -> SimcoreS3FileID: - s3_file_name = file_name - if file_base_path: - s3_file_name = f"{file_base_path / file_name}" - clean_path = Path(f"{project_id}/{node_id}/{s3_file_name}") - return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}") - - return _creator - - @pytest.fixture async def with_versioning_enabled( s3_client: S3Client, @@ -455,11 +469,12 @@ async def with_versioning_enabled( async def create_empty_directory( create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], - client: TestClient, - project_id: ProjectID, - node_id: NodeID, -) -> Callable[..., Awaitable[FileUploadSchema]]: - async def _directory_creator(dir_name: str): + initialized_app: FastAPI, + client: httpx.AsyncClient, +) -> Callable[[str, ProjectID, NodeID], Awaitable[SimcoreS3FileID]]: + async def _directory_creator( + dir_name: str, project_id: ProjectID, node_id: NodeID + ) -> SimcoreS3FileID: # creating an empty directory goes through the same procedure as uploading a multipart file # done by using 3 calls: # 1. create the link as a directory @@ -467,8 +482,8 @@ async def _directory_creator(dir_name: str): # 3. call file_upload_complete_response until it replies OK directory_file_id = create_simcore_file_id(project_id, node_id, dir_name) - directory_file_upload: FileUploadSchema = await create_upload_file_link_v2( - directory_file_id, link_type="s3", is_directory="true", file_size=-1 + directory_file_upload = await create_upload_file_link_v2( + directory_file_id, link_type="S3", is_directory="true", file_size=0 ) # always returns a v2 link when dealing with directories assert isinstance(directory_file_upload, FileUploadSchema) @@ -481,15 +496,15 @@ async def _directory_creator(dir_name: str): json=jsonable_encoder(FileUploadCompletionBody(parts=[])), ) response.raise_for_status() - data, error = await assert_status(response, status.HTTP_202_ACCEPTED) + file_upload_complete_response, error = assert_status( + response, status.HTTP_202_ACCEPTED, FileUploadCompleteResponse + ) assert not error - assert data - file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + assert file_upload_complete_response state_url = URL(f"{file_upload_complete_response.links.state}").relative() # check that it finished updating - assert client.app - client.app[UPLOAD_TASKS_KEY].clear() + get_completed_upload_tasks(initialized_app).clear() # now check for the completion async for attempt in AsyncRetrying( reraise=True, @@ -497,15 +512,19 @@ async def _directory_creator(dir_name: str): stop=stop_after_delay(60), retry=retry_if_exception_type(AssertionError), ): - with attempt, log_context( - logging.INFO, - f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", - ) as ctx: + with ( + attempt, + log_context( + logging.INFO, + f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", + ) as ctx, + ): response = await client.post(f"{state_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + future, error = assert_status( + response, status.HTTP_200_OK, FileUploadCompleteFutureResponse + ) assert not error - assert data - future = FileUploadCompleteFutureResponse.model_validate(data) + assert future assert future.state == FileUploadCompleteState.OK assert future.e_tag is None ctx.logger.info( @@ -513,82 +532,127 @@ async def _directory_creator(dir_name: str): f"--> done waiting, data is completely uploaded [{attempt.retry_state.retry_object.statistics}]", ) - return directory_file_upload + return directory_file_id return _directory_creator +async def _upload_file_to_s3( + s3_client: SimcoreS3API, + faker: Faker, + *, + s3_bucket: S3BucketName, + local_file: Path, + file_id: SimcoreS3FileID, +) -> dict[SHA256Str, FileIDDict]: + await s3_client.upload_file( + bucket=s3_bucket, + file=local_file, + object_key=file_id, + bytes_transfered_cb=None, + ) + return {file_id: FileIDDict(path=local_file, sha256_checksum=f"{faker.sha256()}")} + + @pytest.fixture async def populate_directory( create_file_of_size: Callable[[ByteSize, str | None], Path], storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, - project_id: ProjectID, - node_id: NodeID, -) -> Callable[..., Awaitable[None]]: + faker: Faker, +) -> Callable[ + [ByteSize, str, ProjectID, NodeID, int, int], + Awaitable[tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]], +]: async def _create_content( file_size_in_dir: ByteSize, dir_name: str, - subdir_count: int = 4, - file_count: int = 5, - ) -> None: - file = create_file_of_size(file_size_in_dir, "some_file") - - async def _create_file(s: int, f: int): - file_name = f"{dir_name}/sub-dir-{s}/file-{f}" - clean_path = Path(f"{project_id}/{node_id}/{file_name}") - await storage_s3_client.upload_file( - bucket=storage_s3_bucket, - file=file, - object_key=TypeAdapter(SimcoreS3FileID).validate_python( - f"{clean_path}" - ), - bytes_transfered_cb=None, + project_id: ProjectID, + node_id: NodeID, + subdir_count: int, + file_count: int, + ) -> tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]: + assert subdir_count >= 1, "cannot use fixture with subdir_count < 1!" + assert file_count >= 1, "cannot use fixture with file_count < 1!" + + local_file = create_file_of_size(file_size_in_dir, None) + + # Create subdirectories + s3_base_path = Path(f"{project_id}") / f"{node_id}" / dir_name + # NOTE: add a space in the sub directory + s3_subdirs = [ + s3_base_path / f"sub-dir_ect ory-{i}" for i in range(subdir_count) + ] + # Randomly distribute files across subdirectories + selected_subdirs = random.choices(s3_subdirs, k=file_count) # noqa: S311 + # Upload to S3 + with log_context( + logging.INFO, + msg=f"Uploading {file_count} files to S3 (each {file_size_in_dir.human_readable()}, total: {ByteSize(file_count * file_size_in_dir).human_readable()})", + ): + # we ensure the file name contain a space + def _file_name_with_space(): + file_name = faker.unique.file_name() + return f"{file_name[:1]} {file_name[1:]}" + + results = await asyncio.gather( + *( + _upload_file_to_s3( + storage_s3_client, + faker, + s3_bucket=storage_s3_bucket, + local_file=local_file, + file_id=TypeAdapter(SimcoreS3FileID).validate_python( + f"{selected_subdir / _file_name_with_space()}" + ), + ) + for selected_subdir in selected_subdirs + ) ) - tasks = [ - _create_file(s, f) for f in range(file_count) for s in range(subdir_count) - ] + assert len(results) == file_count - await asyncio.gather(*tasks) + # check this is true + counted_uploaded_objects = await storage_s3_client.count_objects( + bucket=storage_s3_bucket, + prefix=s3_base_path, + is_partial_prefix=True, + start_after=None, + use_delimiter=False, + ) + assert counted_uploaded_objects == file_count - file.unlink() + return node_id, {k: v for r in results for k, v in r.items()} return _create_content @pytest.fixture async def delete_directory( - client: TestClient, - storage_s3_client: SimcoreS3API, - storage_s3_bucket: S3BucketName, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, location_id: LocationID, ) -> Callable[..., Awaitable[None]]: - async def _dir_remover(directory_file_upload: FileUploadSchema) -> None: - assert directory_file_upload.urls[0].path - directory_file_id = directory_file_upload.urls[0].path.strip("/") - assert client.app - delete_url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(directory_file_id, safe=""), - ) - .with_query(user_id=user_id) - ) + async def _dir_remover(directory_s3: StorageFileID) -> None: + delete_url = url_from_operation_id( + client, + initialized_app, + "delete_file", + location_id=f"{location_id}", + file_id=directory_s3, + ).with_query(user_id=user_id) + response = await client.delete(f"{delete_url}") - await assert_status(response, status.HTTP_204_NO_CONTENT) + assert_status(response, status.HTTP_204_NO_CONTENT, None) # NOTE: ensures no more files are left in the directory, # even if one file is left this will detect it - list_files_metadata_url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{location_id}") - .with_query(user_id=user_id, uuid_filter=directory_file_id) - ) + list_files_metadata_url = url_from_operation_id( + client, initialized_app, "list_files_metadata", location_id=f"{location_id}" + ).with_query(user_id=user_id, uuid_filter=directory_s3) response = await client.get(f"{list_files_metadata_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + data, error = assert_status(response, status.HTTP_200_OK, list[FileMetaDataGet]) assert error is None assert data == [] @@ -597,27 +661,287 @@ async def _dir_remover(directory_file_upload: FileUploadSchema) -> None: @pytest.fixture async def create_directory_with_files( - create_empty_directory: Callable[..., Awaitable[FileUploadSchema]], - populate_directory: Callable[..., Awaitable[None]], + create_empty_directory: Callable[ + [str, ProjectID, NodeID], Awaitable[SimcoreS3FileID] + ], + populate_directory: Callable[ + [ByteSize, str, ProjectID, NodeID, int, int], + Awaitable[tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]], + ], delete_directory: Callable[..., Awaitable[None]], -) -> Callable[..., AbstractAsyncContextManager[FileUploadSchema]]: - @asynccontextmanager - async def _create_context( - dir_name: str, file_size_in_dir: ByteSize, subdir_count: int, file_count: int - ) -> AsyncIterator[FileUploadSchema]: - directory_file_upload: FileUploadSchema = await create_empty_directory( - dir_name=dir_name +) -> AsyncIterator[ + Callable[ + [str, ByteSize, int, int, ProjectID, NodeID], + Awaitable[ + tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], + ] +]: + uploaded_directories = [] + + async def _( + dir_name: str, + file_size_in_dir: ByteSize, + subdir_count: int, + file_count: int, + project_id: ProjectID, + node_id: NodeID, + ) -> tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]]: + directory_file_id = await create_empty_directory(dir_name, project_id, node_id) + + uploaded_files = await populate_directory( + file_size_in_dir, + dir_name, + project_id, + node_id, + subdir_count, + file_count, + ) + + uploaded_directories.append(directory_file_id) + + return directory_file_id, uploaded_files + + yield _ + + await asyncio.gather(*(delete_directory(_) for _ in uploaded_directories)) + + +async def _upload_one_file_task( + upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], + allowed_file_sizes: tuple[ByteSize, ...], + allowed_file_checksums: tuple[SHA256Str, ...], + *, + file_name: str, + file_id: SimcoreS3FileID, + node_id: NodeID, +) -> tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]: + selected_checksum = random.choice(allowed_file_checksums) # noqa: S311 + uploaded_file, uploaded_file_id = await upload_file( + file_size=random.choice(allowed_file_sizes), # noqa: S311 + file_name=file_name, + file_id=file_id, + sha256_checksum=selected_checksum, + ) + assert uploaded_file_id == file_id + return ( + node_id, + { + uploaded_file_id: FileIDDict( + path=uploaded_file, sha256_checksum=selected_checksum + ) + }, + ) + + +async def _upload_folder_task( + create_directory_with_files: Callable[ + ..., + Awaitable[ + tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], + ], + allowed_file_sizes: tuple[ByteSize, ...], + *, + dir_name: str, + project_id: ProjectID, + node_id: NodeID, + workspace_file_count: int, +) -> tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]: + dir_file_id, node_files_map = await create_directory_with_files( + dir_name=dir_name, + file_size_in_dir=random.choice(allowed_file_sizes), # noqa: S311 + subdir_count=3, + file_count=workspace_file_count, + project_id=project_id, + node_id=node_id, + ) + assert dir_file_id + return node_files_map + + +@pytest.fixture +async def random_project_with_files( + sqlalchemy_async_engine: AsyncEngine, + create_project: Callable[..., Awaitable[dict[str, Any]]], + create_project_node: Callable[..., Awaitable[NodeID]], + create_simcore_file_id: Callable[ + [ProjectID, NodeID, str, Path | None], SimcoreS3FileID + ], + faker: Faker, + create_directory_with_files: Callable[ + ..., + Awaitable[ + tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], + ], + upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], +) -> Callable[ + [ProjectWithFilesParams], + Awaitable[tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]], +]: + async def _creator( + project_params: ProjectWithFilesParams, + ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]: + assert len(project_params.allowed_file_sizes) == len( + project_params.allowed_file_checksums ) + project = await create_project(name="random-project") + node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]] = {} + upload_tasks = [] + for _ in range(project_params.num_nodes): + # Create a node with outputs (files and others) + project_id = ProjectID(project["uuid"]) + node_id = cast(NodeID, faker.uuid4(cast_to=None)) + node_to_files_mapping[node_id] = {} + output3_file_name = faker.file_name() + output3_file_id = create_simcore_file_id( + project_id, node_id, output3_file_name, Path("outputs/output_3") + ) + created_node_id = await create_project_node( + ProjectID(project["uuid"]), + node_id, + outputs={ + "output_1": faker.pyint(), + "output_2": faker.pystr(), + "output_3": f"{output3_file_id}", + }, + ) + assert created_node_id == node_id + + upload_tasks.append( + _upload_one_file_task( + upload_file, + project_params.allowed_file_sizes, + project_params.allowed_file_checksums, + file_name=output3_file_name, + file_id=output3_file_id, + node_id=node_id, + ) + ) + + # some workspace files (these are not referenced in the file_meta_data, only as a folder) + if project_params.workspace_files_count > 0: + upload_tasks.append( + _upload_folder_task( + create_directory_with_files, + project_params.allowed_file_sizes, + dir_name="workspace", + project_id=project_id, + node_id=node_id, + workspace_file_count=project_params.workspace_files_count, + ) + ) + + # add a few random files in the node root space for good measure + for _ in range(random.randint(1, 3)): # noqa: S311 + root_file_name = faker.file_name() + root_file_id = create_simcore_file_id( + project_id, node_id, root_file_name, None + ) + upload_tasks.append( + _upload_one_file_task( + upload_file, + project_params.allowed_file_sizes, + project_params.allowed_file_checksums, + file_name=root_file_name, + file_id=root_file_id, + node_id=node_id, + ), + ) - await populate_directory( - file_size_in_dir=file_size_in_dir, - dir_name=dir_name, - subdir_count=subdir_count, - file_count=file_count, + # upload everything of the node + results = await limited_gather(*upload_tasks, limit=10) + + for node_id, file_id_to_dict_mapping in results: + for file_id, file_dict in file_id_to_dict_mapping.items(): + node_to_files_mapping[node_id][file_id] = file_dict + + project = await get_updated_project(sqlalchemy_async_engine, project["uuid"]) + return project, node_to_files_mapping + + return _creator + + +@pytest.fixture +async def with_random_project_with_files( + random_project_with_files: Callable[ + [ProjectWithFilesParams], + Awaitable[ + tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], + ], + project_params: ProjectWithFilesParams, +) -> tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], +]: + return await random_project_with_files(project_params) + + +@pytest.fixture() +async def output_file( + user_id: UserID, project_id: str, sqlalchemy_async_engine: AsyncEngine, faker: Faker +) -> AsyncIterator[FileMetaData]: + node_id = "fd6f9737-1988-341b-b4ac-0614b646fa82" + + # pylint: disable=no-value-for-parameter + + file = FileMetaData.from_simcore_node( + user_id=user_id, + file_id=f"{project_id}/{node_id}/filename.txt", + bucket=TypeAdapter(S3BucketName).validate_python("master-simcore"), + location_id=SimcoreS3DataManager.get_location_id(), + location_name=SimcoreS3DataManager.get_location_name(), + sha256_checksum=faker.sha256(), + ) + file.entity_tag = "df9d868b94e53d18009066ca5cd90e9f" + file.file_size = ByteSize(12) + file.user_id = user_id + async with sqlalchemy_async_engine.begin() as conn: + stmt = ( + file_meta_data.insert() + .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) + .returning(literal_column("*")) + ) + result = await conn.execute(stmt) + row = result.one() + assert row + + yield file + + async with sqlalchemy_async_engine.begin() as conn: + result = await conn.execute( + file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) ) - yield directory_file_upload - await delete_directory(directory_file_upload=directory_file_upload) +@pytest.fixture +async def fake_datcore_tokens( + user_id: UserID, sqlalchemy_async_engine: AsyncEngine, faker: Faker +) -> AsyncIterator[tuple[str, str]]: + token_key = cast(str, faker.uuid4()) + token_secret = cast(str, faker.uuid4()) + created_token_ids = [] + async with sqlalchemy_async_engine.begin() as conn: + result = await conn.execute( + tokens.insert() + .values( + user_id=user_id, + token_service="pytest", # noqa: S106 + token_data={ + "service": "pytest", + "token_secret": token_secret, + "token_key": token_key, + }, + ) + .returning(tokens.c.token_id) + ) + row = result.one() + created_token_ids.append(row.token_id) + yield token_key, token_secret - return _create_context + async with sqlalchemy_async_engine.begin() as conn: + await conn.execute( + tokens.delete().where(tokens.c.token_id.in_(created_token_ids)) + ) diff --git a/services/storage/tests/data/file_meta_data.csv b/services/storage/tests/data/file_meta_data.csv deleted file mode 100644 index 24bf7dc5ef8..00000000000 --- a/services/storage/tests/data/file_meta_data.csv +++ /dev/null @@ -1,3 +0,0 @@ -file_id,location_id,location,bucket_name,object_name,project_id,node_id,user_id -161b8782-b13e-5840-9ae2-e2250c231001/ad9bda7f-1dc5-5480-ab22-5fef4fc53eac/outputController.dat,0,simcore.s3,pytestbucket,161b8782-b13e-5840-9ae2-e2250c231001/ad9bda7f-1dc5-5480-ab22-5fef4fc53eac/outputController.dat,161b8782-b13e-5840-9ae2-e2250c231001,ad9bda7f-1dc5-5480-ab22-5fef4fc53eac,21 -161b8782-b13e-5840-9ae2-e2250c231001/a3941ea0-37c4-5c1d-a7b3-01b5fd8a80c8/notebooks.zip,0,simcore.s3,pytestbucket,161b8782-b13e-5840-9ae2-e2250c231001/a3941ea0-37c4-5c1d-a7b3-01b5fd8a80c8/notebooks.zip,161b8782-b13e-5840-9ae2-e2250c231001,a3941ea0-37c4-5c1d-a7b3-01b5fd8a80c8,21 diff --git a/services/storage/tests/data/notebooks.zip b/services/storage/tests/data/notebooks.zip deleted file mode 100644 index 94ebaf90016..00000000000 --- a/services/storage/tests/data/notebooks.zip +++ /dev/null @@ -1,4 +0,0 @@ -1 -2 -3 -4 diff --git a/services/storage/tests/data/outputController.dat b/services/storage/tests/data/outputController.dat deleted file mode 100644 index 94ebaf90016..00000000000 --- a/services/storage/tests/data/outputController.dat +++ /dev/null @@ -1,4 +0,0 @@ -1 -2 -3 -4 diff --git a/services/storage/tests/data/projects.csv b/services/storage/tests/data/projects.csv deleted file mode 100644 index 9ea1d463545..00000000000 --- a/services/storage/tests/data/projects.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,hidden -151,STANDARD,161b8782-b13e-5840-9ae2-e2250c231001,Kember use case,Kember Cordiac Model with PostPro Viewer,"",21,2019-06-27 11:42:03.168,2019-06-27 11:43:49.128,"{""ad9bda7f-1dc5-5480-ab22-5fef4fc53eac"": {""key"": ""simcore/services/comp/kember-cardiac-model"", ""version"": ""1.0.0"", ""label"": ""Kember cardiac model"", ""inputs"": {""dt"": 0.01, ""T"": 1000, ""forcing_factor"": 0}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 100, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 100}}, ""a3941ea0-37c4-5c1d-a7b3-01b5fd8a80c8"": {""key"": ""simcore/services/dynamic/kember-viewer"", ""version"": ""2.9.0"", ""label"": ""kember-viewer"", ""inputs"": {""outputController"": {""nodeUuid"": ""ad9bda7f-1dc5-5480-ab22-5fef4fc53eac"", ""output"": ""out_1""}}, ""inputNodes"": [""ad9bda7f-1dc5-5480-ab22-5fef4fc53eac""], ""outputs"": {}, ""progress"": 100, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 100}}}",false,"{}",false diff --git a/services/storage/tests/data/users.csv b/services/storage/tests/data/users.csv deleted file mode 100644 index d35b80d9626..00000000000 --- a/services/storage/tests/data/users.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,name,email,password_hash,status,role,created_at -21,devops,devops@itis.swiss,$5$rounds=1000$jjUWjHSG5F2dMKw.$9VRlE4YLl4bPfIrWkDz/8GtEx1XkzTpuZzyc/uiBFE4,ACTIVE,USER,2019-06-27 11:35:44.828696 diff --git a/services/storage/tests/fixtures/data_models.py b/services/storage/tests/fixtures/data_models.py deleted file mode 100644 index ab225928f62..00000000000 --- a/services/storage/tests/fixtures/data_models.py +++ /dev/null @@ -1,369 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -from collections import deque -from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import asynccontextmanager -from pathlib import Path -from random import choice, randint -from typing import Any - -import pytest -import sqlalchemy as sa -from aiopg.sa.connection import SAConnection -from aiopg.sa.engine import Engine -from faker import Faker -from models_library.basic_types import SHA256Str -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from models_library.users import UserID -from pydantic import ByteSize, TypeAdapter -from pytest_simcore.helpers.faker_factories import random_project, random_user -from servicelib.utils import limited_gather -from simcore_postgres_database.models.project_to_groups import project_to_groups -from simcore_postgres_database.storage_models import projects, users -from sqlalchemy.dialects.postgresql import insert as pg_insert - -from ..helpers.utils import get_updated_project - - -@asynccontextmanager -async def _user_context(aiopg_engine: Engine, *, name: str) -> AsyncIterator[UserID]: - # inject a random user in db - - # NOTE: Ideally this (and next fixture) should be done via webserver API but at this point - # in time, the webserver service would bring more dependencies to other services - # which would turn this test too complex. - - # pylint: disable=no-value-for-parameter - stmt = users.insert().values(**random_user(name=name)).returning(users.c.id) - async with aiopg_engine.acquire() as conn: - result = await conn.execute(stmt) - row = await result.fetchone() - assert row - assert isinstance(row.id, int) - - try: - yield TypeAdapter(UserID).validate_python(row.id) - finally: - async with aiopg_engine.acquire() as conn: - await conn.execute(users.delete().where(users.c.id == row.id)) - - -@pytest.fixture -async def user_id(aiopg_engine: Engine) -> AsyncIterator[UserID]: - async with _user_context(aiopg_engine, name="test-user") as new_user_id: - yield new_user_id - - -@pytest.fixture -async def other_user_id(aiopg_engine: Engine) -> AsyncIterator[UserID]: - async with _user_context(aiopg_engine, name="test-other-user") as new_user_id: - yield new_user_id - - -@pytest.fixture -async def create_project( - user_id: UserID, aiopg_engine: Engine -) -> AsyncIterator[Callable[[], Awaitable[dict[str, Any]]]]: - created_project_uuids = [] - - async def _creator(**kwargs) -> dict[str, Any]: - prj_config = {"prj_owner": user_id} - prj_config.update(kwargs) - async with aiopg_engine.acquire() as conn: - result = await conn.execute( - projects.insert() - .values(**random_project(**prj_config)) - .returning(sa.literal_column("*")) - ) - row = await result.fetchone() - assert row - created_project_uuids.append(row[projects.c.uuid]) - return dict(row) - - yield _creator - # cleanup - async with aiopg_engine.acquire() as conn: - await conn.execute( - projects.delete().where(projects.c.uuid.in_(created_project_uuids)) - ) - - -@pytest.fixture -async def create_project_access_rights( - aiopg_engine: Engine, -) -> AsyncIterator[Callable[[ProjectID, UserID, bool, bool, bool], Awaitable[None]]]: - _created = [] - - async def _creator( - project_id: ProjectID, user_id: UserID, read: bool, write: bool, delete: bool - ) -> None: - async with aiopg_engine.acquire() as conn: - result = await conn.execute( - project_to_groups.insert() - .values( - project_uuid=f"{project_id}", - gid=sa.select(users.c.primary_gid) - .where(users.c.id == f"{user_id}") - .scalar_subquery(), - read=read, - write=write, - delete=delete, - ) - .returning(sa.literal_column("*")) - ) - row = await result.fetchone() - assert row - _created.append( - (row[project_to_groups.c.project_uuid], row[project_to_groups.c.gid]) - ) - - yield _creator - - # cleanup - async with aiopg_engine.acquire() as conn: - await conn.execute( - project_to_groups.delete().where( - sa.or_( - *( - (project_to_groups.c.project_uuid == pid) - & (project_to_groups.c.gid == gid) - for pid, gid in _created - ) - ) - ) - ) - - -@pytest.fixture -async def project_id( - create_project: Callable[[], Awaitable[dict[str, Any]]] -) -> ProjectID: - project = await create_project() - return ProjectID(project["uuid"]) - - -@pytest.fixture -async def collaborator_id(aiopg_engine: Engine) -> AsyncIterator[UserID]: - - async with _user_context(aiopg_engine, name="collaborator") as new_user_id: - yield TypeAdapter(UserID).validate_python(new_user_id) - - -@pytest.fixture -def share_with_collaborator( - aiopg_engine: Engine, - collaborator_id: UserID, - user_id: UserID, - project_id: ProjectID, -) -> Callable[[], Awaitable[None]]: - async def _get_user_group(conn: SAConnection, query_user: int) -> int: - result = await conn.execute( - sa.select(users.c.primary_gid).where(users.c.id == query_user) - ) - row = await result.fetchone() - assert row - primary_gid: int = row[users.c.primary_gid] - return primary_gid - - async def _() -> None: - async with aiopg_engine.acquire() as conn: - result = await conn.execute( - sa.select(projects.c.access_rights).where( - projects.c.uuid == f"{project_id}" - ) - ) - row = await result.fetchone() - assert row - access_rights: dict[str | int, Any] = row[projects.c.access_rights] - - access_rights[await _get_user_group(conn, user_id)] = { - "read": True, - "write": True, - "delete": True, - } - access_rights[await _get_user_group(conn, collaborator_id)] = { - "read": True, - "write": True, - "delete": False, - } - - await conn.execute( - projects.update() - .where(projects.c.uuid == f"{project_id}") - .values(access_rights=access_rights) - ) - - # project_to_groups needs to be updated - for group_id, permissions in access_rights.items(): - insert_stmt = pg_insert(project_to_groups).values( - project_uuid=f"{project_id}", - gid=int(group_id), - read=permissions["read"], - write=permissions["write"], - delete=permissions["delete"], - created=sa.func.now(), - modified=sa.func.now(), - ) - on_update_stmt = insert_stmt.on_conflict_do_update( - index_elements=[ - project_to_groups.c.project_uuid, - project_to_groups.c.gid, - ], - set_={ - "read": insert_stmt.excluded.read, - "write": insert_stmt.excluded.write, - "delete": insert_stmt.excluded.delete, - "modified": sa.func.now(), - }, - ) - await conn.execute(on_update_stmt) - - return _ - - -@pytest.fixture -async def create_project_node( - user_id: UserID, aiopg_engine: Engine, faker: Faker -) -> Callable[..., Awaitable[NodeID]]: - async def _creator( - project_id: ProjectID, node_id: NodeID | None = None, **kwargs - ) -> NodeID: - async with aiopg_engine.acquire() as conn: - result = await conn.execute( - sa.select(projects.c.workbench).where( - projects.c.uuid == f"{project_id}" - ) - ) - row = await result.fetchone() - assert row - project_workbench: dict[str, Any] = row[projects.c.workbench] - new_node_id = node_id or NodeID(f"{faker.uuid4()}") - node_data = { - "key": "simcore/services/frontend/file-picker", - "version": "1.0.0", - "label": "pytest_fake_node", - } - node_data.update(**kwargs) - project_workbench.update({f"{new_node_id}": node_data}) - await conn.execute( - projects.update() - .where(projects.c.uuid == f"{project_id}") - .values(workbench=project_workbench) - ) - return new_node_id - - return _creator - - -@pytest.fixture -async def random_project_with_files( - aiopg_engine: Engine, - create_project: Callable[..., Awaitable[dict[str, Any]]], - create_project_node: Callable[..., Awaitable[NodeID]], - create_simcore_file_id: Callable[ - [ProjectID, NodeID, str, Path | None], SimcoreS3FileID - ], - upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], - faker: Faker, -) -> Callable[ - [int, tuple[ByteSize, ...], tuple[SHA256Str, ...]], - Awaitable[ - tuple[ - dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]] - ] - ], -]: - async def _creator( - num_nodes: int = 12, - file_sizes: tuple[ByteSize, ...] = ( - TypeAdapter(ByteSize).validate_python("7Mib"), - TypeAdapter(ByteSize).validate_python("110Mib"), - TypeAdapter(ByteSize).validate_python("1Mib"), - ), - file_checksums: tuple[SHA256Str, ...] = ( - TypeAdapter(SHA256Str).validate_python( - "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd" - ), - TypeAdapter(SHA256Str).validate_python( - "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e" - ), - TypeAdapter(SHA256Str).validate_python( - "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3" - ), - ), - ) -> tuple[ - dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]] - ]: - assert len(file_sizes) == len(file_checksums) - project = await create_project(name="random-project") - src_projects_list: dict[ - NodeID, dict[SimcoreS3FileID, dict[str, Path | str]] - ] = {} - upload_tasks: deque[Awaitable] = deque() - for _node_index in range(num_nodes): - # NOTE: we put some more outputs in there to simulate a real case better - new_node_id = NodeID(f"{faker.uuid4()}") - output3_file_id = create_simcore_file_id( - ProjectID(project["uuid"]), - new_node_id, - faker.file_name(), - Path("outputs/output3"), - ) - src_node_id = await create_project_node( - ProjectID(project["uuid"]), - new_node_id, - outputs={ - "output_1": faker.pyint(), - "output_2": faker.pystr(), - "output_3": f"{output3_file_id}", - }, - ) - assert src_node_id == new_node_id - - # upload the output 3 and some random other files at the root of each node - src_projects_list[src_node_id] = {} - checksum: SHA256Str = choice(file_checksums) # noqa: S311 - src_file, _ = await upload_file( - file_size=choice(file_sizes), # noqa: S311 - file_name=Path(output3_file_id).name, - file_id=output3_file_id, - sha256_checksum=checksum, - ) - src_projects_list[src_node_id][output3_file_id] = { - "path": src_file, - "sha256_checksum": checksum, - } - - async def _upload_file_and_update_project(project, src_node_id): - src_file_name = faker.file_name() - src_file_uuid = create_simcore_file_id( - ProjectID(project["uuid"]), src_node_id, src_file_name, None - ) - checksum: SHA256Str = choice(file_checksums) # noqa: S311 - src_file, _ = await upload_file( - file_size=choice(file_sizes), # noqa: S311 - file_name=src_file_name, - file_id=src_file_uuid, - sha256_checksum=checksum, - ) - src_projects_list[src_node_id][src_file_uuid] = { - "path": src_file, - "sha256_checksum": checksum, - } - - # add a few random files in the node storage - upload_tasks.extend( - [ - _upload_file_and_update_project(project, src_node_id) - for _ in range(randint(0, 3)) # noqa: S311 - ] - ) - await limited_gather(*upload_tasks, limit=10) - - project = await get_updated_project(aiopg_engine, project["uuid"]) - return project, src_projects_list - - return _creator diff --git a/services/storage/tests/fixtures/datcore_adapter.py b/services/storage/tests/fixtures/datcore_adapter.py deleted file mode 100644 index ba7569b027b..00000000000 --- a/services/storage/tests/fixtures/datcore_adapter.py +++ /dev/null @@ -1,26 +0,0 @@ -import re - -import pytest -from aioresponses import aioresponses as AioResponsesMock -from servicelib.aiohttp import status -from simcore_service_storage.datcore_adapter.datcore_adapter_settings import ( - DatcoreAdapterSettings, -) - - -@pytest.fixture -def datcore_adapter_service_mock( - aioresponses_mocker: AioResponsesMock, -) -> AioResponsesMock: - dat_core_settings = DatcoreAdapterSettings.create_from_envs() - datcore_adapter_base_url = dat_core_settings.endpoint - # mock base endpoint - aioresponses_mocker.get( - datcore_adapter_base_url, status=status.HTTP_200_OK, repeat=True - ) - list_datasets_re = re.compile(rf"^{datcore_adapter_base_url}/datasets") - aioresponses_mocker.get(list_datasets_re, status=status.HTTP_200_OK, repeat=True) - aioresponses_mocker.get( - datcore_adapter_base_url, status=status.HTTP_200_OK, repeat=True, payload={} - ) - return aioresponses_mocker diff --git a/services/storage/tests/helpers/utils.py b/services/storage/tests/helpers/utils.py deleted file mode 100644 index dc98a400073..00000000000 --- a/services/storage/tests/helpers/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging -import os -from typing import Any - -import sqlalchemy as sa -from aiopg.sa.engine import Engine -from simcore_postgres_database.storage_models import projects - -log = logging.getLogger(__name__) - - -def has_datcore_tokens() -> bool: - # TODO: activate tests against BF services in the CI. - # - # CI shall add BF_API_KEY, BF_API_SECRET environs as secrets - # - if not os.environ.get("BF_API_KEY") or not os.environ.get("BF_API_SECRET"): - return False - return True - - -async def get_updated_project(aiopg_engine: Engine, project_id: str) -> dict[str, Any]: - async with aiopg_engine.acquire() as conn: - result = await conn.execute( - sa.select(projects).where(projects.c.uuid == project_id) - ) - row = await result.fetchone() - assert row - return dict(row) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py new file mode 100644 index 00000000000..8bbb621ef0b --- /dev/null +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -0,0 +1,102 @@ +from collections.abc import Callable, Iterable +from datetime import timedelta +from typing import Any + +import pytest +from celery import Celery +from celery.contrib.testing.worker import TestWorkController, start_worker +from celery.signals import worker_init, worker_shutdown +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient +from simcore_service_storage.modules.celery.signals import ( + on_worker_init, + on_worker_shutdown, +) +from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + "SC_BOOT_MODE": "local-development", + "RABBIT_HOST": "localhost", + "RABBIT_PORT": "5672", + "RABBIT_USER": "mock", + "RABBIT_SECURE": True, + "RABBIT_PASSWORD": "", + }, + ) + + +@pytest.fixture +def celery_conf() -> dict[str, Any]: + return { + "broker_url": "memory://", + "result_backend": "cache+memory://", + "result_expires": timedelta(days=7), + "result_extended": True, + "pool": "threads", + "worker_send_task_events": True, + "task_track_started": True, + "task_send_sent_event": True, + } + + +@pytest.fixture +def celery_app(celery_conf: dict[str, Any]): + return Celery(**celery_conf) + + +@pytest.fixture +def register_celery_tasks() -> Callable[[Celery], None]: + msg = "please define a callback that registers the tasks" + raise NotImplementedError(msg) + + +@pytest.fixture +def celery_client( + app_environment: EnvVarsDict, celery_app: Celery +) -> CeleryTaskQueueClient: + return CeleryTaskQueueClient(celery_app) + + +@pytest.fixture +def celery_worker_controller( + app_environment: EnvVarsDict, + register_celery_tasks: Callable[[Celery], None], + celery_app: Celery, +) -> Iterable[TestWorkController]: + + # Signals must be explicitily connected + worker_init.connect(on_worker_init) + worker_shutdown.connect(on_worker_shutdown) + + register_celery_tasks(celery_app) + + with start_worker( + celery_app, + pool="threads", + loglevel="info", + perform_ping_check=False, + worker_kwargs={"hostname": "celery@worker1"}, + ) as worker: + worker_init.send(sender=worker) + + yield worker + + worker_shutdown.send(sender=worker) + + +@pytest.fixture +def celery_worker( + celery_worker_controller: TestWorkController, +) -> CeleryTaskQueueWorker: + assert isinstance(celery_worker_controller.app, Celery) + return CeleryTaskQueueWorker(celery_worker_controller.app) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py new file mode 100644 index 00000000000..77dd5cf3e2a --- /dev/null +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -0,0 +1,191 @@ +import asyncio +import logging +import time +from collections.abc import Callable +from random import randint + +import pytest +from celery import Celery, Task +from celery.contrib.abortable import AbortableTask +from common_library.errors_classes import OsparcErrorMixin +from models_library.progress_bar import ProgressReport +from pydantic import TypeAdapter, ValidationError +from servicelib.logging_utils import log_context +from simcore_service_storage.modules.celery import get_event_loop +from simcore_service_storage.modules.celery._task import define_task +from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient +from simcore_service_storage.modules.celery.models import ( + TaskContext, + TaskError, + TaskState, +) +from simcore_service_storage.modules.celery.utils import ( + get_celery_worker, + get_fastapi_app, +) +from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed + +_logger = logging.getLogger(__name__) + + +async def _async_archive( + celery_app: Celery, task_name: str, task_id: str, files: list[str] +) -> str: + worker = get_celery_worker(celery_app) + + def sleep_for(seconds: float) -> None: + time.sleep(seconds) + + for n, file in enumerate(files, start=1): + with log_context(_logger, logging.INFO, msg=f"Processing file {file}"): + worker.set_task_progress( + task_name=task_name, + task_id=task_id, + report=ProgressReport(actual_value=n / len(files) * 10), + ) + await asyncio.get_event_loop().run_in_executor(None, sleep_for, 1) + + return "archive.zip" + + +def sync_archive(task: Task, files: list[str]) -> str: + assert task.name + _logger.info("Calling async_archive") + return asyncio.run_coroutine_threadsafe( + _async_archive(task.app, task.name, task.request.id, files), + get_event_loop(get_fastapi_app(task.app)), + ).result() + + +class MyError(OsparcErrorMixin, Exception): + msg_template = "Something strange happened: {msg}" + + +def failure_task(task: Task): + msg = "BOOM!" + raise MyError(msg=msg) + + +def dreamer_task(task: AbortableTask) -> list[int]: + numbers = [] + for _ in range(30): + if task.is_aborted(): + _logger.warning("Alarm clock") + return numbers + numbers.append(randint(1, 90)) # noqa: S311 + time.sleep(1) + return numbers + + +@pytest.fixture +def register_celery_tasks() -> Callable[[Celery], None]: + def _(celery_app: Celery) -> None: + define_task(celery_app, sync_archive) + define_task(celery_app, failure_task) + define_task(celery_app, dreamer_task) + + return _ + + +@pytest.mark.usefixtures("celery_worker") +async def test_submitting_task_calling_async_function_results_with_success_state( + celery_client: CeleryTaskQueueClient, +): + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task( + "sync_archive", + task_context=task_context, + files=[f"file{n}" for n in range(5)], + ) + + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), + stop=stop_after_delay(30), + ): + with attempt: + status = await celery_client.get_task_status(task_context, task_uuid) + assert status.task_state == TaskState.SUCCESS + + assert ( + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.SUCCESS + assert ( + await celery_client.get_task_result(task_context, task_uuid) + ) == "archive.zip" + + +@pytest.mark.usefixtures("celery_worker") +async def test_submitting_task_with_failure_results_with_error( + celery_client: CeleryTaskQueueClient, +): + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task("failure_task", task_context=task_context) + + for attempt in Retrying( + retry=retry_if_exception_type((AssertionError, ValidationError)), + wait=wait_fixed(1), + stop=stop_after_delay(30), + ): + with attempt: + raw_result = await celery_client.get_task_result(task_context, task_uuid) + result = TypeAdapter(TaskError).validate_python(raw_result) + assert isinstance(result, TaskError) + + assert ( + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.ERROR + raw_result = await celery_client.get_task_result(task_context, task_uuid) + result = TypeAdapter(TaskError).validate_python(raw_result) + assert f"{result.exc_msg}" == "Something strange happened: BOOM!" + + +@pytest.mark.usefixtures("celery_worker") +async def test_aborting_task_results_with_aborted_state( + celery_client: CeleryTaskQueueClient, +): + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task( + "dreamer_task", + task_context=task_context, + ) + + await celery_client.abort_task(task_context, task_uuid) + + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), + stop=stop_after_delay(30), + ): + with attempt: + progress = await celery_client.get_task_status(task_context, task_uuid) + assert progress.task_state == TaskState.ABORTED + + assert ( + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.ABORTED + + +@pytest.mark.usefixtures("celery_worker") +async def test_listing_task_uuids_contains_submitted_task( + celery_client: CeleryTaskQueueClient, +): + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task( + "dreamer_task", + task_context=task_context, + ) + + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), + stop=stop_after_delay(10), + ): + with attempt: + assert task_uuid in await celery_client.get_task_uuids(task_context) + + assert task_uuid in await celery_client.get_task_uuids(task_context) diff --git a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py index 5e94a17d3bc..0531f02c6b1 100644 --- a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py +++ b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py @@ -11,27 +11,92 @@ used in simcore_sdk since legacy services are planned to be deprecated. """ +import logging +from collections.abc import AsyncIterator from pathlib import Path +from threading import Thread import aiohttp +import httpx import pytest -from aiohttp.test_utils import TestClient +import uvicorn from faker import Faker -from models_library.projects_nodes_io import LocationID, SimcoreS3FileID +from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID +from pytest_simcore.helpers.logging_tools import log_context +from servicelib.utils import unused_port +from simcore_service_storage._meta import API_VTAG +from simcore_service_storage.core.application import create_app +from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from simcore_service_storage_sdk import ApiClient, Configuration, UsersApi - -pytest_simcore_core_services_selection = [ - "postgres", -] +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_delay, + wait_fixed, +) +from yarl import URL + +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = [ "adminer", ] +_logger = logging.getLogger(__name__) + + +@retry( + wait=wait_fixed(1), + stop=stop_after_delay(10), + retry=retry_if_exception_type(), + reraise=True, + before_sleep=before_sleep_log(_logger, logging.WARNING), +) +async def _wait_for_server_ready(server: URL) -> None: + async with httpx.AsyncClient(follow_redirects=True) as client: + response = await client.get(f"{server}") + response.raise_for_status() + + +@pytest.fixture +async def real_storage_server(app_settings: ApplicationSettings) -> AsyncIterator[URL]: + settings = ApplicationSettings.create_from_envs() + app = create_app(settings) + storage_port = unused_port() + with log_context( + logging.INFO, + msg=f"with fake storage server on 127.0.0.1:{storage_port}/{API_VTAG}", + ) as ctx: + config = uvicorn.Config( + app, + host="127.0.0.1", + port=storage_port, + log_level="error", + ) + server = uvicorn.Server(config) + + thread = Thread(target=server.run) + thread.daemon = True + thread.start() + + ctx.logger.info( + "health at : %s", + f"http://127.0.0.1:{storage_port}/{API_VTAG}", + ) + server_url = URL(f"http://127.0.0.1:{storage_port}") + + await _wait_for_server_ready(server_url / API_VTAG) + + yield server_url + + server.should_exit = True + thread.join(timeout=10) + @pytest.fixture -def user_id(user_id: UserID) -> str: +def str_user_id(user_id: UserID) -> str: """overrides tests/fixtures/data_models.py::user_id and adapts to simcore_service_storage_sdk API """ @@ -47,21 +112,22 @@ def file_id(simcore_file_id: SimcoreS3FileID) -> str: @pytest.fixture -def location_id() -> LocationID: - return SimcoreS3DataManager.get_location_id() - - -@pytest.fixture -def location_name() -> str: +def simcore_location_name() -> str: return SimcoreS3DataManager.get_location_name() -async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 - client: TestClient, +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +async def test_storage_client_used_in_simcore_sdk_0_3_2( + real_storage_server: URL, + str_user_id: str, file_id: str, - user_id: str, location_id: int, - location_name: str, + simcore_location_name: str, tmp_path: Path, faker: Faker, ): @@ -76,14 +142,12 @@ async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 the OAS had already change!!! """ - assert client.app - assert client.server # -------- cfg = Configuration() - cfg.host = f"http://{client.host}:{client.port}/v0" + cfg.host = f"{real_storage_server / API_VTAG}" cfg.debug = True - assert cfg.host == f'{client.make_url("/v0")}' + # assert cfg.host == f"{client.make_url('/v0')}" print(f"{cfg=}") print(f"{cfg.to_debug_report()=}") @@ -101,7 +165,7 @@ async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 response_payload, status_code, response_headers, - ) = await api.get_storage_locations_with_http_info(user_id) + ) = await api.get_storage_locations_with_http_info(str_user_id) print(f"{response_payload=}") print(f"{status_code=}") print(f"{response_headers=}") @@ -112,7 +176,7 @@ async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py#L132 resp_model = await api.upload_file( location_id=location_id, - user_id=user_id, + user_id=str_user_id, file_id=file_id, _request_timeout=1000, ) @@ -138,24 +202,24 @@ async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 # A bug in the response of this call was preventing downloading data # with the new storage API # - resp_model = await api.get_file_metadata(file_id, location_id, user_id) + resp_model = await api.get_file_metadata(file_id, location_id, str_user_id) print(type(resp_model), ":\n", resp_model) assert resp_model.data.object_name is not None assert resp_model.error is None # _get_location_id_from_location_name # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py#L89 - resp_model = await api.get_storage_locations(user_id=user_id) + resp_model = await api.get_storage_locations(user_id=str_user_id) print(f"{resp_model=}") for location in resp_model.data: - assert location["name"] == location_name + assert location["name"] == simcore_location_name assert location["id"] == location_id # _get_download_link # https://github.com/ITISFoundation/osparc-simcore/blob/cfdf4f86d844ebb362f4f39e9c6571d561b72897/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py#L123 resp_model = await api.download_file( location_id=location_id, - user_id=user_id, + user_id=str_user_id, file_id=file_id, _request_timeout=1000, ) diff --git a/services/storage/tests/unit/test__openapi_specs.py b/services/storage/tests/unit/test__openapi_specs.py deleted file mode 100644 index a32ae3ace6e..00000000000 --- a/services/storage/tests/unit/test__openapi_specs.py +++ /dev/null @@ -1,72 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=too-many-arguments -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -from collections.abc import Callable -from pathlib import Path - -import pytest -import simcore_service_storage.application -from aiohttp import web -from faker import Faker -from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict -from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.openapi_specs import Entrypoint -from simcore_service_storage._meta import API_VTAG -from simcore_service_storage.resources import storage_resources -from simcore_service_storage.settings import Settings - - -@pytest.fixture(scope="session") -def openapi_specs_path() -> Path: - # overrides pytest_simcore.openapi_specs.app_openapi_specs_path fixture - spec_path: Path = storage_resources.get_path(f"api/{API_VTAG}/openapi.yaml") - return spec_path - - -@pytest.fixture -def app_environment( - mock_env_devel_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - faker: Faker, -) -> EnvVarsDict: - return mock_env_devel_environment | setenvs_from_dict( - monkeypatch, - { - # disable index and statics routings - "WEBSERVER_STATICWEB": "null", - }, - ) - - -@pytest.fixture -def app(app_environment: EnvVarsDict) -> web.Application: - assert app_environment - # Expects that: - # - routings happen during setup! - # - all plugins are setup but app is NOT started (i.e events are not triggered) - # - settings = Settings.create_from_envs() - return simcore_service_storage.application.create(settings) - - -@pytest.fixture -def app_rest_entrypoints( - app: web.Application, - create_aiohttp_app_rest_entrypoints: Callable[[web.Application], set[Entrypoint]], -) -> set[Entrypoint]: - # check whether exposed routes implements openapi.json contract - return create_aiohttp_app_rest_entrypoints(app) - - -def test_app_named_resources_against_openapi_specs( - openapi_specs_entrypoints: set[Entrypoint], - app_rest_entrypoints: set[Entrypoint], -): - assert app_rest_entrypoints == openapi_specs_entrypoints - - # NOTE: missing here is: - # - input schemas (path, query and body) - # - output schemas (success/error responses and bodies) - # diff --git a/services/storage/tests/unit/test__worker_tasks_paths.py b/services/storage/tests/unit/test__worker_tasks_paths.py new file mode 100644 index 00000000000..c8d508d4129 --- /dev/null +++ b/services/storage/tests/unit/test__worker_tasks_paths.py @@ -0,0 +1,216 @@ +# pylint:disable=no-name-in-module +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=too-many-positional-arguments +# pylint:disable=unused-argument +# pylint:disable=unused-variable + + +import random +from pathlib import Path +from typing import Any, TypeAlias + +import httpx +import pytest +from celery import Celery, Task +from faker import Faker +from fastapi import FastAPI +from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID +from models_library.users import UserID +from pydantic import ByteSize, TypeAdapter +from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams +from simcore_service_storage.api._worker_tasks._paths import compute_path_size +from simcore_service_storage.modules.celery.utils import set_fastapi_app +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager + +pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_ops_services_selection = ["adminer"] + +_IsFile: TypeAlias = bool + + +def _filter_and_group_paths_one_level_deeper( + paths: list[Path], prefix: Path +) -> list[tuple[Path, _IsFile]]: + relative_paths = (path for path in paths if path.is_relative_to(prefix)) + return sorted( + { + ( + (path, len(path.relative_to(prefix).parts) == 1) + if len(path.relative_to(prefix).parts) == 1 + else (prefix / path.relative_to(prefix).parts[0], False) + ) + for path in relative_paths + }, + key=lambda x: x[0], + ) + + +async def _assert_compute_path_size( + celery_task: Task, + location_id: LocationID, + user_id: UserID, + *, + path: Path, + expected_total_size: int, +) -> ByteSize: + response = await compute_path_size( + celery_task, user_id=user_id, location_id=location_id, path=path + ) + assert isinstance(response, ByteSize) + assert response == expected_total_size + return response + + +@pytest.fixture +def fake_celery_task(celery_app: Celery, initialized_app: FastAPI) -> Task: + celery_task = Task() + celery_task.app = celery_app + set_fastapi_app(celery_app, initialized_app) + return celery_task + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=5, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ) + ], + ids=str, +) +async def test_path_compute_size( + fake_celery_task: Task, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + project_params: ProjectWithFilesParams, +): + assert ( + len(project_params.allowed_file_sizes) == 1 + ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" + project, list_of_files = with_random_project_with_files + + total_num_files = sum( + len(files_in_node) for files_in_node in list_of_files.values() + ) + + # get size of a full project + expected_total_size = project_params.allowed_file_sizes[0] * total_num_files + path = Path(project["uuid"]) + await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of one of the nodes + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + path = Path(project["uuid"]) / f"{selected_node_id}" + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of the outputs of one of the nodes + path = Path(project["uuid"]) / f"{selected_node_id}" / "outputs" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of workspace in one of the nodes (this is semi-cached in the DB) + path = Path(project["uuid"]) / f"{selected_node_id}" / "workspace" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + workspace_total_size = await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of folders inside the workspace + folders_inside_workspace = [ + p[0] + for p in _filter_and_group_paths_one_level_deeper(selected_node_s3_keys, path) + if p[1] is False + ] + accumulated_subfolder_size = 0 + for workspace_subfolder in folders_inside_workspace: + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{workspace_subfolder}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + accumulated_subfolder_size += await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=workspace_subfolder, + expected_total_size=expected_total_size, + ) + + assert workspace_total_size == accumulated_subfolder_size + + +async def test_path_compute_size_inexistent_path( + fake_celery_task: Task, + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + faker: Faker, + fake_datcore_tokens: tuple[str, str], +): + await _assert_compute_path_size( + fake_celery_task, + location_id, + user_id, + path=Path(faker.file_path(absolute=False)), + expected_total_size=0, + ) diff --git a/services/storage/tests/unit/test_cli.py b/services/storage/tests/unit/test_cli.py index ad31a85e31f..a81274745c4 100644 --- a/services/storage/tests/unit/test_cli.py +++ b/services/storage/tests/unit/test_cli.py @@ -2,48 +2,34 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import contextlib -import json import os -from io import StringIO -import pytest -from dotenv import dotenv_values +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_storage._meta import API_VERSION from simcore_service_storage.cli import main -from simcore_service_storage.settings import Settings +from simcore_service_storage.core.settings import ApplicationSettings from typer.testing import CliRunner -@pytest.mark.parametrize( - "arguments", ["--help", "run --help".split(), "settings --help".split()] -) -def test_cli_help(arguments: list[str] | str, cli_runner: CliRunner): - result = cli_runner.invoke(main, arguments) - assert result.exit_code == os.EX_OK, result +def test_cli_help_and_version(cli_runner: CliRunner): + result = cli_runner.invoke(main, "--help") + assert result.exit_code == os.EX_OK, result.output + result = cli_runner.invoke(main, "--version") + assert result.exit_code == os.EX_OK, result.output + assert result.stdout.strip() == API_VERSION -def test_cli_settings_as_json( - project_env_devel_environment: None, cli_runner: CliRunner -): - result = cli_runner.invoke(main, ["settings", "--as-json"]) - assert result.exit_code == os.EX_OK, result - # reuse resulting json to build settings - settings: dict = json.loads(result.stdout) - assert Settings(settings) +def test_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): + result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"]) + assert result.exit_code == os.EX_OK -def test_cli_settings_env_file( - project_env_devel_environment: None, cli_runner: CliRunner -): - result = cli_runner.invoke(main, ["settings", "--compact"]) - assert result.exit_code == os.EX_OK, result + print(result.output) + settings = ApplicationSettings(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() - # reuse resulting env_file to build settings - env_file = StringIO(result.stdout) - settings = dotenv_values(stream=env_file) - for key, value in settings.items(): - with contextlib.suppress(json.decoder.JSONDecodeError): - settings[key] = json.loads(str(value)) - - assert Settings(settings) +def test_run(cli_runner: CliRunner): + result = cli_runner.invoke(main, ["run"]) + assert result.exit_code == 0 + assert "disabled" in result.stdout diff --git a/services/storage/tests/unit/test_data_export.py b/services/storage/tests/unit/test_data_export.py new file mode 100644 index 00000000000..05c0f99a176 --- /dev/null +++ b/services/storage/tests/unit/test_data_export.py @@ -0,0 +1,601 @@ +# pylint: disable=W0621 +# pylint: disable=W0613 +# pylint: disable=R6301 +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, NamedTuple +from uuid import UUID + +import pytest +from celery.exceptions import CeleryError +from faker import Faker +from fastapi import FastAPI +from models_library.api_schemas_long_running_tasks.tasks import TaskResult +from models_library.api_schemas_rpc_async_jobs.async_jobs import ( + AsyncJobGet, + AsyncJobId, + AsyncJobNameData, + AsyncJobResult, + AsyncJobStatus, +) +from models_library.api_schemas_rpc_async_jobs.exceptions import ( + JobAbortedError, + JobError, + JobMissingError, + JobNotDoneError, + JobSchedulerError, +) +from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE +from models_library.api_schemas_storage.data_export_async_jobs import ( + DataExportTaskStartInput, +) +from models_library.progress_bar import ProgressReport +from models_library.projects_nodes_io import NodeID, SimcoreS3FileID +from models_library.users import UserID +from pydantic import ByteSize, TypeAdapter +from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams +from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs +from servicelib.rabbitmq.rpc_interfaces.storage.data_export import start_data_export +from settings_library.rabbit import RabbitSettings +from simcore_service_storage.api.rpc._data_export import AccessRightError +from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.modules.celery.client import TaskUUID +from simcore_service_storage.modules.celery.models import TaskState, TaskStatus +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager + +pytest_plugins = [ + "pytest_simcore.rabbit_service", +] + + +pytest_simcore_core_services_selection = [ + "rabbit", + "postgres", +] + +_faker = Faker() + + +@dataclass +class _MockCeleryClient: + send_task_object: UUID | Exception | None = None + get_task_status_object: TaskStatus | Exception | None = None + get_task_result_object: TaskResult | Exception | None = None + get_task_uuids_object: set[UUID] | Exception | None = None + abort_task_object: Exception | None = None + + async def send_task(self, *args, **kwargs) -> TaskUUID: + assert self.send_task_object is not None + if isinstance(self.send_task_object, Exception): + raise self.send_task_object + return self.send_task_object + + async def get_task_status(self, *args, **kwargs) -> TaskStatus: + assert self.get_task_status_object is not None + if isinstance(self.get_task_status_object, Exception): + raise self.get_task_status_object + return self.get_task_status_object + + async def get_task_result(self, *args, **kwargs) -> Any: + assert self.get_task_result_object is not None + if isinstance(self.get_task_result_object, Exception): + raise self.get_task_result_object + return self.get_task_result_object + + async def get_task_uuids(self, *args, **kwargs) -> set[TaskUUID]: + assert self.get_task_uuids_object is not None + if isinstance(self.get_task_uuids_object, Exception): + raise self.get_task_uuids_object + return self.get_task_uuids_object + + async def abort_task(self, *args, **kwargs) -> None: + if isinstance(self.abort_task_object, Exception): + raise self.abort_task_object + return self.abort_task_object + + +@pytest.fixture +async def mock_celery_client( + mocker: MockerFixture, + request: pytest.FixtureRequest, +) -> _MockCeleryClient: + params = request.param if hasattr(request, "param") else {} + _celery_client = _MockCeleryClient( + send_task_object=params.get("send_task_object", None), + get_task_status_object=params.get("get_task_status_object", None), + get_task_result_object=params.get("get_task_result_object", None), + get_task_uuids_object=params.get("get_task_uuids_object", None), + abort_task_object=params.get("abort_task_object", None), + ) + mocker.patch( + "simcore_service_storage.api.rpc._async_jobs.get_celery_client", + return_value=_celery_client, + ) + mocker.patch( + "simcore_service_storage.api.rpc._data_export.get_celery_client", + return_value=_celery_client, + ) + return _celery_client + + +@pytest.fixture +async def app_environment( + app_environment: EnvVarsDict, + rabbit_service: RabbitSettings, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.STORAGE_RABBITMQ + + return new_envs + + +@pytest.fixture +async def rpc_client( + initialized_app: FastAPI, + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +class UserWithFile(NamedTuple): + user: UserID + file: Path + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params,selection_type", + [ + ( + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + "file", + ), + ( + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + "folder", + ), + ], + ids=str, +) +@pytest.mark.parametrize( + "mock_celery_client", + [ + {"send_task_object": TaskUUID(_faker.uuid4())}, + ], + indirect=True, +) +async def test_start_data_export_success( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + user_id: UserID, + selection_type: Literal["file", "folder"], +): + _, list_of_files = with_random_project_with_files + workspace_files = [ + p for p in next(iter(list_of_files.values())) if "/workspace/" in p + ] + assert len(workspace_files) > 0 + file_or_folder_id: SimcoreS3FileID + if selection_type == "file": + file_or_folder_id = workspace_files[0] + elif selection_type == "folder": + parts = Path(workspace_files[0]).parts + parts = parts[0 : parts.index("workspace") + 1] + assert len(parts) > 0 + folder = Path(*parts) + assert folder.name == "workspace" + file_or_folder_id = f"{folder}" + else: + pytest.fail(f"invalid parameter: {selection_type=}") + + result = await start_data_export( + rpc_client, + job_id_data=AsyncJobNameData(user_id=user_id, product_name="osparc"), + data_export_start=DataExportTaskStartInput( + location_id=0, + file_and_folder_ids=[file_or_folder_id], + ), + ) + assert isinstance(result, AsyncJobGet) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + ], + ids=str, +) +@pytest.mark.parametrize( + "mock_celery_client", + [ + {"send_task_object": CeleryError("error")}, + ], + indirect=True, +) +async def test_start_data_export_scheduler_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + user_id: UserID, +): + + _, list_of_files = with_random_project_with_files + workspace_files = [ + p for p in list(list_of_files.values())[0].keys() if "/workspace/" in p + ] + assert len(workspace_files) > 0 + file_or_folder_id = workspace_files[0] + + with pytest.raises(JobSchedulerError): + _ = await start_data_export( + rpc_client, + job_id_data=AsyncJobNameData(user_id=user_id, product_name="osparc"), + data_export_start=DataExportTaskStartInput( + location_id=0, + file_and_folder_ids=[file_or_folder_id], + ), + ) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + {"send_task_object": TaskUUID(_faker.uuid4())}, + ], + indirect=True, +) +async def test_start_data_export_access_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + user_id: UserID, + faker: Faker, +): + with pytest.raises(AccessRightError): + _ = await async_jobs.submit( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + method_name="start_data_export", + job_id_data=AsyncJobNameData(user_id=user_id, product_name="osparc"), + data_export_start=DataExportTaskStartInput( + location_id=0, + file_and_folder_ids=[ + f"{faker.uuid4()}/{faker.uuid4()}/{faker.file_name()}" + ], + ), + ) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + { + "abort_task_object": None, + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + ], + indirect=True, +) +async def test_abort_data_export_success( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, +): + assert mock_celery_client.get_task_uuids_object is not None + assert not isinstance(mock_celery_client.get_task_uuids_object, Exception) + await async_jobs.cancel( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + job_id=next(iter(mock_celery_client.get_task_uuids_object)), + ) + + +@pytest.mark.parametrize( + "mock_celery_client, expected_exception_type", + [ + ({"abort_task_object": None, "get_task_uuids_object": []}, JobMissingError), + ( + { + "abort_task_object": CeleryError("error"), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobSchedulerError, + ), + ], + indirect=["mock_celery_client"], +) +async def test_abort_data_export_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + expected_exception_type: type[Exception], +): + job_ids = mock_celery_client.get_task_uuids_object + assert job_ids is not None + assert not isinstance(job_ids, Exception) + _job_id = next(iter(job_ids)) if len(job_ids) > 0 else AsyncJobId(_faker.uuid4()) + with pytest.raises(expected_exception_type): + await async_jobs.cancel( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + job_id=_job_id, + ) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + { + "get_task_status_object": TaskStatus( + task_uuid=TaskUUID(_faker.uuid4()), + task_state=TaskState.RUNNING, + progress_report=ProgressReport(actual_value=0), + ), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + ], + indirect=True, +) +async def test_get_data_export_status( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, +): + job_ids = mock_celery_client.get_task_uuids_object + assert job_ids is not None + assert not isinstance(job_ids, Exception) + _job_id = next(iter(job_ids)) if len(job_ids) > 0 else AsyncJobId(_faker.uuid4()) + result = await async_jobs.status( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id=_job_id, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + ) + assert isinstance(result, AsyncJobStatus) + assert result.job_id == _job_id + + +@pytest.mark.parametrize( + "mock_celery_client, expected_exception_type", + [ + ( + {"get_task_status_object": None, "get_task_uuids_object": []}, + JobMissingError, + ), + ( + { + "get_task_status_object": CeleryError("error"), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobSchedulerError, + ), + ], + indirect=["mock_celery_client"], +) +async def test_get_data_export_status_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + expected_exception_type: type[Exception], +): + job_ids = mock_celery_client.get_task_uuids_object + assert job_ids is not None + assert not isinstance(job_ids, Exception) + _job_id = next(iter(job_ids)) if len(job_ids) > 0 else AsyncJobId(_faker.uuid4()) + with pytest.raises(expected_exception_type): + _ = await async_jobs.status( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id=_job_id, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + ) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + { + "get_task_status_object": TaskStatus( + task_uuid=TaskUUID(_faker.uuid4()), + task_state=TaskState.SUCCESS, + progress_report=ProgressReport(actual_value=100), + ), + "get_task_result_object": "result", + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + ], + indirect=True, +) +async def test_get_data_export_result_success( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, +): + job_ids = mock_celery_client.get_task_uuids_object + assert job_ids is not None + assert not isinstance(job_ids, Exception) + _job_id = next(iter(job_ids)) if len(job_ids) > 0 else AsyncJobId(_faker.uuid4()) + result = await async_jobs.result( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id=_job_id, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + ) + assert isinstance(result, AsyncJobResult) + + +@pytest.mark.parametrize( + "mock_celery_client, expected_exception", + [ + ( + { + "get_task_status_object": TaskStatus( + task_uuid=TaskUUID(_faker.uuid4()), + task_state=TaskState.RUNNING, + progress_report=ProgressReport(actual_value=50), + ), + "get_task_result_object": _faker.text(), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobNotDoneError, + ), + ( + { + "get_task_status_object": TaskStatus( + task_uuid=TaskUUID(_faker.uuid4()), + task_state=TaskState.ABORTED, + progress_report=ProgressReport(actual_value=100), + ), + "get_task_result_object": _faker.text(), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobAbortedError, + ), + ( + { + "get_task_status_object": TaskStatus( + task_uuid=TaskUUID(_faker.uuid4()), + task_state=TaskState.ERROR, + progress_report=ProgressReport(actual_value=100), + ), + "get_task_result_object": _faker.text(), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobError, + ), + ( + { + "get_task_status_object": CeleryError("error"), + "get_task_result_object": _faker.text(), + "get_task_uuids_object": [AsyncJobId(_faker.uuid4())], + }, + JobSchedulerError, + ), + ( + { + "get_task_uuids_object": [], + }, + JobMissingError, + ), + ], + indirect=["mock_celery_client"], +) +async def test_get_data_export_result_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: _MockCeleryClient, + expected_exception: type[Exception], +): + job_ids = mock_celery_client.get_task_uuids_object + assert job_ids is not None + assert not isinstance(job_ids, Exception) + _job_id = next(iter(job_ids)) if len(job_ids) > 0 else AsyncJobId(_faker.uuid4()) + + with pytest.raises(expected_exception): + _ = await async_jobs.result( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id=_job_id, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + ) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + {"get_task_uuids_object": [_faker.uuid4() for _ in range(_faker.pyint(1, 10))]}, + ], + indirect=True, +) +async def test_list_jobs_success( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): + result = await async_jobs.list_jobs( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + filter_="", + ) + assert isinstance(result, list) + assert all(isinstance(elm, AsyncJobGet) for elm in result) + + +@pytest.mark.parametrize( + "mock_celery_client", + [ + {"get_task_uuids_object": CeleryError("error")}, + ], + indirect=True, +) +async def test_list_jobs_error( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): + with pytest.raises(JobSchedulerError): + _ = await async_jobs.list_jobs( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + filter_="", + ) diff --git a/services/storage/tests/unit/test_db_access_layer.py b/services/storage/tests/unit/test_db_access_layer.py index 452e09e1ead..4c268f38fb8 100644 --- a/services/storage/tests/unit/test_db_access_layer.py +++ b/services/storage/tests/unit/test_db_access_layer.py @@ -6,42 +6,50 @@ import pytest import sqlalchemy as sa -from aiopg.sa.engine import Engine from models_library.projects import ProjectID from models_library.users import UserID from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users from simcore_postgres_database.models.workspaces import workspaces -from simcore_service_storage.db_access_layer import ( +from simcore_service_storage.modules.db.access_layer import ( + AccessLayerRepository, AccessRights, - get_file_access_rights, - get_project_access_rights, ) +from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_ops_services_selection = [ + "adminer", +] async def test_access_rights_on_owned_project( - user_id: UserID, project_id: ProjectID, aiopg_engine: Engine + user_id: UserID, project_id: ProjectID, sqlalchemy_async_engine: AsyncEngine ): - async with aiopg_engine.acquire() as conn: - access = await get_project_access_rights(conn, user_id, project_id) - assert access == AccessRights.all() + access = await AccessLayerRepository.instance( + sqlalchemy_async_engine + ).get_project_access_rights(user_id=user_id, project_id=project_id) + assert access == AccessRights.all() - # still NOT registered in file_meta_data BUT with prefix {project_id} owned by user - access = await get_file_access_rights( - conn, user_id, f"{project_id}/node_id/not-in-file-metadata-table.txt" - ) - assert access == AccessRights.all() + # still NOT registered in file_meta_data BUT with prefix {project_id} owned by user + access = await AccessLayerRepository.instance( + sqlalchemy_async_engine + ).get_file_access_rights( + user_id=user_id, + file_id=f"{project_id}/node_id/not-in-file-metadata-table.txt", + ) + assert access == AccessRights.all() @pytest.fixture -async def prepare_db(user_id: UserID, project_id: ProjectID, aiopg_engine: Engine): - async with aiopg_engine.acquire() as conn: +async def prepare_db( + user_id: UserID, project_id: ProjectID, sqlalchemy_async_engine: AsyncEngine +): + async with sqlalchemy_async_engine.connect() as conn: result = await conn.execute( sa.select(users.c.primary_gid).where(users.c.id == user_id) ) - row = await result.first() + row = result.one() user_primary_id = row[0] result = await conn.execute( @@ -57,7 +65,7 @@ async def prepare_db(user_id: UserID, project_id: ProjectID, aiopg_engine: Engin ) .returning(workspaces.c.workspace_id) ) - row = await result.first() + row = result.one() workspace_id = row[0] await conn.execute( @@ -72,14 +80,20 @@ async def prepare_db(user_id: UserID, project_id: ProjectID, aiopg_engine: Engin async def test_access_rights_based_on_workspace( - user_id: UserID, project_id: ProjectID, aiopg_engine: Engine, prepare_db + user_id: UserID, + project_id: ProjectID, + sqlalchemy_async_engine: AsyncEngine, + prepare_db, ): - async with aiopg_engine.acquire() as conn: - access = await get_project_access_rights(conn, user_id, project_id) - assert access == AccessRights.all() + access = await AccessLayerRepository.instance( + sqlalchemy_async_engine + ).get_project_access_rights(user_id=user_id, project_id=project_id) + assert access == AccessRights.all() - # still NOT registered in file_meta_data BUT with prefix {project_id} owned by user - access = await get_file_access_rights( - conn, user_id, f"{project_id}/node_id/not-in-file-metadata-table.txt" - ) - assert access == AccessRights.all() + # still NOT registered in file_meta_data BUT with prefix {project_id} owned by user + access = await AccessLayerRepository.instance( + sqlalchemy_async_engine + ).get_file_access_rights( + user_id=user_id, file_id=f"{project_id}/node_id/not-in-file-metadata-table.txt" + ) + assert access == AccessRights.all() diff --git a/services/storage/tests/unit/test_db_file_meta_data.py b/services/storage/tests/unit/test_db_file_meta_data.py index c362fabe82f..5da3b9b0692 100644 --- a/services/storage/tests/unit/test_db_file_meta_data.py +++ b/services/storage/tests/unit/test_db_file_meta_data.py @@ -8,10 +8,10 @@ from faker import Faker from simcore_postgres_database.utils import as_postgres_sql_query_str -from simcore_service_storage.db_file_meta_data import ( +from simcore_service_storage.models import UserOrProjectFilter +from simcore_service_storage.modules.db.file_meta_data import ( _list_filter_with_partial_file_id_stmt, ) -from simcore_service_storage.models import UserOrProjectFilter def test_building_sql_statements(faker: Faker): diff --git a/services/storage/tests/unit/test_dsm.py b/services/storage/tests/unit/test_dsm.py index 22c78955581..e99dfda1916 100644 --- a/services/storage/tests/unit/test_dsm.py +++ b/services/storage/tests/unit/test_dsm.py @@ -12,9 +12,8 @@ from models_library.users import UserID from pydantic import ByteSize, TypeAdapter from servicelib.utils import limited_gather -from simcore_service_storage.models import FileMetaData, S3BucketName +from simcore_service_storage.models import FileMetaData from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -from types_aiobotocore_s3 import S3Client pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] @@ -43,31 +42,3 @@ async def dsm_mockup_complete_db( assert len(fmds) == 2 return (fmds[0], fmds[1]) - - -async def test_sync_table_meta_data( - simcore_s3_dsm: SimcoreS3DataManager, - dsm_mockup_complete_db: tuple[FileMetaData, FileMetaData], - storage_s3_bucket: S3BucketName, - s3_client: S3Client, -): - expected_removed_files = [] - # the list should be empty on start - list_changes = await simcore_s3_dsm.synchronise_meta_data_table(dry_run=True) - assert list_changes == expected_removed_files - - # now remove the files - for file_entry in dsm_mockup_complete_db: - s3_key = f"{file_entry.project_id}/{file_entry.node_id}/{file_entry.file_name}" - await s3_client.delete_object(Bucket=storage_s3_bucket, Key=s3_key) - expected_removed_files.append(s3_key) - - # the list should now contain the removed entries - list_changes = await simcore_s3_dsm.synchronise_meta_data_table(dry_run=True) - assert set(list_changes) == set(expected_removed_files) - - # now effectively call the function should really remove the files - list_changes = await simcore_s3_dsm.synchronise_meta_data_table(dry_run=False) - # listing again will show an empty list again - list_changes = await simcore_s3_dsm.synchronise_meta_data_table(dry_run=True) - assert list_changes == [] diff --git a/services/storage/tests/unit/test_dsm_cleaner.py b/services/storage/tests/unit/test_dsm_cleaner.py index 70134dbdb88..3e5f065a7f8 100644 --- a/services/storage/tests/unit/test_dsm_cleaner.py +++ b/services/storage/tests/unit/test_dsm_cleaner.py @@ -6,7 +6,7 @@ from unittest import mock import pytest -from aiohttp.test_utils import TestClient +from fastapi import FastAPI from pytest_mock import MockerFixture from simcore_service_storage.dsm_cleaner import _TASK_NAME_PERIODICALY_CLEAN_DSM @@ -34,7 +34,7 @@ def short_dsm_cleaner_interval(monkeypatch: pytest.MonkeyPatch) -> int: return 1 -async def test_setup_dsm_cleaner(client: TestClient): +async def test_setup_dsm_cleaner(initialized_app: FastAPI): all_tasks = asyncio.all_tasks() assert any( t.get_name().startswith(f"{_TASK_NAME_PERIODICALY_CLEAN_DSM}") @@ -42,7 +42,7 @@ async def test_setup_dsm_cleaner(client: TestClient): ) -async def test_disable_dsm_cleaner(disable_dsm_cleaner, client: TestClient): +async def test_disable_dsm_cleaner(disable_dsm_cleaner, initialized_app: FastAPI): all_tasks = asyncio.all_tasks() assert not any( t.get_name().startswith(f"{_TASK_NAME_PERIODICALY_CLEAN_DSM}") @@ -51,7 +51,9 @@ async def test_disable_dsm_cleaner(disable_dsm_cleaner, client: TestClient): async def test_dsm_cleaner_task_restarts_if_error( - mocked_dsm_clean: mock.Mock, short_dsm_cleaner_interval: int, client: TestClient + mocked_dsm_clean: mock.Mock, + short_dsm_cleaner_interval: int, + initialized_app: FastAPI, ): num_calls = mocked_dsm_clean.call_count await asyncio.sleep(short_dsm_cleaner_interval + 1) diff --git a/services/storage/tests/unit/test_dsm_dsmcleaner.py b/services/storage/tests/unit/test_dsm_dsmcleaner.py index 1683a9d0a0d..2f48b85bd03 100644 --- a/services/storage/tests/unit/test_dsm_dsmcleaner.py +++ b/services/storage/tests/unit/test_dsm_dsmcleaner.py @@ -13,27 +13,28 @@ from pathlib import Path from typing import Final -import arrow import pytest -from aiopg.sa.engine import Engine from aws_library.s3 import MultiPartUploadLinks, SimcoreS3API from faker import Faker -from models_library.api_schemas_storage import LinkType +from models_library.api_schemas_storage.storage_schemas import LinkType from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import SimcoreS3DirectoryID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.parametrizations import byte_size_ids -from simcore_postgres_database.storage_models import file_meta_data -from simcore_service_storage import db_file_meta_data -from simcore_service_storage.exceptions import ( +from simcore_postgres_database.storage_models import ( + file_meta_data as file_meta_data_table, +) +from simcore_service_storage.exceptions.errors import ( FileAccessRightError, FileMetaDataNotFoundError, ) from simcore_service_storage.models import FileMetaData, S3BucketName, UploadID +from simcore_service_storage.modules.db.file_meta_data import FileMetaDataRepository from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy.ext.asyncio import AsyncEngine -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = ["adminer"] _faker: Faker = Faker() @@ -72,7 +73,7 @@ def simcore_directory_id(simcore_file_id: SimcoreS3FileID) -> SimcoreS3FileID: @pytest.mark.parametrize("checksum", [None, _faker.sha256()]) async def test_regression_collaborator_creates_file_upload_links( # pylint:disable=too-many-positional-arguments disabled_dsm_cleaner_task, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, simcore_s3_dsm: SimcoreS3DataManager, simcore_file_id: SimcoreS3FileID, simcore_directory_id: SimcoreS3FileID, @@ -142,7 +143,7 @@ async def test_regression_collaborator_creates_file_upload_links( # pylint:disa @pytest.mark.parametrize("checksum", [None, _faker.sha256()]) async def test_clean_expired_uploads_deletes_expired_pending_uploads( disabled_dsm_cleaner_task, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, simcore_s3_dsm: SimcoreS3DataManager, simcore_file_id: SimcoreS3FileID, simcore_directory_id: SimcoreS3FileID, @@ -168,8 +169,9 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( is_directory=is_directory, ) # ensure the database is correctly set up - async with aiopg_engine.acquire() as conn: - fmd = await db_file_meta_data.get(conn, file_or_directory_id) + fmd = await FileMetaDataRepository.instance(sqlalchemy_async_engine).get( + file_id=file_or_directory_id + ) assert fmd assert fmd.upload_expires_at # ensure we have now an upload id IF the link was presigned ONLY @@ -184,8 +186,10 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( # now run the cleaner, nothing should happen since the expiration was set to the default of 3600 await simcore_s3_dsm.clean_expired_uploads() # check the entries are still the same - async with aiopg_engine.acquire() as conn: - fmd_after_clean = await db_file_meta_data.get(conn, file_or_directory_id) + async with sqlalchemy_async_engine.connect() as conn: + fmd_after_clean = await FileMetaDataRepository.instance( + sqlalchemy_async_engine + ).get(file_id=file_or_directory_id) assert fmd_after_clean == fmd assert ( await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) @@ -193,25 +197,33 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( ) # now change the upload_expires_at entry to simulate and expired entry - async with aiopg_engine.acquire() as conn: + async with sqlalchemy_async_engine.begin() as conn: await conn.execute( - file_meta_data.update() - .where(file_meta_data.c.file_id == file_or_directory_id) - .values(upload_expires_at=arrow.utcnow().datetime) + file_meta_data_table.update() + .where(file_meta_data_table.c.file_id == file_or_directory_id) + .values(upload_expires_at=datetime.datetime.utcnow()) ) - await asyncio.sleep(1) + await asyncio.sleep(5) await simcore_s3_dsm.clean_expired_uploads() # check the entries were removed - async with aiopg_engine.acquire() as conn: + async with sqlalchemy_async_engine.connect() as conn: with pytest.raises(FileMetaDataNotFoundError): - await db_file_meta_data.get(conn, simcore_file_id) + await FileMetaDataRepository.instance(sqlalchemy_async_engine).get( + file_id=simcore_file_id + ) # since there is no entry in the db, this upload shall be cleaned up assert not await storage_s3_client.list_ongoing_multipart_uploads( bucket=storage_s3_bucket ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -228,7 +240,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi ..., Awaitable[tuple[Path, SimcoreS3FileID]], ], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, simcore_s3_dsm: SimcoreS3DataManager, user_id: UserID, link_type: LinkType, @@ -247,8 +259,10 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi file_id=None, sha256_checksum=checksum, ) - async with aiopg_engine.acquire() as conn: - original_fmd = await db_file_meta_data.get(conn, file_id) + async with sqlalchemy_async_engine.connect() as conn: + original_fmd = await FileMetaDataRepository.instance( + sqlalchemy_async_engine + ).get(file_id=file_id) # now create a new link to the VERY SAME FILE UUID await simcore_s3_dsm.create_file_upload_links( @@ -260,8 +274,10 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi is_directory=False, ) # ensure the database is correctly set up - async with aiopg_engine.acquire() as conn: - fmd = await db_file_meta_data.get(conn, file_id) + async with sqlalchemy_async_engine.connect() as conn: + fmd = await FileMetaDataRepository.instance(sqlalchemy_async_engine).get( + file_id=file_id + ) assert fmd assert fmd.upload_expires_at # ensure we have now an upload id IF the link was presigned ONLY @@ -276,8 +292,9 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi # now run the cleaner, nothing should happen since the expiration was set to the default of 3600 await simcore_s3_dsm.clean_expired_uploads() # check the entries are still the same - async with aiopg_engine.acquire() as conn: - fmd_after_clean = await db_file_meta_data.get(conn, file_id) + fmd_after_clean = await FileMetaDataRepository.instance( + sqlalchemy_async_engine + ).get(file_id=file_id) assert fmd_after_clean == fmd assert ( await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) @@ -285,18 +302,19 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi ) # now change the upload_expires_at entry to simulate an expired entry - async with aiopg_engine.acquire() as conn: + async with sqlalchemy_async_engine.begin() as conn: await conn.execute( - file_meta_data.update() - .where(file_meta_data.c.file_id == file_id) - .values(upload_expires_at=arrow.utcnow().datetime) + file_meta_data_table.update() + .where(file_meta_data_table.c.file_id == file_id) + .values(upload_expires_at=datetime.datetime.utcnow()) ) await asyncio.sleep(1) await simcore_s3_dsm.clean_expired_uploads() # check the entries were reverted - async with aiopg_engine.acquire() as conn: - reverted_fmd = await db_file_meta_data.get(conn, file_id) + reverted_fmd = await FileMetaDataRepository.instance(sqlalchemy_async_engine).get( + file_id=file_id + ) assert original_fmd.model_dump(exclude={"created_at"}) == reverted_fmd.model_dump( exclude={"created_at"} ) @@ -320,7 +338,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi @pytest.mark.parametrize("checksum", [_faker.sha256(), None]) async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation( disabled_dsm_cleaner_task, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, simcore_s3_dsm: SimcoreS3DataManager, simcore_file_id: SimcoreS3FileID, simcore_directory_id: SimcoreS3FileID, @@ -335,7 +353,7 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation the cleaner in between to ensure the cleaner does not break the mechanism""" file_or_directory_id = simcore_directory_id if is_directory else simcore_file_id - later_than_now = arrow.utcnow().datetime + datetime.timedelta(minutes=5) + later_than_now = datetime.datetime.utcnow() + datetime.timedelta(minutes=5) fmd = FileMetaData.from_simcore_node( user_id, file_or_directory_id, @@ -347,11 +365,12 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation sha256_checksum=checksum, ) # we create the entry in the db - async with aiopg_engine.acquire() as conn: - await db_file_meta_data.upsert(conn, fmd) + await FileMetaDataRepository.instance(sqlalchemy_async_engine).upsert(fmd=fmd) - # ensure the database is correctly set up - fmd_in_db = await db_file_meta_data.get(conn, file_or_directory_id) + # ensure the database is correctly set up + fmd_in_db = await FileMetaDataRepository.instance(sqlalchemy_async_engine).get( + file_id=file_or_directory_id + ) assert fmd_in_db assert fmd_in_db.upload_expires_at # we create the multipart upload link @@ -388,9 +407,9 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation assert len(started_multipart_uploads_upload_id) == len(file_ids_to_upload) # ensure we have now an upload id - all_ongoing_uploads: list[ - tuple[UploadID, SimcoreS3FileID] - ] = await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + all_ongoing_uploads: list[tuple[UploadID, SimcoreS3FileID]] = ( + await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + ) assert len(all_ongoing_uploads) == len(file_ids_to_upload) for ongoing_upload_id, ongoing_file_id in all_ongoing_uploads: diff --git a/services/storage/tests/unit/test_dsm_soft_links.py b/services/storage/tests/unit/test_dsm_soft_links.py index dd822ea2165..aa2d1be9161 100644 --- a/services/storage/tests/unit/test_dsm_soft_links.py +++ b/services/storage/tests/unit/test_dsm_soft_links.py @@ -4,62 +4,15 @@ import uuid from functools import lru_cache -from typing import AsyncIterator -import pytest -from aiopg.sa.engine import Engine -from faker import Faker -from models_library.api_schemas_storage import S3BucketName from models_library.projects_nodes_io import SimcoreS3FileID -from models_library.users import UserID -from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, TypeAdapter -from simcore_postgres_database.storage_models import file_meta_data -from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB +from simcore_service_storage.models import FileMetaData from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -from sqlalchemy.sql.expression import literal_column pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] -@pytest.fixture() -async def output_file( - user_id: UserID, project_id: str, aiopg_engine: Engine, faker: Faker -) -> AsyncIterator[FileMetaData]: - node_id = "fd6f9737-1988-341b-b4ac-0614b646fa82" - - # pylint: disable=no-value-for-parameter - - file = FileMetaData.from_simcore_node( - user_id=user_id, - file_id=SimcoreS3FileID(f"{project_id}/{node_id}/filename.txt"), - bucket=TypeAdapter(S3BucketName).validate_python("master-simcore"), - location_id=SimcoreS3DataManager.get_location_id(), - location_name=SimcoreS3DataManager.get_location_name(), - sha256_checksum=faker.sha256(), - ) - file.entity_tag = "df9d868b94e53d18009066ca5cd90e9f" - file.file_size = ByteSize(12) - file.user_id = user_id - - async with aiopg_engine.acquire() as conn: - stmt = ( - file_meta_data.insert() - .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) - .returning(literal_column("*")) - ) - result = await conn.execute(stmt) - row = await result.fetchone() - assert row - - yield file - - result = await conn.execute( - file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) - ) - - def create_reverse_dns(*resource_name_parts) -> str: """ Returns a name for the resource following the reverse domain name notation diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index d207005c2d2..5808a63f1f1 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -4,76 +4,95 @@ # pylint:disable=too-many-arguments # pylint:disable=no-name-in-module - from collections.abc import Awaitable, Callable from pathlib import Path import pytest -from aiohttp.test_utils import TestClient from faker import Faker -from models_library.api_schemas_storage import DatasetMetaDataGet, FileMetaDataGet +from fastapi import FastAPI +from httpx import AsyncClient +from models_library.api_schemas_storage.storage_schemas import ( + DatasetMetaDataGet, + FileMetaDataGet, +) from models_library.projects import ProjectID -from models_library.projects_nodes_io import SimcoreS3FileID +from models_library.projects_nodes_io import LocationID, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, TypeAdapter +from pydantic import ByteSize from pytest_mock import MockerFixture -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.parametrizations import ( byte_size_ids, parametrized_file_size, ) from servicelib.aiohttp import status +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] -async def test_get_files_metadata_dataset_with_no_files_returns_empty_array( - client: TestClient, +async def test_list_dataset_files_metadata_with_no_files_returns_empty_array( + initialized_app: FastAPI, + client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, + fake_datcore_tokens: tuple[str, str], ): - assert client.app - url = ( - client.app.router["get_files_metadata_dataset"] - .url_for(location_id=f"{location_id}", dataset_id=f"{project_id}") - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "list_dataset_files_metadata", + location_id=location_id, + dataset_id=project_id, + ).with_query(user_id=user_id) + response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + data, error = assert_status(response, status.HTTP_200_OK, list[FileMetaDataGet]) assert data == [] assert not error +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [parametrized_file_size("100Mib")], ids=byte_size_ids, ) -async def test_get_files_metadata_dataset( +async def test_list_dataset_files_metadata( upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - client: TestClient, + initialized_app: FastAPI, + client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, file_size: ByteSize, faker: Faker, ): - assert client.app NUM_FILES = 3 for n in range(NUM_FILES): file, file_id = await upload_file(file_size, faker.file_name()) - url = ( - client.app.router["get_files_metadata_dataset"] - .url_for(location_id=f"{location_id}", dataset_id=f"{project_id}") - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "list_dataset_files_metadata", + location_id=location_id, + dataset_id=project_id, + ).with_query(user_id=user_id) + response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) + assert list_fmds assert len(list_fmds) == (n + 1) fmd = list_fmds[n] assert fmd.file_name == file.name @@ -82,48 +101,64 @@ async def test_get_files_metadata_dataset( assert fmd.file_size == file.stat().st_size -async def test_get_datasets_metadata( - client: TestClient, +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +async def test_list_datasets_metadata( + initialized_app: FastAPI, + client: AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, ): - assert client.app - - url = ( - client.app.router["get_datasets_metadata"] - .url_for(location_id=f"{location_id}") - .with_query(user_id=f"{user_id}") - ) + url = url_from_operation_id( + client, + initialized_app, + "list_datasets_metadata", + location_id=location_id, + ).with_query(user_id=user_id) response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data - assert not error - list_datasets = TypeAdapter(list[DatasetMetaDataGet]).validate_python(data) + list_datasets, error = assert_status( + response, status.HTTP_200_OK, list[DatasetMetaDataGet] + ) + assert response.status_code == status.HTTP_200_OK + assert list_datasets assert len(list_datasets) == 1 dataset = list_datasets[0] assert dataset.dataset_id == project_id +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_ensure_expand_dirs_defaults_true( mocker: MockerFixture, - client: TestClient, + initialized_app: FastAPI, + client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, ): mocked_object = mocker.patch( "simcore_service_storage.simcore_s3_dsm.SimcoreS3DataManager.list_files_in_dataset", autospec=True, ) - assert client.app - url = ( - client.app.router["get_files_metadata_dataset"] - .url_for(location_id=f"{location_id}", dataset_id=f"{project_id}") - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "list_dataset_files_metadata", + location_id=location_id, + dataset_id=project_id, + ).with_query(user_id=user_id) + await client.get(f"{url}") assert len(mocked_object.call_args_list) == 1 diff --git a/services/storage/tests/unit/test_handlers_datcore.py b/services/storage/tests/unit/test_handlers_datcore.py new file mode 100644 index 00000000000..a7162610031 --- /dev/null +++ b/services/storage/tests/unit/test_handlers_datcore.py @@ -0,0 +1,48 @@ +import httpx +import pytest +from fastapi import FastAPI, status +from models_library.projects_nodes_io import LocationID +from models_library.users import UserID +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from simcore_service_storage.datcore_dsm import DatCoreDataManager + +pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_ops_services_selection = ["adminer"] + + +@pytest.mark.parametrize( + "entrypoint", + [ + "list_datasets_metadata", + # "list_dataset_files_metadata", needs dataset_id + "list_files_metadata", + # "get_file_metadata", needs file_id + # "download_file", needs file_id + "list_paths", + ], +) +@pytest.mark.parametrize( + "location_id", + [DatCoreDataManager.get_location_id()], + ids=[DatCoreDataManager.get_location_name()], + indirect=True, +) +async def test_entrypoint_without_api_tokens_return_401( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + entrypoint: str, + user_id: UserID, +): + url = url_from_operation_id( + client, initialized_app, entrypoint, location_id=f"{location_id}" + ).with_query( + user_id=user_id, + ) + response = await client.get(f"{url}") + assert_status( + response, + status.HTTP_401_UNAUTHORIZED, + None, + ) diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 40d4b72f15e..3304e226dd9 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -12,21 +12,20 @@ import logging import urllib.parse from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from pathlib import Path from random import choice from typing import Any, Literal from uuid import uuid4 +import httpx import pytest from aiohttp import ClientSession -from aiohttp.test_utils import TestClient -from aiopg.sa import Engine from aws_library.s3 import S3KeyNotFoundError, S3ObjectKey, SimcoreS3API from aws_library.s3._constants import MULTIPART_UPLOADS_MIN_TOTAL_SIZE from faker import Faker -from models_library.api_schemas_storage import ( +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( FileMetaDataGet, FileUploadCompleteFutureResponse, FileUploadCompleteResponse, @@ -38,26 +37,33 @@ SoftCopyBody, UploadedPart, ) -from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyHttpUrl, ByteSize, HttpUrl, TypeAdapter +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_mock import MockerFixture -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context from pytest_simcore.helpers.parametrizations import byte_size_ids from pytest_simcore.helpers.s3 import upload_file_part, upload_file_to_presigned_link +from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams +from pytest_simcore.helpers.storage_utils_file_meta_data import ( + assert_file_meta_data_in_db, +) from servicelib.aiohttp import status from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID -from simcore_service_storage.handlers_files import UPLOAD_TASKS_KEY -from simcore_service_storage.models import S3BucketName, UploadID +from simcore_service_storage.models import FileDownloadResponse, S3BucketName, UploadID +from simcore_service_storage.modules.long_running_tasks import ( + get_completed_upload_tasks, +) +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy.ext.asyncio import AsyncEngine from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db from types_aiobotocore_s3 import S3Client from yarl import URL @@ -82,9 +88,9 @@ async def assert_multipart_uploads_in_progress( expected_upload_ids: list[str] | None, ): """if None is passed, then it checks that no uploads are in progress""" - list_uploads: list[ - tuple[UploadID, S3ObjectKey] - ] = await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + list_uploads: list[tuple[UploadID, S3ObjectKey]] = ( + await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + ) if expected_upload_ids is None: assert ( not list_uploads @@ -104,6 +110,12 @@ class SingleLinkParam: expected_chunk_size: ByteSize +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "single_link_param", [ @@ -118,7 +130,7 @@ class SingleLinkParam: ), pytest.param( SingleLinkParam( - {"link_type": "presigned"}, + {"link_type": "PRESIGNED"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, TypeAdapter(ByteSize).validate_python("5GiB"), @@ -127,7 +139,7 @@ class SingleLinkParam: ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, + {"link_type": "S3"}, "s3", [], TypeAdapter(ByteSize).validate_python("5TiB"), @@ -141,7 +153,7 @@ async def test_create_upload_file_with_file_size_0_returns_single_link( storage_s3_bucket: S3BucketName, simcore_file_id: SimcoreS3FileID, single_link_param: SingleLinkParam, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], cleanup_user_projects_file_metadata: None, ): @@ -171,7 +183,7 @@ async def test_create_upload_file_with_file_size_0_returns_single_link( # now check the entry in the database is correct, there should be only one await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -189,28 +201,29 @@ async def test_create_upload_file_with_file_size_0_returns_single_link( @pytest.fixture async def create_upload_file_link_v1( - client: TestClient, user_id: UserID, location_id: LocationID + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, + location_id: LocationID, ) -> AsyncIterator[Callable[..., Awaitable[PresignedLink]]]: file_params: list[tuple[UserID, int, SimcoreS3FileID]] = [] async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLink: - assert client.app - url = ( - client.app.router["upload_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(file_id, safe=""), - ) - .with_query(**query_kwargs, user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "upload_file", + location_id=f"{location_id}", + file_id=file_id, + ).with_query(**query_kwargs, user_id=user_id) assert ( "file_size" not in url.query ), "v1 call to upload_file MUST NOT contain file_size field, this is reserved for v2 call" response = await client.put(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + received_file_upload_link, error = assert_status( + response, status.HTTP_200_OK, PresignedLink + ) assert not error - assert data - received_file_upload_link = TypeAdapter(PresignedLink).validate_python(data) assert received_file_upload_link file_params.append((user_id, location_id, file_id)) return received_file_upload_link @@ -218,21 +231,26 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi yield _link_creator # cleanup - assert client.app + clean_tasks = [] for u_id, loc_id, file_id in file_params: - url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{loc_id}", - file_id=urllib.parse.quote(file_id, safe=""), - ) - .with_query(user_id=u_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "upload_file", + location_id=f"{location_id}", + file_id=file_id, + ).with_query(user_id=u_id) clean_tasks.append(client.delete(f"{url}")) await asyncio.gather(*clean_tasks) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "single_link_param", [ @@ -247,7 +265,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi ), pytest.param( SingleLinkParam( - {"link_type": "presigned"}, + {"link_type": "PRESIGNED"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, TypeAdapter(ByteSize).validate_python("5GiB"), @@ -256,7 +274,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, + {"link_type": "S3"}, "s3", [], TypeAdapter(ByteSize).validate_python("5TiB"), @@ -270,7 +288,7 @@ async def test_create_upload_file_with_no_file_size_query_returns_v1_structure( storage_s3_bucket: S3BucketName, simcore_file_id: SimcoreS3FileID, single_link_param: SingleLinkParam, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, create_upload_file_link_v1: Callable[..., Awaitable[PresignedLink]], cleanup_user_projects_file_metadata: None, ): @@ -289,7 +307,7 @@ async def test_create_upload_file_with_no_file_size_query_returns_v1_structure( ) # now check the entry in the database is correct, there should be only one await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -314,6 +332,12 @@ class MultiPartParam: expected_chunk_size: ByteSize +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "test_param", [ @@ -374,14 +398,14 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin storage_s3_bucket: S3BucketName, simcore_file_id: SimcoreS3FileID, test_param: MultiPartParam, - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], cleanup_user_projects_file_metadata: None, ): # create upload file link received_file_upload = await create_upload_file_link_v2( simcore_file_id, - link_type=test_param.link_type.value.lower(), + link_type=test_param.link_type.value, file_size=f"{test_param.file_size}", ) # number of links @@ -395,7 +419,7 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin # now check the entry in the database is correct, there should be only one expect_upload_id = bool(test_param.file_size >= MULTIPART_UPLOADS_MIN_TOTAL_SIZE) upload_id: UploadID | None = await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -412,6 +436,12 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "link_type, file_size", [ @@ -421,8 +451,8 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin ids=byte_size_ids, ) async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( - aiopg_engine: Engine, - client: TestClient, + sqlalchemy_async_engine: AsyncEngine, + client: httpx.AsyncClient, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, with_versioning_enabled: None, @@ -431,15 +461,14 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( file_size: ByteSize, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], ): - assert client.app # create upload file link upload_link = await create_upload_file_link_v2( - simcore_file_id, link_type=link_type.value.lower(), file_size=file_size + simcore_file_id, link_type=link_type.value, file_size=file_size ) expect_upload_id = bool(file_size >= MULTIPART_UPLOADS_MIN_TOTAL_SIZE) # we shall have an entry in the db, waiting for upload upload_id = await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -457,11 +486,11 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( # delete/abort file upload abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") - await assert_status(response, status.HTTP_204_NO_CONTENT) + assert_status(response, status.HTTP_204_NO_CONTENT, None) # the DB shall be cleaned up await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=False, expected_file_size=None, @@ -477,6 +506,12 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "link_type, file_size", [ @@ -488,8 +523,8 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( ids=byte_size_ids, ) async def test_upload_same_file_uuid_aborts_previous_upload( - aiopg_engine: Engine, - client: TestClient, + sqlalchemy_async_engine: AsyncEngine, + client: httpx.AsyncClient, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, simcore_file_id: SimcoreS3FileID, @@ -497,17 +532,16 @@ async def test_upload_same_file_uuid_aborts_previous_upload( file_size: ByteSize, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], ): - assert client.app # create upload file link file_upload_link = await create_upload_file_link_v2( - simcore_file_id, link_type=link_type.value.lower(), file_size=file_size + simcore_file_id, link_type=link_type.value, file_size=file_size ) expect_upload_id = bool( file_size >= MULTIPART_UPLOADS_MIN_TOTAL_SIZE or link_type == LinkType.S3 ) # we shall have an entry in the db, waiting for upload upload_id = await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -527,7 +561,7 @@ async def test_upload_same_file_uuid_aborts_previous_upload( # we should abort the previous upload to prevent unwanted costs await asyncio.sleep(1) new_file_upload_link = await create_upload_file_link_v2( - simcore_file_id, link_type=link_type.value.lower(), file_size=file_size + simcore_file_id, link_type=link_type.value, file_size=file_size ) if link_type == LinkType.PRESIGNED: assert file_upload_link != new_file_upload_link @@ -535,7 +569,7 @@ async def test_upload_same_file_uuid_aborts_previous_upload( assert file_upload_link == new_file_upload_link # we shall have an entry in the db, waiting for upload new_upload_id = await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -564,6 +598,12 @@ def complex_file_name(faker: Faker) -> str: return f"subfolder_1/sub_folder 2/some file name with spaces and special characters -_ü!öäàé+|}} {{3245_{faker.file_name()}" +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -583,6 +623,12 @@ async def test_upload_real_file( await upload_file(file_size, complex_file_name) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -594,7 +640,8 @@ async def test_upload_real_file( async def test_upload_real_file_with_emulated_storage_restart_after_completion_was_called( complex_file_name: str, file_size: ByteSize, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -602,7 +649,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], create_file_of_size: Callable[[ByteSize, str | None], Path], create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, ): @@ -611,11 +658,11 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w if after running the completion task, storage restarts then the task is lost. Nevertheless the client still has a reference to the completion future and shall be able to ask for its status""" - assert client.app + file = create_file_of_size(file_size, complex_file_name) file_id = create_simcore_file_id(project_id, node_id, complex_file_name) file_upload_link = await create_upload_file_link_v2( - file_id, link_type="presigned", file_size=file_size + file_id, link_type="PRESIGNED", file_size=file_size ) # upload the file part_to_etag: list[UploadedPart] = await upload_file_to_presigned_link( @@ -628,14 +675,15 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w json=jsonable_encoder(FileUploadCompletionBody(parts=part_to_etag)), ) response.raise_for_status() - data, error = await assert_status(response, status.HTTP_202_ACCEPTED) + file_upload_complete_response, error = assert_status( + response, status.HTTP_202_ACCEPTED, FileUploadCompleteResponse + ) assert not error - assert data - file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + assert file_upload_complete_response state_url = URL(f"{file_upload_complete_response.links.state}").relative() # here we do not check now for the state completion. instead we simulate a restart where the tasks disappear - client.app[UPLOAD_TASKS_KEY].clear() + get_completed_upload_tasks(initialized_app).clear() # now check for the completion completion_etag = None async for attempt in AsyncRetrying( @@ -644,15 +692,19 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w stop=stop_after_delay(60), retry=retry_if_exception_type(AssertionError), ): - with attempt, log_context( - logging.INFO, - f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", - ) as ctx: + with ( + attempt, + log_context( + logging.INFO, + f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", + ) as ctx, + ): response = await client.post(f"{state_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + future, error = assert_status( + response, status.HTTP_200_OK, FileUploadCompleteFutureResponse + ) assert not error - assert data - future = FileUploadCompleteFutureResponse.model_validate(data) + assert future assert future.state == FileUploadCompleteState.OK assert future.e_tag is not None completion_etag = future.e_tag @@ -662,7 +714,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w ) # check the entry in db now has the correct file size, and the upload id is gone await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=file_id, expected_entry_exists=True, expected_file_size=file_size, @@ -679,11 +731,17 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w assert s3_metadata.e_tag == completion_etag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_of_single_presigned_link_lazily_update_database_on_get( - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, - client: TestClient, + client: httpx.AsyncClient, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], create_file_of_size: Callable[[ByteSize, str | None], Path], create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], @@ -693,7 +751,6 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( get_file_meta_data: Callable[..., Awaitable[FileMetaDataGet]], s3_client: S3Client, ): - assert client.app file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file @@ -701,7 +758,7 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( simcore_file_id = create_simcore_file_id(project_id, node_id, file_name) # get an S3 upload link file_upload_link = await create_upload_file_link_v2( - simcore_file_id, link_type="s3", file_size=file_size + simcore_file_id, link_type="S3", file_size=file_size ) assert file_upload_link # let's use the storage s3 internal client to upload @@ -723,11 +780,17 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( assert received_fmd.entity_tag == upload_e_tag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_real_file_with_s3_client( - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, - client: TestClient, + client: httpx.AsyncClient, create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], create_file_of_size: Callable[[ByteSize, str | None], Path], create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], @@ -736,7 +799,6 @@ async def test_upload_real_file_with_s3_client( faker: Faker, s3_client: S3Client, ): - assert client.app file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file @@ -744,7 +806,7 @@ async def test_upload_real_file_with_s3_client( simcore_file_id = create_simcore_file_id(project_id, node_id, file_name) # get an S3 upload link file_upload_link = await create_upload_file_link_v2( - simcore_file_id, link_type="s3", file_size=file_size + simcore_file_id, link_type="S3", file_size=file_size ) # let's use the storage s3 internal client to upload with file.open("rb") as fp: @@ -766,10 +828,11 @@ async def test_upload_real_file_with_s3_client( with log_context(logging.INFO, f"completing upload of {file=}"): response = await client.post(f"{complete_url}", json={"parts": []}) response.raise_for_status() - data, error = await assert_status(response, status.HTTP_202_ACCEPTED) + file_upload_complete_response, error = assert_status( + response, status.HTTP_202_ACCEPTED, FileUploadCompleteResponse + ) assert not error - assert data - file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + assert file_upload_complete_response state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None async for attempt in AsyncRetrying( @@ -778,18 +841,22 @@ async def test_upload_real_file_with_s3_client( stop=stop_after_delay(60), retry=retry_if_exception_type(ValueError), ): - with attempt, log_context( - logging.INFO, - f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", - ) as ctx: + with ( + attempt, + log_context( + logging.INFO, + f"waiting for upload completion {state_url=}, {attempt.retry_state.attempt_number}", + ) as ctx, + ): response = await client.post(f"{state_url}") response.raise_for_status() - data, error = await assert_status(response, status.HTTP_200_OK) + future, error = assert_status( + response, status.HTTP_200_OK, FileUploadCompleteFutureResponse + ) assert not error - assert data - future = FileUploadCompleteFutureResponse.model_validate(data) + assert future if future.state != FileUploadCompleteState.OK: - msg = f"{data=}" + msg = f"{future=}" raise ValueError(msg) assert future.state == FileUploadCompleteState.OK assert future.e_tag is not None @@ -801,7 +868,7 @@ async def test_upload_real_file_with_s3_client( # check the entry in db now has the correct file size, and the upload id is gone await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=simcore_file_id, expected_entry_exists=True, expected_file_size=file_size, @@ -818,6 +885,12 @@ async def test_upload_real_file_with_s3_client( assert s3_metadata.e_tag == completion_etag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -827,8 +900,8 @@ async def test_upload_real_file_with_s3_client( ids=byte_size_ids, ) async def test_upload_twice_and_fail_second_time_shall_keep_first_version( - aiopg_engine: Engine, - client: TestClient, + sqlalchemy_async_engine: AsyncEngine, + client: httpx.AsyncClient, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, with_versioning_enabled: None, @@ -840,18 +913,17 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( user_id: UserID, location_id: LocationID, ): - assert client.app # 1. upload a valid file file_name = faker.file_name() _, uploaded_file_id = await upload_file(file_size, file_name) # 2. create an upload link for the second file upload_link = await create_upload_file_link_v2( - uploaded_file_id, link_type="presigned", file_size=file_size + uploaded_file_id, link_type="PRESIGNED", file_size=file_size ) # we shall have an entry in the db, waiting for upload await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=uploaded_file_id, expected_entry_exists=True, expected_file_size=-1, @@ -863,7 +935,7 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( # 3. upload part of the file to simulate a network issue in the upload new_file = create_file_of_size(file_size, file_name) with pytest.raises(RuntimeError): - async with ClientSession() as session: + async with httpx.AsyncClient() as session: await upload_file_part( session, new_file, @@ -878,11 +950,11 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( # 4. abort file upload abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") - await assert_status(response, status.HTTP_204_NO_CONTENT) + assert_status(response, status.HTTP_204_NO_CONTENT, None) # we should have the original file still in now... await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=uploaded_file_id, expected_entry_exists=True, expected_file_size=file_size, @@ -903,7 +975,7 @@ def file_size() -> ByteSize: async def _assert_file_downloaded( - faker: Faker, tmp_path: Path, link: HttpUrl, uploaded_file: Path + faker: Faker, tmp_path: Path, link: AnyUrl, uploaded_file: Path ): dest_file = tmp_path / faker.file_name() async with ClientSession() as session: @@ -917,16 +989,16 @@ async def _assert_file_downloaded( async def test_download_file_no_file_was_uploaded( - client: TestClient, - location_id: int, + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, project_id: ProjectID, node_id: NodeID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, + fake_datcore_tokens: tuple[str, str], ): - assert client.app - missing_file = TypeAdapter(SimcoreS3FileID).validate_python( f"{project_id}/{node_id}/missing.file" ) @@ -936,33 +1008,39 @@ async def test_download_file_no_file_was_uploaded( ) is False ) + download_url = url_from_operation_id( + client, + initialized_app, + "download_file", + location_id=f"{location_id}", + file_id=missing_file, + ).with_query(user_id=user_id) - download_url = ( - client.app.router["download_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(missing_file, safe=""), - ) - .with_query(user_id=user_id) - ) response = await client.get(f"{download_url}") - data, error = await assert_status(response, status.HTTP_404_NOT_FOUND) + data, error = assert_status(response, status.HTTP_404_NOT_FOUND, None) assert data is None - assert missing_file in error["message"] + assert len(error["errors"]) == 1 + assert missing_file in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_1_to_1_with_file_meta_data( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, file_size: ByteSize, upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - location_id: int, + location_id: LocationID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, tmp_path: Path, faker: Faker, ): - assert client.app # 2. file_meta_data entry corresponds to a file # upload a single file as a file_meta_data entry and check link uploaded_file, uploaded_file_uuid = await upload_file( @@ -975,46 +1053,50 @@ async def test_download_file_1_to_1_with_file_meta_data( is True ) - download_url = ( - client.app.router["download_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(uploaded_file_uuid, safe=""), - ) - .with_query(user_id=user_id) - ) + download_url = url_from_operation_id( + client, + initialized_app, + "download_file", + location_id=f"{location_id}", + file_id=uploaded_file_uuid, + ).with_query(user_id=user_id) response = await client.get(f"{download_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + data, error = assert_status(response, status.HTTP_200_OK, FileDownloadResponse) assert not error assert data - assert "link" in data - assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) await _assert_file_downloaded( - faker, tmp_path, link=data["link"], uploaded_file=uploaded_file + faker, tmp_path, link=data.link, uploaded_file=uploaded_file ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_from_inside_a_directory( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, file_size: ByteSize, - location_id: int, + location_id: LocationID, user_id: UserID, - create_empty_directory: Callable[..., Awaitable[FileUploadSchema]], + project_id: ProjectID, + node_id: NodeID, + create_empty_directory: Callable[ + [str, ProjectID, NodeID], Awaitable[SimcoreS3FileID] + ], create_file_of_size: Callable[[ByteSize, str | None], Path], storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, tmp_path: Path, faker: Faker, ): - assert client.app # 3. file_meta_data entry corresponds to a directory # upload a file inside a directory and check the download link directory_name = "a-test-dir" - directory_file_upload = await create_empty_directory(directory_name) - - assert directory_file_upload.urls[0].path - dir_path_in_s3 = directory_file_upload.urls[0].path.strip("/") + dir_path_in_s3 = await create_empty_directory(directory_name, project_id, node_id) file_name = "meta_data_entry_is_dir.file" file_to_upload_in_dir = create_file_of_size(file_size, file_name) @@ -1036,67 +1118,79 @@ async def test_download_file_from_inside_a_directory( ) # finally check the download link - download_url = ( - client.app.router["download_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(s3_file_id, safe=""), - ) - .with_query(user_id=user_id) - ) + download_url = url_from_operation_id( + client, + initialized_app, + "download_file", + location_id=f"{location_id}", + file_id=s3_file_id, + ).with_query(user_id=user_id) response = await client.get(f"{download_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + file_download, error = assert_status( + response, status.HTTP_200_OK, FileDownloadResponse + ) assert not error - assert data - assert "link" in data - assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) + assert file_download + await _assert_file_downloaded( - faker, tmp_path, link=data["link"], uploaded_file=file_to_upload_in_dir + faker, tmp_path, link=file_download.link, uploaded_file=file_to_upload_in_dir ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_the_file_is_missing_from_the_directory( - client: TestClient, - location_id: int, + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, user_id: UserID, - create_empty_directory: Callable[..., Awaitable[FileUploadSchema]], + project_id: ProjectID, + node_id: NodeID, + create_empty_directory: Callable[ + [str, ProjectID, NodeID], Awaitable[SimcoreS3FileID] + ], ): - assert client.app # file_meta_data entry corresponds to a directory but file is not present in directory directory_name = "a-second-test-dir" - directory_file_upload = await create_empty_directory(directory_name) - - assert directory_file_upload.urls[0].path - dir_path_in_s3 = directory_file_upload.urls[0].path.strip("/") + dir_path_in_s3 = await create_empty_directory(directory_name, project_id, node_id) missing_s3_file_id = TypeAdapter(SimcoreS3FileID).validate_python( f"{dir_path_in_s3}/missing_inside_dir.file" ) - download_url = ( - client.app.router["download_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(missing_s3_file_id, safe=""), - ) - .with_query(user_id=user_id) - ) + download_url = url_from_operation_id( + client, + initialized_app, + "download_file", + location_id=f"{location_id}", + file_id=missing_s3_file_id, + ).with_query(user_id=user_id) + response = await client.get(f"{download_url}") - data, error = await assert_status(response, status.HTTP_404_NOT_FOUND) + data, error = assert_status(response, status.HTTP_404_NOT_FOUND, None) assert data is None - assert missing_s3_file_id in error["message"] + assert missing_s3_file_id in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_access_rights( - client: TestClient, - location_id: int, + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, faker: Faker, ): - assert client.app - # project_id does not exist missing_file = TypeAdapter(SimcoreS3FileID).validate_python( f"{faker.uuid4()}/{faker.uuid4()}/project_id_is_missing" @@ -1107,21 +1201,26 @@ async def test_download_file_access_rights( ) is False ) + download_url = url_from_operation_id( + client, + initialized_app, + "download_file", + location_id=f"{location_id}", + file_id=missing_file, + ).with_query(user_id=user_id) - download_url = ( - client.app.router["download_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(missing_file, safe=""), - ) - .with_query(user_id=user_id) - ) response = await client.get(f"{download_url}") - data, error = await assert_status(response, status.HTTP_403_FORBIDDEN) + data, error = assert_status(response, status.HTTP_403_FORBIDDEN, None) assert data is None - assert "Insufficient access rights" in error["message"] + assert "Insufficient access rights" in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -1130,33 +1229,32 @@ async def test_download_file_access_rights( ids=byte_size_ids, ) async def test_delete_file( - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, file_size: ByteSize, upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - location_id: int, + location_id: LocationID, user_id: UserID, faker: Faker, ): - assert client.app _, uploaded_file_uuid = await upload_file(file_size, faker.file_name()) - delete_url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(uploaded_file_uuid, safe=""), - ) - .with_query(user_id=user_id) - ) + delete_url = url_from_operation_id( + client, + initialized_app, + "delete_file", + location_id=f"{location_id}", + file_id=uploaded_file_uuid, + ).with_query(user_id=user_id) response = await client.delete(f"{delete_url}") - await assert_status(response, status.HTTP_204_NO_CONTENT) + assert_status(response, status.HTTP_204_NO_CONTENT, None) # check the entry in db is removed await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=uploaded_file_uuid, expected_entry_exists=False, expected_file_size=None, @@ -1171,8 +1269,15 @@ async def test_delete_file( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_copy_as_soft_link( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -1180,101 +1285,105 @@ async def test_copy_as_soft_link( create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], faker: Faker, ): - assert client.app - # missing simcore_file_id returns 404 missing_file_uuid = create_simcore_file_id(project_id, node_id, faker.file_name()) invalid_link_id = create_simcore_file_id(uuid4(), uuid4(), faker.file_name()) - url = ( - client.app.router["copy_as_soft_link"] - .url_for( - file_id=urllib.parse.quote(missing_file_uuid, safe=""), - ) - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "copy_as_soft_link", + file_id=missing_file_uuid, + ).with_query(user_id=user_id) response = await client.post( f"{url}", json=jsonable_encoder(SoftCopyBody(link_id=invalid_link_id)) ) - await assert_status(response, status.HTTP_404_NOT_FOUND) + assert_status(response, status.HTTP_404_NOT_FOUND, None) # now let's try with whatever link id file, original_file_uuid = await upload_file( TypeAdapter(ByteSize).validate_python("10Mib"), faker.file_name() ) - url = ( - client.app.router["copy_as_soft_link"] - .url_for( - file_id=urllib.parse.quote(original_file_uuid, safe=""), - ) - .with_query(user_id=user_id) - ) + url = url_from_operation_id( + client, + initialized_app, + "copy_as_soft_link", + file_id=original_file_uuid, + ).with_query(user_id=user_id) + link_id = TypeAdapter(SimcoreS3FileID).validate_python( f"api/{node_id}/{faker.file_name()}" ) response = await client.post( f"{url}", json=jsonable_encoder(SoftCopyBody(link_id=link_id)) ) - data, error = await assert_status(response, status.HTTP_200_OK) + fmd, error = assert_status(response, status.HTTP_200_OK, FileMetaDataGet) assert not error - fmd = TypeAdapter(FileMetaDataGet).validate_python(data) + assert fmd assert fmd.file_id == link_id -async def __list_files( - client: TestClient, +async def _list_files( + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, location_id: LocationID, *, - path: str, expand_dirs: bool, ) -> list[FileMetaDataGet]: - assert client.app - get_url = ( - client.app.router["get_files_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(path, safe=""), - ) - .with_query(user_id=user_id, expand_dirs=f"{expand_dirs}".lower()) - ) + get_url = url_from_operation_id( + client, + initialized_app, + "list_files_metadata", + location_id=f"{location_id}", + ).with_query(user_id=user_id, expand_dirs=f"{expand_dirs}".lower()) response = await client.get(f"{get_url}") - data, error = await assert_status(response, status.HTTP_200_OK) + fmds, error = assert_status(response, status.HTTP_200_OK, list[FileMetaDataGet]) assert not error - return TypeAdapter(list[FileMetaDataGet]).validate_python(data) + assert fmds is not None + return fmds async def _list_files_legacy( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, location_id: LocationID, - directory_file_upload: FileUploadSchema, ) -> list[FileMetaDataGet]: - assert directory_file_upload.urls[0].path - directory_file_id = directory_file_upload.urls[0].path.strip("/") - return await __list_files( - client, user_id, location_id, path=directory_file_id, expand_dirs=True + return await _list_files( + initialized_app, + client, + user_id, + location_id, + expand_dirs=True, ) async def _list_files_and_directories( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, location_id: LocationID, - directory_file_upload: FileUploadSchema, ) -> list[FileMetaDataGet]: - assert directory_file_upload.urls[0].path - directory_parent_path = Path(directory_file_upload.urls[0].path).parent - directory_file_id = f"{directory_parent_path}".strip("/") - return await __list_files( - client, user_id, location_id, path=directory_file_id, expand_dirs=False + return await _list_files( + initialized_app, + client, + user_id, + location_id, + expand_dirs=False, ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("link_type", LinkType) @pytest.mark.parametrize( "file_size", [ - ByteSize(-1), + ByteSize(0), TypeAdapter(ByteSize).validate_python("0"), TypeAdapter(ByteSize).validate_python("1TB"), ], @@ -1284,7 +1393,8 @@ async def test_is_directory_link_forces_link_type_and_size( node_id: NodeID, create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, location_id: LocationID, user_id: UserID, link_type: LinkType, @@ -1294,7 +1404,7 @@ async def test_is_directory_link_forces_link_type_and_size( directory_file_id = create_simcore_file_id(project_id, node_id, DIR_NAME) directory_file_upload: FileUploadSchema = await create_upload_file_link_v2( directory_file_id, - link_type=link_type.value.lower(), + link_type=link_type.value, is_directory="true", file_size=file_size, ) @@ -1302,7 +1412,7 @@ async def test_is_directory_link_forces_link_type_and_size( assert len(directory_file_upload.urls) == 1 files_and_directories: list[FileMetaDataGet] = await _list_files_and_directories( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) assert len(files_and_directories) == 1 assert files_and_directories[0].is_directory is True @@ -1310,26 +1420,30 @@ async def test_is_directory_link_forces_link_type_and_size( assert files_and_directories[0].file_size == 0 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_ensure_expand_dirs_defaults_true( + initialized_app: FastAPI, mocker: MockerFixture, - client: TestClient, + client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, ): mocked_object = mocker.patch( "simcore_service_storage.simcore_s3_dsm.SimcoreS3DataManager.list_files", autospec=True, ) - assert client.app - get_url = ( - client.app.router["get_files_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote("mocked_path", safe=""), - ) - .with_query(user_id=user_id) - ) + get_url = url_from_operation_id( + client, + initialized_app, + "list_files_metadata", + location_id=f"{location_id}", + ).with_query(user_id=user_id) await client.get(f"{get_url}") assert len(mocked_object.call_args_list) == 1 @@ -1338,189 +1452,215 @@ async def test_ensure_expand_dirs_defaults_true( assert call_args_list.kwargs["expand_dirs"] is True +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_file_is_directory_and_remove_content( - create_empty_directory: Callable[..., Awaitable[FileUploadSchema]], - populate_directory: Callable[..., Awaitable[None]], + initialized_app: FastAPI, + create_empty_directory: Callable[ + [str, ProjectID, NodeID], Awaitable[SimcoreS3FileID] + ], + populate_directory: Callable[ + [ByteSize, str, ProjectID, NodeID, int, int], + Awaitable[tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]], + ], delete_directory: Callable[..., Awaitable[None]], - client: TestClient, + client: httpx.AsyncClient, location_id: LocationID, user_id: UserID, - faker: Faker, + project_id: ProjectID, + node_id: NodeID, ): FILE_SIZE_IN_DIR = TypeAdapter(ByteSize).validate_python("1Mib") DIR_NAME = "some-dir" SUBDIR_COUNT = 4 - FILE_COUNT = 5 + FILE_COUNT = 20 # DIRECTORY CREATION (is empty) - directory_file_upload: FileUploadSchema = await create_empty_directory( - dir_name=DIR_NAME - ) + directory_in_s3 = await create_empty_directory(DIR_NAME, project_id, node_id) files_and_directories: list[FileMetaDataGet] = await _list_files_and_directories( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) assert len(files_and_directories) == 1 list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) assert len(list_of_files) == 0 # DIRECTORY WITH CONTENT await populate_directory( - file_size_in_dir=FILE_SIZE_IN_DIR, - dir_name=DIR_NAME, - subdir_count=SUBDIR_COUNT, - file_count=FILE_COUNT, + FILE_SIZE_IN_DIR, + DIR_NAME, + project_id, + node_id, + SUBDIR_COUNT, + FILE_COUNT, ) files_and_directories: list[FileMetaDataGet] = await _list_files_and_directories( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) assert len(files_and_directories) == 1 list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) - assert len(list_of_files) == SUBDIR_COUNT * FILE_COUNT + assert len(list_of_files) == FILE_COUNT # DELETE NOT EXISTING - assert client.app - - delete_url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote( - "/".join(list_of_files[0].file_id.split("/")[:2]) + "/does_not_exist", - safe="", - ), - ) - .with_query(user_id=user_id) - ) + delete_url = url_from_operation_id( + client, + initialized_app, + "delete_file", + location_id=f"{location_id}", + file_id="/".join(list_of_files[0].file_id.split("/")[:2]) + "/does_not_exist", + ).with_query(user_id=user_id) response = await client.delete(f"{delete_url}") - _, error = await assert_status(response, status.HTTP_204_NO_CONTENT) + _, error = assert_status(response, status.HTTP_204_NO_CONTENT, None) assert error is None list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload + initialized_app, client, user_id, location_id ) - assert len(list_of_files) == SUBDIR_COUNT * FILE_COUNT + assert len(list_of_files) == FILE_COUNT # DELETE ONE FILE FROM THE DIRECTORY - assert client.app - delete_url = ( - client.app.router["delete_file"] - .url_for( - location_id=f"{location_id}", - file_id=urllib.parse.quote(list_of_files[0].file_id, safe=""), - ) - .with_query(user_id=user_id) - ) + delete_url = url_from_operation_id( + client, + initialized_app, + "delete_file", + location_id=f"{location_id}", + file_id=list_of_files[0].file_id, + ).with_query(user_id=user_id) response = await client.delete(f"{delete_url}") - _, error = await assert_status(response, status.HTTP_204_NO_CONTENT) + _, error = assert_status(response, status.HTTP_204_NO_CONTENT, None) assert error is None - list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload + list_of_files = await _list_files_legacy( + initialized_app, client, user_id, location_id ) - assert len(list_of_files) == SUBDIR_COUNT * FILE_COUNT - 1 + assert len(list_of_files) == FILE_COUNT - 1 # DIRECTORY REMOVAL - await delete_directory(directory_file_upload=directory_file_upload) + await delete_directory(directory_in_s3) - list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload + list_of_files = await _list_files_legacy( + initialized_app, client, user_id, location_id ) assert len(list_of_files) == 0 - files_and_directories: list[FileMetaDataGet] = await _list_files_and_directories( - client, user_id, location_id, directory_file_upload + files_and_directories = await _list_files_and_directories( + initialized_app, client, user_id, location_id ) assert len(files_and_directories) == 0 -@pytest.mark.parametrize("files_in_dir", [1002]) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize("files_count", [1002]) async def test_listing_more_than_1000_objects_in_bucket( create_directory_with_files: Callable[ - ..., AbstractAsyncContextManager[FileUploadSchema] + [str, ByteSize, int, int, ProjectID, NodeID], + Awaitable[ + tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], ], - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, location_id: LocationID, user_id: UserID, - files_in_dir: int, + project_id: ProjectID, + node_id: NodeID, + files_count: int, ): - async with create_directory_with_files( - dir_name="some-random", - file_size_in_dir=TypeAdapter(ByteSize).validate_python("1"), - subdir_count=1, - file_count=files_in_dir, - ) as directory_file_upload: - list_of_files: list[FileMetaDataGet] = await _list_files_legacy( - client, user_id, location_id, directory_file_upload - ) - # for now no more than 1000 objects will be returned - assert len(list_of_files) == 1000 + SUBDIR_COUNT = 1 + await create_directory_with_files( + "random-directory", + TypeAdapter(ByteSize).validate_python("1"), + SUBDIR_COUNT, + files_count, + project_id, + node_id, + ) + list_of_files = await _list_files_legacy( + initialized_app, client, user_id, location_id + ) + # for now no more than 1000 objects will be returned + assert len(list_of_files) == 1000 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("uuid_filter", [True, False]) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=0, + ), + ], + ids=str, +) async def test_listing_with_project_id_filter( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, location_id: LocationID, user_id: UserID, faker: Faker, random_project_with_files: Callable[ - [int, tuple[ByteSize, ...]], + [ProjectWithFilesParams], Awaitable[ - tuple[ - dict[str, Any], - dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]], - ] + tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] ], ], uuid_filter: bool, + project_params: ProjectWithFilesParams, ): - project, src_projects_list = await random_project_with_files( - num_nodes=1, - file_sizes=(ByteSize(1),), - file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), - ) - _, _ = await random_project_with_files( - num_nodes=1, - file_sizes=(ByteSize(1),), - file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), - ) + src_project, src_projects_list = await random_project_with_files(project_params) + _, _ = await random_project_with_files(project_params) assert len(src_projects_list.keys()) > 0 node_id = next(iter(src_projects_list.keys())) project_files_in_db = set(src_projects_list[node_id]) assert len(project_files_in_db) > 0 - project_id = project["uuid"] + project_id = src_project["uuid"] project_file_name = Path(choice(list(project_files_in_db))).name # noqa: S311 - assert client.app query = { "user_id": user_id, - "project_id": project_id, + "project_id": f"{project_id}", "uuid_filter": project_file_name if uuid_filter else None, } - url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{location_id}") - .with_query(**{k: v for k, v in query.items() if v is not None}) - ) + url = url_from_operation_id( + client, initialized_app, "list_files_metadata", location_id=f"{location_id}" + ).with_query(**{k: v for k, v in query.items() if v is not None}) response = await client.get(f"{url}") - data, _ = await assert_status(response, status.HTTP_200_OK) - - list_of_files = TypeAdapter(list[FileMetaDataGet]).validate_python(data) + list_of_files, _ = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_of_files if uuid_filter: assert len(list_of_files) == 1 diff --git a/services/storage/tests/unit/test_handlers_files_metadata.py b/services/storage/tests/unit/test_handlers_files_metadata.py index 9abd834d21a..dd8bd4a2728 100644 --- a/services/storage/tests/unit/test_handlers_files_metadata.py +++ b/services/storage/tests/unit/test_handlers_files_metadata.py @@ -2,23 +2,29 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument - -import urllib.parse from collections.abc import Awaitable, Callable from copy import deepcopy from pathlib import Path from random import choice from typing import Protocol +import httpx import pytest -from aiohttp.test_utils import TestClient from faker import Faker -from models_library.api_schemas_storage import FileMetaDataGet, SimcoreS3FileID +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( + FileMetaDataGet, + SimcoreS3FileID, +) from models_library.projects import ProjectID +from models_library.projects_nodes_io import LocationID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from servicelib.aiohttp import status +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from yarl import URL pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] @@ -32,34 +38,42 @@ async def __call__( read: bool, write: bool, delete: bool, - ) -> None: - ... + ) -> None: ... -async def test_get_files_metadata( +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +async def test_list_files_metadata( upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], create_project_access_rights: CreateProjectAccessRightsCallable, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, other_user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, faker: Faker, ): - assert client.app - url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{location_id}") + URL(f"{client.base_url}") + .with_path( + initialized_app.url_path_for("list_files_metadata", location_id=location_id) + ) .with_query(user_id=f"{user_id}") ) # this should return an empty list response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_fmds == [] assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) - assert not list_fmds # now add some stuff there NUM_FILES = 10 @@ -71,9 +85,11 @@ async def test_get_files_metadata( # we should find these files now response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_fmds assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == NUM_FILES # checks project_id filter! @@ -84,15 +100,18 @@ async def test_get_files_metadata( write=True, delete=True, ) + previous_data = deepcopy(list_fmds) response = await client.get( f"{url.update_query(project_id=str(project_id), user_id=other_user_id)}" ) - previous_data = deepcopy(data) - data, error = await assert_status(response, status.HTTP_200_OK) + + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_fmds assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) - assert previous_data == data + assert previous_data == list_fmds # create some more files but with a base common name NUM_FILES = 10 @@ -105,16 +124,20 @@ async def test_get_files_metadata( # we should find these files now response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_fmds assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (2 * NUM_FILES) # we can filter them now response = await client.get(f"{url.update_query(uuid_filter='common_name')}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) + assert list_fmds assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) @@ -122,72 +145,80 @@ async def test_get_files_metadata( reason="storage get_file_metadata must return a 200 with no payload as long as legacy services are around!!" ) async def test_get_file_metadata_is_legacy_services_compatible( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, simcore_file_id: SimcoreS3FileID, ): - assert client.app - url = ( - client.app.router["get_file_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=f"{urllib.parse.quote(simcore_file_id, safe='')}", + URL(f"{client.base_url}") + .with_path( + initialized_app.url_path_for( + "get_file_metadata", + location_id=location_id, + file_id=simcore_file_id, + ) ) .with_query(user_id=f"{user_id}") ) + # this should return an empty list response = await client.get(f"{url}") - await assert_status(response, status.HTTP_404_NOT_FOUND) + assert response.status_code == status.HTTP_404_NOT_FOUND +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_get_file_metadata( upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, simcore_file_id: SimcoreS3FileID, faker: Faker, ): - assert client.app + url = url_from_operation_id( + client, + initialized_app, + "get_file_metadata", + location_id=f"{location_id}", + file_id=simcore_file_id, + ).with_query(user_id=user_id) - url = ( - client.app.router["get_file_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=f"{urllib.parse.quote(simcore_file_id, safe='')}", - ) - .with_query(user_id=f"{user_id}") - ) # this should return an empty list response = await client.get(f"{url}") # await assert_status(response, status.HTTP_404_NOT_FOUND) # NOTE: This needs to be a Ok response with empty data until ALL legacy services are gone, then it should be changed to 404! see test above - assert response.status == status.HTTP_200_OK - assert await response.json() == {"data": {}, "error": "No result found"} + data, error = assert_status(response, status.HTTP_200_OK, dict) + assert error == "No result found" + assert data == {} # now add some stuff there NUM_FILES = 10 file_size = TypeAdapter(ByteSize).validate_python("15Mib") - files_owned_by_us = [] - for _ in range(NUM_FILES): - files_owned_by_us.append(await upload_file(file_size, faker.file_name())) - selected_file, selected_file_uuid = choice(files_owned_by_us) - url = ( - client.app.router["get_file_metadata"] - .url_for( - location_id=f"{location_id}", - file_id=f"{urllib.parse.quote(selected_file_uuid, safe='')}", - ) - .with_query(user_id=f"{user_id}") - ) + files_owned_by_us = [ + await upload_file(file_size, faker.file_name()) for _ in range(NUM_FILES) + ] + selected_file, selected_file_uuid = choice(files_owned_by_us) # noqa: S311 + url = url_from_operation_id( + client, + initialized_app, + "get_file_metadata", + location_id=f"{location_id}", + file_id=selected_file_uuid, + ).with_query(user_id=user_id) + response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + fmd, error = assert_status(response, status.HTTP_200_OK, FileMetaDataGet) assert not error - assert data - fmd = TypeAdapter(FileMetaDataGet).validate_python(data) + assert fmd assert fmd.file_id == selected_file_uuid assert fmd.file_size == selected_file.stat().st_size diff --git a/services/storage/tests/unit/test_handlers_health.py b/services/storage/tests/unit/test_handlers_health.py index 8705c4c8e36..640fbb376b2 100644 --- a/services/storage/tests/unit/test_handlers_health.py +++ b/services/storage/tests/unit/test_handlers_health.py @@ -4,50 +4,50 @@ # pylint: disable=protected-access +import httpx import simcore_service_storage._meta -from aiohttp.test_utils import TestClient -from models_library.api_schemas_storage import S3BucketName +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import HealthCheck, S3BucketName from models_library.app_diagnostics import AppStatusCheck from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from servicelib.aiohttp import status -from simcore_service_storage.handlers_health import HealthCheck from types_aiobotocore_s3 import S3Client pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] -async def test_health_check(client: TestClient): - assert client.app - url = client.app.router["health_check"].url_for() +async def test_health_check(initialized_app: FastAPI, client: httpx.AsyncClient): + url = url_from_operation_id(client, initialized_app, "get_health") response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_health, error = assert_status(response, status.HTTP_200_OK, HealthCheck) + assert app_health assert not error - app_health = HealthCheck.model_validate(data) assert app_health.name == simcore_service_storage._meta.PROJECT_NAME # noqa: SLF001 assert app_health.version == str( - simcore_service_storage._meta.VERSION - ) # noqa: SLF001 + simcore_service_storage._meta.VERSION # noqa: SLF001 + ) -async def test_health_status(client: TestClient): - assert client.app - url = client.app.router["get_status"].url_for() +async def test_health_status(initialized_app: FastAPI, client: httpx.AsyncClient): + url = url_from_operation_id(client, initialized_app, "get_status") response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert ( - app_status_check.app_name == simcore_service_storage._meta.PROJECT_NAME - ) # noqa: SLF001 + app_status_check.app_name + == simcore_service_storage._meta.PROJECT_NAME # noqa: SLF001 + ) assert app_status_check.version == str( - simcore_service_storage._meta.VERSION - ) # noqa: SLF001 + simcore_service_storage._meta.VERSION # noqa: SLF001 + ) assert len(app_status_check.services) == 2 assert "postgres" in app_status_check.services assert "healthy" in app_status_check.services["postgres"] @@ -58,55 +58,61 @@ async def test_health_status(client: TestClient): async def test_bad_health_status_if_bucket_missing( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, storage_s3_bucket: S3BucketName, s3_client: S3Client, ): - assert client.app - url = client.app.router["get_status"].url_for() + url = url_from_operation_id(client, initialized_app, "get_status") response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now delete the bucket await s3_client.delete_bucket(Bucket=storage_s3_bucket) # check again the health response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "no access to S3 bucket" async def test_bad_health_status_if_s3_server_missing( - client: TestClient, mocked_aws_server: ThreadedMotoServer + initialized_app: FastAPI, + client: httpx.AsyncClient, + mocked_aws_server: ThreadedMotoServer, ): - assert client.app - url = client.app.router["get_status"].url_for() + url = url_from_operation_id(client, initialized_app, "get_status") response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now disable the s3 server mocked_aws_server.stop() # check again the health response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "failed" # start the server again mocked_aws_server.start() # should be good again response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) - assert data + app_status_check, error = assert_status( + response, status.HTTP_200_OK, AppStatusCheck + ) + assert app_status_check assert not error - app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" diff --git a/services/storage/tests/unit/test_handlers_locations.py b/services/storage/tests/unit/test_handlers_locations.py index cc236499ee9..4aae75e69de 100644 --- a/services/storage/tests/unit/test_handlers_locations.py +++ b/services/storage/tests/unit/test_handlers_locations.py @@ -4,69 +4,56 @@ # pylint:disable=too-many-arguments # pylint:disable=no-name-in-module -from typing import Any -import pytest -from aiohttp.test_utils import TestClient +import httpx +from fastapi import FastAPI, status +from models_library.api_schemas_storage.storage_schemas import FileLocation from models_library.users import UserID -from pytest_simcore.helpers.assert_checks import assert_status -from servicelib.aiohttp import status -from tests.helpers.utils import has_datcore_tokens +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from simcore_service_storage.datcore_dsm import DatCoreDataManager +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] -async def test_locations(client: TestClient, user_id: UserID): - resp = await client.get(f"/v0/locations?user_id={user_id}") - - payload = await resp.json() - assert resp.status == 200, str(payload) - - data, error = tuple(payload.get(k) for k in ("data", "error")) - - _locs = 2 if has_datcore_tokens() else 1 - assert len(data) == _locs - assert not error - - -@pytest.mark.parametrize( - "dry_run, fire_and_forget, expected_removed", - [ - (None, None, []), - (True, False, []), - (True, True, []), - (False, True, []), - (False, False, []), - ], -) -async def test_synchronise_meta_data_table( - client: TestClient, - location_id: int, +async def test_locations( + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, - dry_run: bool | None, - fire_and_forget: bool | None, - expected_removed: list, + fake_datcore_tokens: tuple[str, str], ): - assert client.app - query_params: dict[str, Any] = {"user_id": user_id} - if dry_run: - query_params["dry_run"] = f"{dry_run}" - if fire_and_forget: - query_params["fire_and_forget"] = f"{fire_and_forget}" - url = ( - client.app.router["synchronise_meta_data_table"] - .url_for(location_id=f"{location_id}") - .with_query(**query_params) + url = url_from_operation_id( + client, initialized_app, "list_storage_locations" + ).with_query(user_id=user_id) + response = await client.get(f"{url}") + data, _ = assert_status(response, status.HTTP_200_OK, list[FileLocation]) + assert data + assert len(data) == 2 + assert data[0] == FileLocation( + id=SimcoreS3DataManager.get_location_id(), + name=SimcoreS3DataManager.get_location_name(), ) - resp = await client.post( - f"{url}", + assert data[1] == FileLocation( + id=DatCoreDataManager.get_location_id(), + name=DatCoreDataManager.get_location_name(), ) - data, error = await assert_status(resp, status.HTTP_200_OK) - assert not error + + +async def test_locations_without_tokens( + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, +): + url = url_from_operation_id( + client, initialized_app, "list_storage_locations" + ).with_query(user_id=user_id) + response = await client.get(f"{url}") + data, _ = assert_status(response, status.HTTP_200_OK, list[FileLocation]) assert data - assert data["dry_run"] == (False if dry_run is None else dry_run) - assert data["fire_and_forget"] == ( - False if fire_and_forget is None else fire_and_forget + assert len(data) == 1 + assert data[0] == FileLocation( + id=SimcoreS3DataManager.get_location_id(), + name=SimcoreS3DataManager.get_location_name(), ) - assert data["removed"] == expected_removed diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py new file mode 100644 index 00000000000..31cb5c85061 --- /dev/null +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -0,0 +1,750 @@ +# pylint:disable=no-name-in-module +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=too-many-positional-arguments +# pylint:disable=unused-argument +# pylint:disable=unused-variable + + +import random +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any, TypeAlias +from urllib.parse import quote + +import httpx +import pytest +import sqlalchemy as sa +from faker import Faker +from fastapi import FastAPI, status +from fastapi_pagination.cursor import CursorPage +from models_library.api_schemas_storage.storage_schemas import ( + PathMetaDataGet, + PathTotalSizeCreate, +) +from models_library.api_schemas_webserver.storage import MAX_NUMBER_OF_PATHS_PER_PAGE +from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID +from models_library.users import UserID +from pydantic import ByteSize, TypeAdapter +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams +from simcore_postgres_database.models.projects import projects +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy.ext.asyncio import AsyncEngine + +pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_ops_services_selection = ["adminer"] + +_IsFile: TypeAlias = bool + + +def _filter_and_group_paths_one_level_deeper( + paths: list[Path], prefix: Path +) -> list[tuple[Path, _IsFile]]: + relative_paths = (path for path in paths if path.is_relative_to(prefix)) + return sorted( + { + ( + (path, len(path.relative_to(prefix).parts) == 1) + if len(path.relative_to(prefix).parts) == 1 + else (prefix / path.relative_to(prefix).parts[0], False) + ) + for path in relative_paths + }, + key=lambda x: x[0], + ) + + +async def _assert_list_paths( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + *, + file_filter: Path | None, + limit: int = 25, + expected_paths: list[tuple[Path, _IsFile]], + check_total: bool = True, +) -> CursorPage[PathMetaDataGet]: + offset = 0 + total_expected = len(expected_paths) + next_cursor = 0 # NOTE: this will initialize + total_received = 0 + while next_cursor is not None: + url = url_from_operation_id( + client, initialized_app, "list_paths", location_id=f"{location_id}" + ).with_query( + user_id=user_id, + size=limit, + ) + if next_cursor: + url = url.update_query(cursor=next_cursor) + + if file_filter is not None: + url = url.update_query(file_filter=f"{file_filter}") + response = await client.get(f"{url}") + + page_of_files, _ = assert_status( + response, + status.HTTP_200_OK, + CursorPage[PathMetaDataGet], + expect_envelope=False, + ) + assert page_of_files + assert len(page_of_files.items) == min(limit, total_expected - offset) + + for (expected_path, is_file), received_path in zip( + expected_paths[offset : offset + limit], page_of_files.items, strict=True + ): + assert received_path.path == expected_path + if is_file: + assert received_path.file_meta_data is not None + else: + assert received_path.file_meta_data is None + + if check_total: + assert page_of_files.total == total_expected + else: + assert page_of_files.total is None + next_cursor = page_of_files.next_page + total_received += len(page_of_files.items) + offset += limit + assert total_received == total_expected + assert page_of_files.next_page is None + return page_of_files + + +async def test_list_paths_root_folder_of_empty_returns_nothing( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + fake_datcore_tokens: tuple[str, str], +): + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=None, + expected_paths=[], + ) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=10, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ) + ], + ids=str, +) +async def test_list_paths_pagination( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], +): + project, list_of_files = with_random_project_with_files + num_nodes = len(list(project["workbench"])) + + # ls the nodes (DB-based) + file_filter = Path(project["uuid"]) + expected_paths = sorted( + ((file_filter / node_key, False) for node_key in project["workbench"]), + key=lambda x: x[0], + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=file_filter, + expected_paths=expected_paths, + limit=int(num_nodes / 2 + 0.5), + ) + + # ls in the workspace (S3-based) + # ls in the workspace + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + workspace_file_filter = file_filter / f"{selected_node_id}" / "workspace" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, workspace_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=workspace_file_filter, + expected_paths=expected_paths, + limit=1, + check_total=False, + ) + # ls in until we get to some files + while selected_subfolders := [p for p in expected_paths if p[1] is False]: + selected_path_filter = random.choice(selected_subfolders) # noqa: S311 + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, selected_path_filter[0] + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=selected_path_filter[0], + expected_paths=expected_paths, + check_total=False, + ) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("0b"),), + workspace_files_count=MAX_NUMBER_OF_PATHS_PER_PAGE, + ) + ], + ids=str, +) +async def test_list_paths_pagination_large_page( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], +): + project, list_of_files = with_random_project_with_files + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + workspace_file_filter = Path(project["uuid"]) / f"{selected_node_id}" / "workspace" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, workspace_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=workspace_file_filter, + expected_paths=expected_paths, + check_total=False, + limit=MAX_NUMBER_OF_PATHS_PER_PAGE, + ) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params, num_projects", + [ + ( + ProjectWithFilesParams( + num_nodes=3, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + 3, + ) + ], + ids=str, +) +async def test_list_paths( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + random_project_with_files: Callable[ + [ProjectWithFilesParams], + Awaitable[ + tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], + ], + project_params: ProjectWithFilesParams, + num_projects: int, +): + project_to_files_mapping = [ + await random_project_with_files(project_params) for _ in range(num_projects) + ] + project_to_files_mapping.sort(key=lambda x: x[0]["uuid"]) + + # ls root returns our projects + expected_paths = sorted( + ((Path(f"{prj_db['uuid']}"), False) for prj_db, _ in project_to_files_mapping), + key=lambda x: x[0], + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=None, + expected_paths=expected_paths, + ) + + # ls with only some part of the path should return only the projects that match + selected_project, selected_project_files = random.choice( # noqa: S311 + project_to_files_mapping + ) + partial_file_filter = Path( + selected_project["uuid"][: len(selected_project["uuid"]) // 2] + ) + partial_expected_paths = [ + p for p in expected_paths if f"{p[0]}".startswith(f"{partial_file_filter}") + ] + + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=partial_file_filter, + expected_paths=partial_expected_paths, + ) + + # now we ls inside one of the projects returns the nodes + file_filter = Path(selected_project["uuid"]) + expected_paths = sorted( + ((file_filter / node_key, False) for node_key in selected_project["workbench"]), + key=lambda x: x[0], + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=file_filter, + expected_paths=expected_paths, + ) + + # now we ls in one of the nodes + selected_node_id = NodeID( + random.choice(list(selected_project["workbench"])) # noqa: S311 + ) + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in selected_project_files[selected_node_id] + ] + file_filter = file_filter / f"{selected_node_id}" + expected_node_files = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, + file_filter, + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=file_filter, + expected_paths=expected_node_files, + ) + + # ls in the outputs will list 1 entry which is a folder + node_outputs_file_filter = file_filter / "outputs" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, node_outputs_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=node_outputs_file_filter, + expected_paths=expected_paths, + ) + + # ls in output_3 shall reveal the file + node_outputs_file_filter = file_filter / "outputs" / "output_3" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, node_outputs_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=node_outputs_file_filter, + expected_paths=expected_paths, + ) + + # ls in the workspace + workspace_file_filter = file_filter / "workspace" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, workspace_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=workspace_file_filter, + expected_paths=expected_paths, + check_total=False, + ) + # ls in until we get to some files + while selected_subfolders := [p for p in expected_paths if p[1] is False]: + selected_path_filter = random.choice(selected_subfolders) # noqa: S311 + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, selected_path_filter[0] + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=selected_path_filter[0], + expected_paths=expected_paths, + check_total=False, + ) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("0b"),), + workspace_files_count=10, + ) + ], + ids=str, +) +async def test_list_paths_with_display_name_containing_slashes( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + sqlalchemy_async_engine: AsyncEngine, +): + project, list_of_files = with_random_project_with_files + project_name_with_slashes = "soméà$èq¨thing with/ slas/h/es/" + node_name_with_non_ascii = "my node / is not ascii: éàèù" + # adjust project to contain "difficult" characters + async with sqlalchemy_async_engine.begin() as conn: + result = await conn.execute( + sa.update(projects) + .where(projects.c.uuid == project["uuid"]) + .values(name=project_name_with_slashes) + .returning(sa.literal_column(f"{projects.c.name}, {projects.c.workbench}")) + ) + row = result.one() + assert row.name == project_name_with_slashes + project_workbench = row.workbench + assert len(project_workbench) == 1 + node = next(iter(project_workbench.values())) + node["label"] = node_name_with_non_ascii + result = await conn.execute( + sa.update(projects) + .where(projects.c.uuid == project["uuid"]) + .values(workbench=project_workbench) + .returning(sa.literal_column(f"{projects.c.name}, {projects.c.workbench}")) + ) + row = result.one() + + # ls the root + file_filter = None + expected_paths = [(Path(project["uuid"]), False)] + + page_of_paths = await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=file_filter, + expected_paths=expected_paths, + ) + + assert page_of_paths.items[0].display_path == Path( + quote(project_name_with_slashes, safe="") + ), "display path parts should be url encoded" + + # ls the nodes to ensure / is still there between project and node + file_filter = Path(project["uuid"]) + expected_paths = sorted( + ((file_filter / node_key, False) for node_key in project["workbench"]), + key=lambda x: x[0], + ) + assert len(expected_paths) == 1, "test configuration problem" + page_of_paths = await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=file_filter, + expected_paths=expected_paths, + ) + assert page_of_paths.items[0].display_path == Path( + quote(project_name_with_slashes, safe="") + ) / quote( + node_name_with_non_ascii, safe="" + ), "display path parts should be url encoded" + + # ls in the node workspace + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + workspace_file_filter = file_filter / f"{selected_node_id}" / "workspace" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, workspace_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=workspace_file_filter, + expected_paths=expected_paths, + check_total=False, + ) + + # ls in until we get to some files + while selected_subfolders := [p for p in expected_paths if p[1] is False]: + selected_path_filter = random.choice(selected_subfolders) # noqa: S311 + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, selected_path_filter[0] + ) + page_of_paths = await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=selected_path_filter[0], + expected_paths=expected_paths, + check_total=False, + ) + + expected_display_path = "/".join( + [ + quote(project_name_with_slashes, safe=""), + quote(node_name_with_non_ascii, safe=""), + *(expected_paths[0][0].parts[2:]), + ], + ) + assert page_of_paths.items[0].display_path == Path( + expected_display_path + ), "display path parts should be url encoded" + + +async def _assert_compute_path_size( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + *, + path: Path, + expected_total_size: int, +) -> ByteSize: + url = url_from_operation_id( + client, + initialized_app, + "compute_path_size", + location_id=f"{location_id}", + path=f"{path}", + ).with_query(user_id=user_id) + response = await client.post(f"{url}") + + received, _ = assert_status( + response, + status.HTTP_200_OK, + PathTotalSizeCreate, + ) + assert received + assert received.path == path + assert received.size == expected_total_size + return received.size + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=5, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ) + ], + ids=str, +) +async def test_path_compute_size( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + project_params: ProjectWithFilesParams, +): + assert ( + len(project_params.allowed_file_sizes) == 1 + ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" + project, list_of_files = with_random_project_with_files + + total_num_files = sum( + len(files_in_node) for files_in_node in list_of_files.values() + ) + + # get size of a full project + expected_total_size = project_params.allowed_file_sizes[0] * total_num_files + path = Path(project["uuid"]) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of one of the nodes + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + path = Path(project["uuid"]) / f"{selected_node_id}" + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of the outputs of one of the nodes + path = Path(project["uuid"]) / f"{selected_node_id}" / "outputs" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of workspace in one of the nodes (this is semi-cached in the DB) + path = Path(project["uuid"]) / f"{selected_node_id}" / "workspace" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + workspace_total_size = await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of folders inside the workspace + folders_inside_workspace = [ + p[0] + for p in _filter_and_group_paths_one_level_deeper(selected_node_s3_keys, path) + if p[1] is False + ] + accumulated_subfolder_size = 0 + for workspace_subfolder in folders_inside_workspace: + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{workspace_subfolder}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + accumulated_subfolder_size += await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=workspace_subfolder, + expected_total_size=expected_total_size, + ) + + assert workspace_total_size == accumulated_subfolder_size + + +async def test_path_compute_size_inexistent_path( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + faker: Faker, + fake_datcore_tokens: tuple[str, str], +): + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=Path(faker.file_path(absolute=False)), + expected_total_size=0, + ) diff --git a/services/storage/tests/unit/test_handlers_simcore_s3.py b/services/storage/tests/unit/test_handlers_simcore_s3.py index bcda9331f2b..d3768fd09eb 100644 --- a/services/storage/tests/unit/test_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_handlers_simcore_s3.py @@ -13,35 +13,44 @@ from pathlib import Path from typing import Any, Literal +import httpx import pytest import sqlalchemy as sa -from aiohttp import ClientResponseError -from aiohttp.test_utils import TestClient -from aiopg.sa.engine import Engine from aws_library.s3 import SimcoreS3API from faker import Faker -from models_library.api_schemas_storage import FileMetaDataGet, FoldersBody +from fastapi import FastAPI +from models_library.api_schemas_storage.storage_schemas import ( + FileMetaDataGet, + FoldersBody, +) from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import ByteSize, TypeAdapter -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context +from pytest_simcore.helpers.storage_utils import ( + FileIDDict, + ProjectWithFilesParams, + get_updated_project, +) +from pytest_simcore.helpers.storage_utils_file_meta_data import ( + assert_file_meta_data_in_db, +) +from pytest_simcore.helpers.storage_utils_project import clone_project_data from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status -from servicelib.aiohttp.long_running_tasks.client import long_running_task_request +from servicelib.fastapi.long_running_tasks.client import long_running_task_request from settings_library.s3 import S3Settings from simcore_postgres_database.storage_models import file_meta_data from simcore_service_storage.models import SearchFilesQueryParams from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db -from tests.helpers.utils_project import clone_project_data +from sqlalchemy.ext.asyncio import AsyncEngine from yarl import URL -from ..helpers.utils import get_updated_project - pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer", "minio"] @@ -71,38 +80,37 @@ async def _fake_download_to_file_or_raise(session, url, dest_path): ) -async def test_simcore_s3_access_returns_default(client: TestClient): - assert client.app - url = ( - client.app.router["get_or_create_temporary_s3_access"] - .url_for() - .with_query(user_id=1) - ) +async def test_simcore_s3_access_returns_default( + initialized_app: FastAPI, client: httpx.AsyncClient +): + url = url_from_operation_id( + client, initialized_app, "get_or_create_temporary_s3_access" + ).with_query(user_id=1) + response = await client.post(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + received_settings, error = assert_status(response, status.HTTP_200_OK, S3Settings) assert not error - assert data - received_settings = S3Settings.model_validate(data) assert received_settings async def _request_copy_folders( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, source_project: dict[str, Any], dst_project: dict[str, Any], nodes_map: dict[NodeID, NodeID], ) -> dict[str, Any]: - assert client.app - url = client.make_url( - f"{(client.app.router['copy_folders_from_project'].url_for().with_query(user_id=user_id))}" - ) + url = url_from_operation_id( + client, initialized_app, "copy_folders_from_project" + ).with_query(user_id=user_id) + with log_context( logging.INFO, f"Copying folders from {source_project['uuid']} to {dst_project['uuid']}", ) as ctx: async for lr_task in long_running_task_request( - client.session, + client, url, json=jsonable_encoder( FoldersBody( @@ -118,13 +126,12 @@ async def _request_copy_folders( async def test_copy_folders_from_non_existing_project( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, create_project: Callable[[], Awaitable[dict[str, Any]]], faker: Faker, ): - assert client.app - src_project = await create_project() incorrect_src_project = deepcopy(src_project) incorrect_src_project["uuid"] = faker.uuid4() @@ -132,36 +139,45 @@ async def test_copy_folders_from_non_existing_project( incorrect_dst_project = deepcopy(dst_project) incorrect_dst_project["uuid"] = faker.uuid4() - with pytest.raises( - ClientResponseError, match=f"{incorrect_src_project['uuid']} was not found" - ) as exc_info: + with pytest.raises(httpx.HTTPStatusError, match="404") as exc_info: await _request_copy_folders( + initialized_app, client, user_id, incorrect_src_project, dst_project, nodes_map={}, ) - assert exc_info.value.status == status.HTTP_404_NOT_FOUND + assert_status( + exc_info.value.response, + status.HTTP_404_NOT_FOUND, + None, + expected_msg=f"{incorrect_src_project['uuid']} was not found", + ) - with pytest.raises( - ClientResponseError, match=f"{incorrect_dst_project['uuid']} was not found" - ) as exc_info: + with pytest.raises(httpx.HTTPStatusError, match="404") as exc_info: await _request_copy_folders( + initialized_app, client, user_id, src_project, incorrect_dst_project, nodes_map={}, ) - assert exc_info.value.status == status.HTTP_404_NOT_FOUND + assert_status( + exc_info.value.response, + status.HTTP_404_NOT_FOUND, + None, + expected_msg=f"{incorrect_dst_project['uuid']} was not found", + ) async def test_copy_folders_from_empty_project( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, create_project: Callable[[], Awaitable[dict[str, Any]]], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, ): # we will copy from src to dst @@ -169,6 +185,7 @@ async def test_copy_folders_from_empty_project( dst_project = await create_project() data = await _request_copy_folders( + initialized_app, client, user_id, src_project, @@ -177,7 +194,7 @@ async def test_copy_folders_from_empty_project( ) assert data == jsonable_encoder(dst_project) # check there is nothing in the dst project - async with aiopg_engine.acquire() as conn: + async with sqlalchemy_async_engine.connect() as conn: num_entries = await conn.scalar( sa.select(sa.func.count()) .select_from(file_meta_data) @@ -192,36 +209,51 @@ def short_dsm_cleaner_interval(monkeypatch: pytest.MonkeyPatch) -> int: return 1 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("210Mib"),), + allowed_file_checksums=( + TypeAdapter(SHA256Str).validate_python( + "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" + ), + ), + workspace_files_count=0, + ), + ], + ids=str, +) async def test_copy_folders_from_valid_project_with_one_large_file( + initialized_app: FastAPI, short_dsm_cleaner_interval: int, - client: TestClient, + client: httpx.AsyncClient, user_id: UserID, create_project: Callable[[], Awaitable[dict[str, Any]]], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, random_project_with_files: Callable[ - [int, tuple[ByteSize], tuple[SHA256Str]], + [ProjectWithFilesParams], Awaitable[ - tuple[ - dict[str, Any], - dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]], - ] + tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] ], ], + project_params: ProjectWithFilesParams, ): # 1. create a src project with 1 large file - sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python( - "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" - ) - src_project, src_projects_list = await random_project_with_files( - 1, - (TypeAdapter(ByteSize).validate_python("210Mib"),), - (sha256_checksum,), - ) + src_project, src_projects_list = await random_project_with_files(project_params) # 2. create a dst project without files dst_project, nodes_map = clone_project_data(src_project) dst_project = await create_project(**dst_project) # copy the project files data = await _request_copy_folders( + initialized_app, client, user_id, src_project, @@ -229,7 +261,7 @@ async def test_copy_folders_from_valid_project_with_one_large_file( nodes_map={NodeID(i): NodeID(j) for i, j in nodes_map.items()}, ) assert data == jsonable_encoder( - await get_updated_project(aiopg_engine, dst_project["uuid"]) + await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) # check that file meta data was effectively copied for src_node_id in src_projects_list: @@ -243,10 +275,10 @@ async def test_copy_folders_from_valid_project_with_one_large_file( checksum: Any = src_file["sha256_checksum"] assert isinstance(checksum, str) await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( - src_project["uuid"], dst_project["uuid"] + f"{src_project['uuid']}", dst_project["uuid"] ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, @@ -259,30 +291,62 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=12, + allowed_file_sizes=( + TypeAdapter(ByteSize).validate_python("7Mib"), + TypeAdapter(ByteSize).validate_python("110Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + ), + allowed_file_checksums=( + TypeAdapter(SHA256Str).validate_python( + "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd" + ), + TypeAdapter(SHA256Str).validate_python( + "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e" + ), + TypeAdapter(SHA256Str).validate_python( + "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3" + ), + ), + workspace_files_count=0, + ), + ], + ids=str, +) async def test_copy_folders_from_valid_project( short_dsm_cleaner_interval: int, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, create_project: Callable[[], Awaitable[dict[str, Any]]], create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, random_project_with_files: Callable[ - ..., + [ProjectWithFilesParams], Awaitable[ - tuple[ - dict[str, Any], - dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | SHA256Str]]], - ] + tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] ], ], + project_params: ProjectWithFilesParams, ): # 1. create a src project with some files - src_project, src_projects_list = await random_project_with_files() + src_project, src_projects_list = await random_project_with_files(project_params) # 2. create a dst project without files dst_project, nodes_map = clone_project_data(src_project) dst_project = await create_project(**dst_project) # copy the project files data = await _request_copy_folders( + initialized_app, client, user_id, src_project, @@ -290,7 +354,7 @@ async def test_copy_folders_from_valid_project( nodes_map={NodeID(i): NodeID(j) for i, j in nodes_map.items()}, ) assert data == jsonable_encoder( - await get_updated_project(aiopg_engine, dst_project["uuid"]) + await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) # check that file meta data was effectively copied @@ -305,10 +369,10 @@ async def test_copy_folders_from_valid_project( checksum: Any = src_file["sha256_checksum"] assert isinstance(checksum, str) await assert_file_meta_data_in_db( - aiopg_engine, + sqlalchemy_async_engine, file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( - src_project["uuid"], dst_project["uuid"] + f"{src_project['uuid']}", dst_project["uuid"] ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, @@ -324,7 +388,8 @@ async def test_copy_folders_from_valid_project( async def _create_and_delete_folders_from_project( user_id: UserID, project: dict[str, Any], - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, project_db_creator: Callable, check_list_files: bool, ) -> None: @@ -333,6 +398,7 @@ async def _create_and_delete_folders_from_project( # creating a copy data = await _request_copy_folders( + initialized_app, client, user_id, project, @@ -347,35 +413,38 @@ async def _create_and_delete_folders_from_project( project_id = data["uuid"] # list data to check all is here - assert client.app + if check_list_files: - url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{SimcoreS3DataManager.get_location_id()}") - .with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") - ) + url = url_from_operation_id( + client, + initialized_app, + "list_files_metadata", + location_id=f"{SimcoreS3DataManager.get_location_id()}", + ).with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") + resp = await client.get(f"{url}") - data, error = await assert_status(resp, status.HTTP_200_OK) + data, error = assert_status(resp, status.HTTP_200_OK, list[FileMetaDataGet]) assert not error # DELETING - url = ( - client.app.router["delete_folders_of_project"] - .url_for(folder_id=project_id) - .with_query(user_id=f"{user_id}") - ) + url = url_from_operation_id( + client, + initialized_app, + "delete_folders_of_project", + folder_id=project_id, + ).with_query(user_id=f"{user_id}") resp = await client.delete(f"{url}") - - await assert_status(resp, expected_status_code=status.HTTP_204_NO_CONTENT) + assert_status(resp, status.HTTP_204_NO_CONTENT, None) # list data is gone if check_list_files: - url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{SimcoreS3DataManager.get_location_id()}") - .with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") - ) + url = url_from_operation_id( + client, + initialized_app, + "list_files_metadata", + location_id=f"{SimcoreS3DataManager.get_location_id()}", + ).with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") resp = await client.get(f"{url}") - data, error = await assert_status(resp, status.HTTP_200_OK) + data, error = assert_status(resp, status.HTTP_200_OK, list[FileMetaDataGet]) assert not error assert not data @@ -386,47 +455,48 @@ def set_log_levels_for_noisy_libraries() -> None: logging.getLogger("werkzeug").setLevel(logging.WARNING) -@pytest.fixture -async def with_random_project_with_files( - random_project_with_files: Callable[ - ..., - Awaitable[ - tuple[ - dict[str, Any], - dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]], - ] - ], - ], -) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],]: - return await random_project_with_files( - file_sizes=( - TypeAdapter(ByteSize).validate_python("1Mib"), - TypeAdapter(ByteSize).validate_python("2Mib"), - TypeAdapter(ByteSize).validate_python("5Mib"), - ) - ) - - async def test_connect_to_external( set_log_levels_for_noisy_libraries: None, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, project_id: ProjectID, ): - assert client.app - url = ( - client.app.router["get_files_metadata"] - .url_for(location_id=f"{SimcoreS3DataManager.get_location_id()}") - .with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") - ) + url = url_from_operation_id( + client, + initialized_app, + "list_files_metadata", + location_id=f"{SimcoreS3DataManager.get_location_id()}", + ).with_query(user_id=f"{user_id}", uuid_filter=f"{project_id}") resp = await client.get(f"{url}") - data, error = await assert_status(resp, status.HTTP_200_OK) + data, error = assert_status(resp, status.HTTP_200_OK, list[FileMetaDataGet]) print(data) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=3, + allowed_file_sizes=( + TypeAdapter(ByteSize).validate_python("7Mib"), + TypeAdapter(ByteSize).validate_python("110Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + ), + workspace_files_count=0, + ) + ], +) async def test_create_and_delete_folders_from_project( set_log_levels_for_noisy_libraries: None, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, create_project: Callable[..., Awaitable[dict[str, Any]]], with_random_project_with_files: tuple[ @@ -437,15 +507,42 @@ async def test_create_and_delete_folders_from_project( ): project_in_db, _ = with_random_project_with_files await _create_and_delete_folders_from_project( - user_id, project_in_db, client, create_project, check_list_files=True + user_id, + project_in_db, + initialized_app, + client, + create_project, + check_list_files=True, ) +@pytest.mark.flaky(max_runs=3) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=3, + allowed_file_sizes=( + TypeAdapter(ByteSize).validate_python("7Mib"), + TypeAdapter(ByteSize).validate_python("110Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + ), + workspace_files_count=0, + ) + ], +) @pytest.mark.parametrize("num_concurrent_calls", [50]) async def test_create_and_delete_folders_from_project_burst( set_log_levels_for_noisy_libraries: None, minio_s3_settings_envs: EnvVarsDict, - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, with_random_project_with_files: tuple[ dict[str, Any], @@ -460,7 +557,12 @@ async def test_create_and_delete_folders_from_project_burst( await asyncio.gather( *[ _create_and_delete_folders_from_project( - user_id, project_in_db, client, create_project, check_list_files=False + user_id, + project_in_db, + initialized_app, + client, + create_project, + check_list_files=False, ) for _ in range(num_concurrent_calls) ] @@ -504,34 +606,35 @@ async def search_files_query_params( return q +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("expected_number_of_user_files", [0, 1, 3]) @pytest.mark.parametrize("query_params_choice", ["default", "limited", "with_offset"]) async def test_search_files_request( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, uploaded_file_ids: list[SimcoreS3FileID], query_params_choice: str, search_files_query_params: SearchFilesQueryParams, ): - assert client.app assert query_params_choice assert search_files_query_params.user_id == user_id - - url = ( - client.app.router["search_files"] - .url_for() - .with_query( - jsonable_encoder( - search_files_query_params, exclude_unset=True, exclude_none=True - ) + url = url_from_operation_id(client, initialized_app, "search_files").with_query( + jsonable_encoder( + search_files_query_params, exclude_unset=True, exclude_none=True ) ) + response = await client.post(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + found, error = assert_status(response, status.HTTP_200_OK, list[FileMetaDataGet]) assert not error - - found = TypeAdapter(list[FileMetaDataGet]).validate_python(data) + assert found is not None expected = uploaded_file_ids[ search_files_query_params.offset : search_files_query_params.offset @@ -540,11 +643,18 @@ async def test_search_files_request( assert [_.file_uuid for _ in found] == expected +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("search_startswith", [True, False]) @pytest.mark.parametrize("search_sha256_checksum", [True, False]) @pytest.mark.parametrize("kind", ["owned", "read", None]) async def test_search_files( - client: TestClient, + initialized_app: FastAPI, + client: httpx.AsyncClient, user_id: UserID, upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], faker: Faker, @@ -552,32 +662,28 @@ async def test_search_files( search_sha256_checksum: bool, kind: Literal["owned"], ): - assert client.app _file_name: str = faker.file_name() _sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python(faker.sha256()) - url = ( - client.app.router["search_files"] - .url_for() - .with_query( - jsonable_encoder( - { - "user_id": user_id, - "kind": kind, - }, - exclude_none=True, - ) + url = url_from_operation_id(client, initialized_app, "search_files").with_query( + jsonable_encoder( + { + "user_id": user_id, + "kind": kind, + }, + exclude_none=True, ) ) response = await client.post(f"{url}") if kind != "owned": - await assert_status(response, status.HTTP_422_UNPROCESSABLE_ENTITY) + assert_status(response, status.HTTP_422_UNPROCESSABLE_ENTITY, None) return - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds # let's upload some files now @@ -588,9 +694,11 @@ async def test_search_files( ) # search again should return something response = await client.post(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) + assert list_fmds assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -604,9 +712,11 @@ async def test_search_files( url.update_query(sha256_checksum=_sha256_checksum) response = await client.post(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) + assert list_fmds assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -624,7 +734,8 @@ async def test_search_files( if search_startswith or search_sha256_checksum: response = await client.post(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + list_fmds, error = assert_status( + response, status.HTTP_200_OK, list[FileMetaDataGet] + ) assert not error - list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds diff --git a/services/storage/tests/unit/test_models.py b/services/storage/tests/unit/test_models.py index 250b037e5cf..da33d24ad50 100644 --- a/services/storage/tests/unit/test_models.py +++ b/services/storage/tests/unit/test_models.py @@ -1,7 +1,7 @@ import uuid import pytest -from models_library.api_schemas_storage import S3BucketName +from models_library.api_schemas_storage.storage_schemas import S3BucketName from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID, StorageFileID from pydantic import TypeAdapter, ValidationError diff --git a/services/storage/tests/unit/test_resources.py b/services/storage/tests/unit/test_resources.py index d74fedcc437..29aa51a6f72 100644 --- a/services/storage/tests/unit/test_resources.py +++ b/services/storage/tests/unit/test_resources.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from simcore_service_storage.resources import storage_resources +from simcore_service_storage.core.resources import storage_resources log = logging.getLogger(__name__) diff --git a/services/storage/tests/unit/test_rpc_handlers_paths.py b/services/storage/tests/unit/test_rpc_handlers_paths.py new file mode 100644 index 00000000000..ee6787b22a3 --- /dev/null +++ b/services/storage/tests/unit/test_rpc_handlers_paths.py @@ -0,0 +1,71 @@ +# pylint:disable=no-name-in-module +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=too-many-positional-arguments +# pylint:disable=unused-argument +# pylint:disable=unused-variable + + +from collections.abc import Awaitable, Callable +from pathlib import Path +from unittest import mock + +import pytest +from faker import Faker +from fastapi import FastAPI +from models_library.projects_nodes_io import LocationID +from models_library.users import UserID +from pytest_mock import MockerFixture +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.storage.paths import compute_path_size +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager + +pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_ops_services_selection = ["adminer"] + + +@pytest.fixture +async def storage_rabbitmq_rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("pytest_storage_rpc_client") + assert rpc_client + return rpc_client + + +@pytest.fixture +async def mock_celery_send_task(mocker: MockerFixture, faker: Faker) -> mock.AsyncMock: + def mocked_send_task(*args, **kwargs): + return faker.uuid4() + + return mocker.patch( + "simcore_service_storage.modules.celery.client.CeleryTaskQueueClient.send_task", + side_effect=mocked_send_task, + ) + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +async def test_path_compute_size_calls_in_celery( + initialized_app: FastAPI, + storage_rabbitmq_rpc_client: RabbitMQRPCClient, + location_id: LocationID, + user_id: UserID, + faker: Faker, + mock_celery_send_task: mock.AsyncMock, +): + received, job_id_data = await compute_path_size( + storage_rabbitmq_rpc_client, + user_id=user_id, + product_name=faker.name(), + location_id=location_id, + path=Path(faker.file_path(absolute=False)), + ) + mock_celery_send_task.assert_called_once() + assert received + assert job_id_data diff --git a/services/storage/tests/unit/test_simcore_s3_dsm.py b/services/storage/tests/unit/test_simcore_s3_dsm.py index 41c69355025..fba91a14707 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm.py @@ -2,21 +2,21 @@ # pylint:disable=redefined-outer-name from collections.abc import Awaitable, Callable -from contextlib import AbstractAsyncContextManager from pathlib import Path import pytest -from aiopg.sa.engine import Engine from faker import Faker -from models_library.api_schemas_storage import FileUploadSchema from models_library.basic_types import SHA256Str -from models_library.projects_nodes_io import SimcoreS3FileID +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter -from simcore_service_storage import db_file_meta_data +from pytest_simcore.helpers.storage_utils import FileIDDict from simcore_service_storage.models import FileMetaData -from simcore_service_storage.s3 import get_s3_client +from simcore_service_storage.modules.db.file_meta_data import FileMetaDataRepository +from simcore_service_storage.modules.s3 import get_s3_client from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] @@ -29,22 +29,32 @@ def file_size() -> ByteSize: @pytest.fixture def mock_copy_transfer_cb() -> Callable[..., None]: - def copy_transfer_cb(total_bytes_copied: int, *, file_name: str) -> None: - ... + def copy_transfer_cb(total_bytes_copied: int, *, file_name: str) -> None: ... return copy_transfer_cb +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test__copy_path_s3_s3( simcore_s3_dsm: SimcoreS3DataManager, create_directory_with_files: Callable[ - ..., AbstractAsyncContextManager[FileUploadSchema] + [str, ByteSize, int, int, ProjectID, NodeID], + Awaitable[ + tuple[SimcoreS3FileID, tuple[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + ], ], upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], file_size: ByteSize, user_id: UserID, + project_id: ProjectID, + node_id: NodeID, mock_copy_transfer_cb: Callable[..., None], - aiopg_engine: Engine, + sqlalchemy_async_engine: AsyncEngine, ): def _get_dest_file_id(src: SimcoreS3FileID) -> SimcoreS3FileID: return TypeAdapter(SimcoreS3FileID).validate_python( @@ -52,12 +62,13 @@ def _get_dest_file_id(src: SimcoreS3FileID) -> SimcoreS3FileID: ) async def _copy_s3_path(s3_file_id_to_copy: SimcoreS3FileID) -> None: - async with aiopg_engine.acquire() as conn: - exiting_fmd = await db_file_meta_data.get(conn, s3_file_id_to_copy) + existing_fmd = await FileMetaDataRepository.instance( + sqlalchemy_async_engine + ).get(file_id=s3_file_id_to_copy) await simcore_s3_dsm._copy_path_s3_s3( # noqa: SLF001 user_id=user_id, - src_fmd=exiting_fmd, + src_fmd=existing_fmd, dst_file_id=_get_dest_file_id(s3_file_id_to_copy), bytes_transfered_cb=mock_copy_transfer_cb, ) @@ -74,24 +85,23 @@ async def _count_files(s3_file_id: SimcoreS3FileID, expected_count: int) -> None # using directory - FILE_COUNT = 4 + FILE_COUNT = 20 SUBDIR_COUNT = 5 - async with create_directory_with_files( + s3_object, _ = await create_directory_with_files( dir_name="some-random", file_size_in_dir=file_size, subdir_count=SUBDIR_COUNT, file_count=FILE_COUNT, - ) as directory_file_upload: - assert len(directory_file_upload.urls) == 1 - assert directory_file_upload.urls[0].path - s3_object = directory_file_upload.urls[0].path.lstrip("/") + project_id=project_id, + node_id=node_id, + ) - s3_file_id_dir_src = TypeAdapter(SimcoreS3FileID).validate_python(s3_object) - s3_file_id_dir_dst = _get_dest_file_id(s3_file_id_dir_src) + s3_file_id_dir_src = TypeAdapter(SimcoreS3FileID).validate_python(s3_object) + s3_file_id_dir_dst = _get_dest_file_id(s3_file_id_dir_src) - await _count_files(s3_file_id_dir_dst, expected_count=0) - await _copy_s3_path(s3_file_id_dir_src) - await _count_files(s3_file_id_dir_dst, expected_count=FILE_COUNT * SUBDIR_COUNT) + await _count_files(s3_file_id_dir_dst, expected_count=0) + await _copy_s3_path(s3_file_id_dir_src) + await _count_files(s3_file_id_dir_dst, expected_count=FILE_COUNT) # using a single file @@ -99,6 +109,12 @@ async def _count_files(s3_file_id: SimcoreS3FileID, expected_count: int) -> None await _copy_s3_path(simcore_file_id) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_and_search( simcore_s3_dsm: SimcoreS3DataManager, upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], diff --git a/services/storage/tests/unit/test_simcore_s3_dsm_utils.py b/services/storage/tests/unit/test_simcore_s3_dsm_utils.py index 01869537c08..74c79a8cf36 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm_utils.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm_utils.py @@ -1,5 +1,5 @@ import pytest -from simcore_service_storage.simcore_s3_dsm_utils import compute_file_id_prefix +from simcore_service_storage.utils.simcore_s3_dsm_utils import compute_file_id_prefix @pytest.mark.parametrize( diff --git a/services/storage/tests/unit/test_utils.py b/services/storage/tests/unit/test_utils.py index 13b132ce045..3ee5d73a85a 100644 --- a/services/storage/tests/unit/test_utils.py +++ b/services/storage/tests/unit/test_utils.py @@ -10,10 +10,10 @@ from pathlib import Path from uuid import uuid4 +import httpx import pytest -from aiohttp import ClientSession from faker import Faker -from models_library.api_schemas_storage import UNDEFINED_SIZE_TYPE +from models_library.api_schemas_storage.storage_schemas import UNDEFINED_SIZE_TYPE from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from pydantic import ByteSize, HttpUrl, TypeAdapter @@ -21,7 +21,7 @@ from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -from simcore_service_storage.utils import ( +from simcore_service_storage.utils.utils import ( MAX_CHUNK_SIZE, download_to_file_or_raise, is_file_entry_valid, @@ -33,7 +33,7 @@ async def test_download_files(tmp_path: Path, httpbin_base_url: HttpUrl): destination = tmp_path / "data" expected_size = MAX_CHUNK_SIZE * 3 + 1000 - async with ClientSession() as session: + async with httpx.AsyncClient() as session: total_size = await download_to_file_or_raise( session, f"{httpbin_base_url}/bytes/{expected_size}", destination ) diff --git a/services/storage/tests/unit/test_utils_handlers.py b/services/storage/tests/unit/test_utils_handlers.py index cc220ceb3e2..c91b34cb9a8 100644 --- a/services/storage/tests/unit/test_utils_handlers.py +++ b/services/storage/tests/unit/test_utils_handlers.py @@ -1,63 +1,187 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable +from collections.abc import AsyncIterator + +import httpx import pytest -from aiohttp import web -from aiohttp.typedefs import Handler -from aws_library.s3 import S3KeyNotFoundError -from pydantic import BaseModel, ValidationError -from pytest_mock import MockerFixture -from servicelib.aiohttp.aiopg_utils import DBAPIError -from simcore_service_storage.db_access_layer import InvalidFileIdentifierError -from simcore_service_storage.exceptions import ( +from asyncpg import PostgresError +from aws_library.s3._errors import S3AccessError, S3KeyNotFoundError +from fastapi import FastAPI, HTTPException, status +from fastapi.exceptions import RequestValidationError +from httpx import AsyncClient +from pydantic import ValidationError +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from simcore_service_storage.exceptions.errors import ( FileAccessRightError, FileMetaDataNotFoundError, + LinkAlreadyExistsError, ProjectAccessRightError, ProjectNotFoundError, ) -from simcore_service_storage.utils_handlers import dsm_exception_handler - - -@pytest.fixture() -async def raising_handler( - mocker: MockerFixture, handler_exception: type[Exception] -) -> Handler: - mock = mocker.patch("aiohttp.typedefs.Handler", autospec=True) - mock.side_effect = handler_exception - return mock +from simcore_service_storage.exceptions.handlers import set_exception_handlers +from simcore_service_storage.modules.datcore_adapter.datcore_adapter_exceptions import ( + DatcoreAdapterTimeoutError, +) +from simcore_service_storage.modules.db.access_layer import InvalidFileIdentifierError @pytest.fixture -def mock_request(mocker: MockerFixture) -> web.Request: - return mocker.patch("aiohttp.web.Request", autospec=True) +def initialized_app() -> FastAPI: + app = FastAPI() + set_exception_handlers(app) + return app -class FakeErrorModel(BaseModel): - dummy: int = 1 +@pytest.fixture +async def client(initialized_app: FastAPI) -> AsyncIterator[AsyncClient]: + async with AsyncClient( + transport=httpx.ASGITransport(app=initialized_app), + base_url="http://test", + headers={"Content-Type": "application/json"}, + ) as client: + yield client @pytest.mark.parametrize( - "handler_exception, expected_web_response", + "exception, status_code", [ - (InvalidFileIdentifierError(identifier="x"), web.HTTPUnprocessableEntity), - (FileMetaDataNotFoundError(file_id="x"), web.HTTPNotFound), - (S3KeyNotFoundError(key="x", bucket="x"), web.HTTPNotFound), - (ProjectNotFoundError(project_id="x"), web.HTTPNotFound), - (FileAccessRightError(file_id="x", access_right="x"), web.HTTPForbidden), - (ProjectAccessRightError(project_id="x", access_right="x"), web.HTTPForbidden), ( - ValidationError.from_exception_data(title="test", line_errors=[]), - web.HTTPUnprocessableEntity, + InvalidFileIdentifierError( + identifier="pytest file identifier", details="pytest details" + ), + status.HTTP_422_UNPROCESSABLE_ENTITY, + ), + ( + FileMetaDataNotFoundError(file_id="pytest file ID"), + status.HTTP_404_NOT_FOUND, + ), + ( + S3KeyNotFoundError(key="pytest key", bucket="pytest bucket"), + status.HTTP_404_NOT_FOUND, + ), + ( + ProjectNotFoundError(project_id="pytest project ID"), + status.HTTP_404_NOT_FOUND, + ), + ( + FileAccessRightError( + access_right="pytest access rights", file_id="pytest file ID" + ), + status.HTTP_403_FORBIDDEN, + ), + ( + ProjectAccessRightError( + access_right="pytest access rights", project_id="pytest project ID" + ), + status.HTTP_403_FORBIDDEN, + ), + ( + LinkAlreadyExistsError(file_id="pytest file ID"), + status.HTTP_422_UNPROCESSABLE_ENTITY, + ), + ( + PostgresError("pytest postgres error"), + status.HTTP_503_SERVICE_UNAVAILABLE, + ), + ( + S3AccessError(), + status.HTTP_503_SERVICE_UNAVAILABLE, + ), + ( + DatcoreAdapterTimeoutError(msg="pytest datcore adapter timeout"), + status.HTTP_504_GATEWAY_TIMEOUT, + ), + ( + NotImplementedError("pytest not implemented error"), + status.HTTP_501_NOT_IMPLEMENTED, ), - (DBAPIError, web.HTTPServiceUnavailable), ], + ids=str, ) -async def test_dsm_exception_handler( - mock_request: web.Request, - raising_handler: Handler, - expected_web_response: type[web.HTTPClientError], +async def test_exception_handlers( + initialized_app: FastAPI, + client: AsyncClient, + exception: Exception, + status_code: int, +): + @initialized_app.get("/test") + async def test_endpoint(): + raise exception + + response = await client.get("/test") + assert_status(response, status_code, None, expected_msg=f"{exception}") + + +async def test_generic_http_exception_handler( + initialized_app: FastAPI, client: AsyncClient +): + @initialized_app.get("/test") + async def test_endpoint(): + raise HTTPException(status_code=status.HTTP_410_GONE) + + response = await client.get("/test") + assert_status(response, status.HTTP_410_GONE, None, expected_msg="Gone") + + +async def test_request_validation_error_handler( + initialized_app: FastAPI, client: AsyncClient ): - with pytest.raises(expected_web_response): - await dsm_exception_handler(mock_request, raising_handler) + _error_msg = "pytest request validation error" + + @initialized_app.get("/test") + async def test_endpoint(): + raise RequestValidationError(errors=[_error_msg]) + + response = await client.get("/test") + assert_status( + response, + status.HTTP_422_UNPROCESSABLE_ENTITY, + None, + expected_msg=_error_msg, + ) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY + + +async def test_validation_error_handler(initialized_app: FastAPI, client: AsyncClient): + _error_msg = "pytest request validation error" + + @initialized_app.get("/test") + async def test_endpoint(): + raise ValidationError.from_exception_data( + _error_msg, + line_errors=[], + ) + + response = await client.get("/test") + assert_status( + response, + status.HTTP_500_INTERNAL_SERVER_ERROR, + None, + expected_msg=f"0 validation errors for {_error_msg}", + ) + + +@pytest.mark.xfail( + reason="Generic exception handler is not working as expected as shown in https://github.com/ITISFoundation/osparc-simcore/blob/5732a12e07e63d5ce55010ede9b9ab543bb9b278/packages/service-library/tests/fastapi/test_exceptions_utils.py" +) +async def test_generic_exception_handler(initialized_app: FastAPI, client: AsyncClient): + _error_msg = "Generic pytest exception" + + @initialized_app.get("/test") + async def test_endpoint(): + raise Exception( # pylint: disable=broad-exception-raised # noqa: TRY002 + _error_msg + ) + + response = await client.get("/test") + assert_status( + response, + status.HTTP_500_INTERNAL_SERVER_ERROR, + None, + expected_msg=_error_msg, + ) diff --git a/services/web/server/VERSION b/services/web/server/VERSION index c5d4cee36a1..bf54d53ec26 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.51.0 +0.61.4 diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh index 08a812f7bb0..6b42600e91a 100755 --- a/services/web/server/docker/boot.sh +++ b/services/web/server/docker/boot.sh @@ -19,7 +19,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/web/server - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -31,7 +31,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi APP_LOG_LEVEL=${WEBSERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index caf883fc166..aecaa5793bb 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -32,10 +32,11 @@ aiosmtplib # email asyncpg # db captcha cryptography # security +deepdiff[optimize] # diffs data-structures faker # Only used in dev-mode for proof-of-concepts gunicorn[setproctitle] +httpx jinja_app_loader # email -json2html jsondiff msgpack openpyxl # excel diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index f35f5ab6212..7cf4d029ed9 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -88,6 +88,7 @@ anyio==4.3.0 # via # fast-depends # faststream + # httpx appdirs==1.4.4 # via pint arrow==1.2.3 @@ -145,6 +146,8 @@ certifi==2023.7.22 # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt + # httpcore + # httpx # requests cffi==1.17.1 # via cryptography @@ -186,6 +189,8 @@ cryptography==41.0.7 # -c requirements/../../../../requirements/constraints.txt # -r requirements/_base.in # aiohttp-session +deepdiff==8.1.1 + # via -r requirements/_base.in deprecated==1.2.14 # via # opentelemetry-api @@ -225,10 +230,46 @@ grpcio==1.66.0 # via opentelemetry-exporter-otlp-proto-grpc gunicorn==23.0.0 # via -r requirements/_base.in +h11==0.14.0 + # via httpcore +httpcore==1.0.7 + # via httpx +httpx==0.28.1 + # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../requirements/constraints.txt + # -r requirements/_base.in idna==3.3 # via # anyio # email-validator + # httpx # requests # yarl importlib-metadata==8.0.0 @@ -267,8 +308,6 @@ jinja2==3.1.2 # -c requirements/../../../../requirements/constraints.txt # aiohttp-jinja2 # swagger-ui-py -json2html==1.3.0 - # via -r requirements/_base.in jsondiff==2.0.0 # via -r requirements/_base.in jsonschema==3.2.0 @@ -416,6 +455,8 @@ opentelemetry-util-http==0.48b0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-requests +orderly-set==5.2.3 + # via deepdiff orjson==3.10.0 # via # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -465,6 +506,7 @@ orjson==3.10.0 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # deepdiff packaging==24.1 # via # -r requirements/../../../../packages/simcore-sdk/requirements/_base.in @@ -499,6 +541,8 @@ pycountry==23.12.11 # via -r requirements/_base.in pycparser==2.21 # via cffi +pycryptodome==3.21.0 + # via stream-zip pydantic==2.10.2 # via # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -585,6 +629,34 @@ pydantic-extra-types==2.9.0 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pydantic-settings==2.5.2 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../requirements/constraints.txt # -r requirements/../../../../packages/models-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -742,6 +814,10 @@ sqlalchemy==1.4.47 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic +stream-zip==0.0.83 + # via + # -r requirements/../../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in swagger-ui-py==23.9.23 # via -r requirements/_base.in tenacity==8.5.0 diff --git a/services/web/server/requirements/_test.in b/services/web/server/requirements/_test.in index 22ed8e423bc..368a9d03f7d 100644 --- a/services/web/server/requirements/_test.in +++ b/services/web/server/requirements/_test.in @@ -15,6 +15,8 @@ click coverage docker Faker +fastapi[standard] +fastapi-pagination flaky hypothesis jsonref @@ -38,6 +40,7 @@ pytest-sugar pytest-xdist python-dotenv redis +respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html tenacity types-aiofiles diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 6beb543d3f8..256b7662b20 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -14,6 +14,16 @@ alembic==1.8.1 # via # -c requirements/_base.txt # -r requirements/_test.in +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic +anyio==4.3.0 + # via + # -c requirements/_base.txt + # httpx + # starlette + # watchfiles async-timeout==4.0.3 # via # -c requirements/_base.txt @@ -31,10 +41,13 @@ attrs==21.4.0 # hypothesis # jsonschema # pytest-docker + # referencing certifi==2023.7.22 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt + # httpcore + # httpx # requests charset-normalizer==2.0.12 # via @@ -45,18 +58,34 @@ click==8.1.3 # via # -c requirements/_base.txt # -r requirements/_test.in -coverage==7.6.10 + # typer + # uvicorn +coverage==7.6.12 # via # -r requirements/_test.in # pytest-cov +dnspython==2.2.1 + # via + # -c requirements/_base.txt + # email-validator docker==7.1.0 # via -r requirements/_test.in +email-validator==2.2.0 + # via + # -c requirements/_base.txt + # fastapi execnet==2.1.1 # via pytest-xdist faker==19.6.1 # via # -c requirements/_base.txt # -r requirements/_test.in +fastapi==0.115.6 + # via -r requirements/_test.in +fastapi-cli==0.0.5 + # via fastapi +fastapi-pagination==0.12.34 + # via -r requirements/_test.in flaky==3.8.1 # via -r requirements/_test.in frozenlist==1.4.1 @@ -68,6 +97,23 @@ greenlet==2.0.2 # via # -c requirements/_base.txt # sqlalchemy +h11==0.14.0 + # via + # -c requirements/_base.txt + # httpcore + # uvicorn +httpcore==1.0.7 + # via + # -c requirements/_base.txt + # httpx +httptools==0.6.4 + # via uvicorn +httpx==0.28.1 + # via + # -c requirements/../../../../requirements/constraints.txt + # -c requirements/_base.txt + # fastapi + # respx hypothesis==6.91.0 # via -r requirements/_test.in icdiff==2.0.7 @@ -75,10 +121,18 @@ icdiff==2.0.7 idna==3.3 # via # -c requirements/_base.txt + # anyio + # email-validator + # httpx # requests # yarl iniconfig==2.0.0 # via pytest +jinja2==3.1.2 + # via + # -c requirements/../../../../requirements/constraints.txt + # -c requirements/_base.txt + # fastapi jsonref==1.1.0 # via -r requirements/_test.in jsonschema==3.2.0 @@ -92,16 +146,25 @@ mako==1.2.2 # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt # alembic +markdown-it-py==3.0.0 + # via + # -c requirements/_base.txt + # rich markupsafe==2.1.1 # via # -c requirements/_base.txt + # jinja2 # mako +mdurl==0.1.2 + # via + # -c requirements/_base.txt + # markdown-it-py multidict==6.1.0 # via # -c requirements/_base.txt # aiohttp # yarl -mypy==1.14.1 +mypy==1.15.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -121,11 +184,26 @@ pprintpp==0.4.0 # via pytest-icdiff py-cpuinfo==9.0.0 # via pytest-benchmark +pydantic==2.10.2 + # via + # -c requirements/../../../../requirements/constraints.txt + # -c requirements/_base.txt + # fastapi + # fastapi-pagination +pydantic-core==2.27.1 + # via + # -c requirements/_base.txt + # pydantic +pygments==2.15.1 + # via + # -c requirements/_base.txt + # rich pyrsistent==0.18.1 # via # -c requirements/_base.txt # jsonschema -pytest==8.3.4 + # referencing +pytest==8.3.5 # via # -r requirements/_test.in # pytest-aiohttp @@ -149,7 +227,7 @@ pytest-benchmark==5.1.0 # via -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in -pytest-docker==3.1.1 +pytest-docker==3.2.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -171,30 +249,52 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in + # uvicorn +python-multipart==0.0.20 + # via fastapi pyyaml==6.0.1 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt # openapi-spec-validator + # uvicorn redis==5.2.1 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in +referencing==0.8.11 + # via + # -c requirements/../../../../requirements/constraints.txt + # types-jsonschema requests==2.32.2 # via # -c requirements/_base.txt # docker +respx==0.22.0 + # via -r requirements/_test.in +rich==13.4.2 + # via + # -c requirements/_base.txt + # typer setuptools==69.1.1 # via # -c requirements/_base.txt # jsonschema # openapi-spec-validator +shellingham==1.5.4 + # via + # -c requirements/_base.txt + # typer six==1.16.0 # via # -c requirements/_base.txt # jsonschema # python-dateutil +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio sortedcontainers==2.4.0 # via hypothesis sqlalchemy==1.4.47 @@ -205,15 +305,23 @@ sqlalchemy==1.4.47 # alembic sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +starlette==0.41.3 + # via + # -c requirements/../../../../requirements/constraints.txt + # fastapi tenacity==8.5.0 # via # -c requirements/_base.txt # -r requirements/_test.in termcolor==2.5.0 # via pytest-sugar +typer==0.12.3 + # via + # -c requirements/_base.txt + # fastapi-cli types-aiofiles==24.1.0.20241221 # via -r requirements/_test.in -types-jsonschema==4.17.0.10 +types-jsonschema==4.23.0.20241208 # via -r requirements/_test.in types-openpyxl==3.1.5.20241225 # via -r requirements/_test.in @@ -225,17 +333,33 @@ typing-extensions==4.12.2 # via # -c requirements/_base.txt # asyncpg-stubs + # fastapi + # fastapi-pagination # mypy + # pydantic + # pydantic-core # sqlalchemy2-stubs + # typer urllib3==2.2.3 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt # docker # requests -websockets==14.2 - # via -r requirements/_test.in +uvicorn==0.34.0 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn +watchfiles==1.0.4 + # via uvicorn +websockets==15.0 + # via + # -r requirements/_test.in + # uvicorn yarl==1.9.4 # via # -c requirements/_base.txt # aiohttp + # referencing diff --git a/services/web/server/requirements/_tools.txt b/services/web/server/requirements/_tools.txt index 44d0a30fe00..731ed3fc72f 100644 --- a/services/web/server/requirements/_tools.txt +++ b/services/web/server/requirements/_tools.txt @@ -1,6 +1,6 @@ astroid==3.3.8 # via pylint -black==24.10.0 +black==25.1.0 # via -r requirements/../../../../requirements/devenv.txt build==1.2.2.post1 # via pip-tools @@ -20,17 +20,17 @@ distlib==0.3.9 # via virtualenv filelock==3.17.0 # via virtualenv -identify==2.6.6 +identify==2.6.8 # via pre-commit inotify==0.2.10 # via -r requirements/_tools.in -isort==5.13.2 +isort==6.0.1 # via # -r requirements/../../../../requirements/devenv.txt # pylint mccabe==0.7.0 # via pylint -mypy==1.14.1 +mypy==1.15.0 # via # -c requirements/_test.txt # -r requirements/../../../../requirements/devenv.txt @@ -51,7 +51,7 @@ packaging==24.1 # build pathspec==0.12.1 # via black -pip==25.0 +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../../requirements/devenv.txt @@ -62,7 +62,7 @@ platformdirs==4.3.6 # virtualenv pre-commit==4.1.0 # via -r requirements/../../../../requirements/devenv.txt -pylint==3.3.3 +pylint==3.3.4 # via -r requirements/../../../../requirements/devenv.txt pyproject-hooks==1.2.0 # via @@ -74,7 +74,7 @@ pyyaml==6.0.1 # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit -ruff==0.9.3 +ruff==0.9.9 # via -r requirements/../../../../requirements/devenv.txt setuptools==69.1.1 # via @@ -90,7 +90,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.29.1 +virtualenv==20.29.2 # via pre-commit wheel==0.45.1 # via pip-tools diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 65736bec36e..ccbfa6b24c9 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.51.0 +current_version = 0.61.4 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/activity/settings.py b/services/web/server/src/simcore_service_webserver/activity/settings.py index bfa727ccb35..f84eede661a 100644 --- a/services/web/server/src/simcore_service_webserver/activity/settings.py +++ b/services/web/server/src/simcore_service_webserver/activity/settings.py @@ -1,7 +1,7 @@ from aiohttp import web from settings_library.prometheus import PrometheusSettings -from .._constants import APP_SETTINGS_KEY +from ..constants import APP_SETTINGS_KEY def get_plugin_settings(app: web.Application) -> PrometheusSettings: diff --git a/services/web/server/src/simcore_service_webserver/announcements/_handlers.py b/services/web/server/src/simcore_service_webserver/announcements/_handlers.py index 596d31a43c2..ca925a39e14 100644 --- a/services/web/server/src/simcore_service_webserver/announcements/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/announcements/_handlers.py @@ -3,7 +3,7 @@ from aiohttp import web from .._meta import api_version_prefix -from ..products.api import get_product_name +from ..products import products_web from ..utils_aiohttp import envelope_json_response from . import _api from ._models import Announcement @@ -14,7 +14,7 @@ @routes.get(f"/{api_version_prefix}/announcements", name="list_announcements") async def list_announcements(request: web.Request) -> web.Response: """Returns non-expired announcements for current product""" - product_name = get_product_name(request) + product_name = products_web.get_product_name(request) announcements: list[Announcement] = await _api.list_announcements( request.app, product_name=product_name ) diff --git a/services/web/server/src/simcore_service_webserver/announcements/plugin.py b/services/web/server/src/simcore_service_webserver/announcements/plugin.py index fd3b3f79b43..88a39940cbf 100644 --- a/services/web/server/src/simcore_service_webserver/announcements/plugin.py +++ b/services/web/server/src/simcore_service_webserver/announcements/plugin.py @@ -7,7 +7,7 @@ from aiohttp import web from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from .._constants import APP_SETTINGS_KEY +from ..constants import APP_SETTINGS_KEY from ..products.plugin import setup_products from ..redis import setup_redis from . import _handlers diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index ea389a6869f..6facdd9ddf1 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.51.0 + version: 0.61.4 servers: - url: '' description: webserver @@ -253,9 +253,8 @@ paths: post: tags: - auth - summary: Reset Password - description: a non logged-in user requests a password reset - operationId: auth_reset_password + summary: Initiate Reset Password + operationId: initiate_reset_password requestBody: content: application/json: @@ -279,9 +278,8 @@ paths: post: tags: - auth - summary: Reset Password Allowed - description: changes password using a token code without being logged in - operationId: auth_reset_password_allowed + summary: Complete Reset Password + operationId: complete_reset_password parameters: - name: code in: path @@ -303,7 +301,7 @@ paths: schema: $ref: '#/components/schemas/Envelope_Log_' '401': - description: unauthorized reset due to invalid token code + description: Invalid token code content: application/json: schema: @@ -1107,13 +1105,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_GetCreditPrice_' + $ref: '#/components/schemas/Envelope_CreditPriceGet_' /v0/products/{product_name}: get: tags: - products - po summary: Get Product + description: 'NOTE: `/products/current` is used to define current project w/o + naming it' operationId: get_product parameters: - name: product_name @@ -1133,43 +1133,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Envelope_GetProduct_' - /v0/products/{product_name}/templates/{template_id}: - put: + $ref: '#/components/schemas/Envelope_ProductGet_' + /v0/products/current/ui: + get: tags: - products - - po - summary: Update Product Template - operationId: update_product_template - parameters: - - name: product_name - in: path - required: true - schema: - anyOf: - - type: string - minLength: 1 - maxLength: 100 - - const: current - type: string - title: Product Name - - name: template_id - in: path - required: true - schema: - type: string - minLength: 1 - maxLength: 100 - title: Template Id - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/UpdateProductTemplate' + summary: Get Current Product Ui + operationId: get_current_product_ui responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_ProductUIGet_' /v0/invitation:generate: post: tags: @@ -1181,7 +1158,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GenerateInvitation' + $ref: '#/components/schemas/InvitationGenerate' required: true responses: '200': @@ -1203,22 +1180,6 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_MyProfileGet_' - put: - tags: - - users - summary: Replace My Profile - description: Use PATCH instead - operationId: replace_my_profile - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/MyProfilePatch' - required: true - responses: - '204': - description: Successful Response - deprecated: true patch: tags: - users @@ -2048,7 +2009,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Page_CatalogServiceGet_' + $ref: '#/components/schemas/Page_CatalogLatestServiceGet_' /v0/catalog/services/{service_key}/{service_version}: get: tags: @@ -2367,7 +2328,8 @@ paths: tags: - catalog - pricing-plans - summary: Retrieve default pricing plan for provided service + summary: Get Service Pricing Plan + description: Retrieve default pricing plan for provided service operationId: get_service_pricing_plan parameters: - name: service_key @@ -2947,7 +2909,8 @@ paths: tags: - folders - workspaces - summary: Move folder to the workspace + summary: Move Folder To Workspace + description: Move folder to the workspace operationId: move_folder_to_workspace parameters: - name: folder_id @@ -3000,7 +2963,8 @@ paths: tags: - long-running-tasks summary: List Tasks - operationId: list_tasks + description: Lists all long running tasks + operationId: get_async_jobs responses: '200': description: Successful Response @@ -3008,12 +2972,37 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_list_TaskGet__' + '404': + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '403': + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '410': + description: Gone + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' /v0/tasks/{task_id}: get: tags: - long-running-tasks summary: Get Task Status - operationId: get_task_status + description: Retrieves the status of a task + operationId: get_async_job_status parameters: - name: task_id in: path @@ -3028,11 +3017,36 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_TaskStatus_' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '410': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Gone + '500': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Internal Server Error delete: tags: - long-running-tasks summary: Cancel And Delete Task - operationId: cancel_and_delete_task + description: Cancels and deletes a task + operationId: abort_async_job parameters: - name: task_id in: path @@ -3043,12 +3057,37 @@ paths: responses: '204': description: Successful Response + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '410': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Gone + '500': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Internal Server Error /v0/tasks/{task_id}/result: get: tags: - long-running-tasks summary: Get Task Result - operationId: get_task_result + description: Retrieves the result of a task + operationId: get_async_job_result parameters: - name: task_id in: path @@ -3062,6 +3101,30 @@ paths: content: application/json: schema: {} + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '410': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Gone + '500': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Internal Server Error /v0/catalog/licensed-items: get: tags: @@ -3124,52 +3187,6 @@ paths: schema: $ref: '#/components/schemas/EnvelopedError' description: Bad Request - /v0/catalog/licensed-items/{licensed_item_id}: - get: - tags: - - licenses - - catalog - summary: Get Licensed Item - operationId: get_licensed_item - parameters: - - name: licensed_item_id - in: path - required: true - schema: - type: string - format: uuid - title: Licensed Item Id - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_LicensedItemRestGet_' - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Not Found - '403': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Forbidden - '402': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Payment Required - '400': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Bad Request /v0/catalog/licensed-items/{licensed_item_id}:purchase: post: tags: @@ -3192,8 +3209,12 @@ paths: schema: $ref: '#/components/schemas/LicensedItemsBodyParams' responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/LicensedItemPurchaseGet' '404': content: application/json: @@ -3448,98 +3469,6 @@ paths: schema: $ref: '#/components/schemas/EnvelopedError' description: Bad Request - /v0/projects/{project_uuid}/checkpoint/{ref_id}/iterations: - get: - tags: - - projects - - metamodeling - summary: List Project Iterations - operationId: list_project_iterations - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: ref_id - in: path - required: true - schema: - type: integer - title: Ref Id - - name: limit - in: query - required: false - schema: - type: integer - minimum: 1 - exclusiveMaximum: true - default: 20 - title: Limit - maximum: 50 - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Page_ProjectIterationItem_' - /v0/projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: - get: - tags: - - projects - - metamodeling - summary: List Project Iterations Results - operationId: list_project_iterations_results - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: ref_id - in: path - required: true - schema: - type: integer - title: Ref Id - - name: limit - in: query - required: false - schema: - type: integer - minimum: 1 - exclusiveMaximum: true - default: 20 - title: Limit - maximum: 50 - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Page_ProjectIterationResultItem_' /v0/services: get: tags: @@ -3694,7 +3623,10 @@ paths: post: tags: - projects - summary: Creates a new project or copies an existing one + summary: Create Project + description: 'Creates a new project or copies an existing one. NOTE: implemented + as a long running task, i.e. requires polling `status_href` (HTTP_200_OK) + to get status and `result_href` (HTTP_201_CREATED) to get created project' operationId: create_project parameters: - name: x_simcore_user_agent @@ -3764,17 +3696,59 @@ paths: - $ref: '#/components/schemas/ProjectCopyOverride' title: ' Body' responses: - '201': + '202': description: Successful Response content: application/json: schema: $ref: '#/components/schemas/Envelope_TaskGet_' - get: - tags: - - projects - summary: List Projects - operationId: list_projects + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + get: + tags: + - projects + summary: List Projects + operationId: list_projects parameters: - name: type in: query @@ -3857,6 +3831,48 @@ paths: application/json: schema: $ref: '#/components/schemas/Page_ProjectListItem_' + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/projects/active: get: tags: @@ -3877,6 +3893,48 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_ProjectGet_' + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/projects/{project_id}: get: tags: @@ -3898,6 +3956,48 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_ProjectGet_' + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required patch: tags: - projects @@ -3920,6 +4020,48 @@ paths: responses: '204': description: Successful Response + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required delete: tags: - projects @@ -3936,6 +4078,48 @@ paths: responses: '204': description: Successful Response + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/projects/{project_id}:clone: post: tags: @@ -3957,11 +4141,53 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_TaskGet_' - /v0/projects:search: - get: - tags: - - projects - summary: List Projects Full Search + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/projects:search: + get: + tags: + - projects + summary: List Projects Full Search operationId: list_projects_full_search parameters: - name: filters @@ -4020,6 +4246,48 @@ paths: application/json: schema: $ref: '#/components/schemas/Page_ProjectListItem_' + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/projects/{project_id}/inactivity: get: tags: @@ -4041,12 +4309,55 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_GetProjectInactivityResponse_' + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/projects/{project_uuid}/comments: post: tags: - projects - comments - summary: Create a new comment for a specific project. The request body should + summary: Create Project Comment + description: Create a new comment for a specific project. The request body should contain the comment contents and user information. operationId: create_project_comment parameters: @@ -4074,7 +4385,8 @@ paths: tags: - projects - comments - summary: Retrieve all comments for a specific project. + summary: List Project Comments + description: Retrieve all comments for a specific project. operationId: list_project_comments parameters: - name: project_uuid @@ -4111,7 +4423,8 @@ paths: tags: - projects - comments - summary: Update the contents of a specific comment for a project. The request + summary: Update Project Comment + description: Update the contents of a specific comment for a project. The request body should contain the updated comment contents. operationId: update_project_comment parameters: @@ -4147,7 +4460,8 @@ paths: tags: - projects - comments - summary: Delete a specific comment associated with a project. + summary: Delete Project Comment + description: Delete a specific comment associated with a project. operationId: delete_project_comment parameters: - name: project_uuid @@ -4172,7 +4486,8 @@ paths: tags: - projects - comments - summary: Retrieve a specific comment by its ID within a project. + summary: Get Project Comment + description: Retrieve a specific comment by its ID within a project. operationId: get_project_comment parameters: - name: project_uuid @@ -4202,7 +4517,8 @@ paths: tags: - projects - folders - summary: Move project to the folder + summary: Replace Project Folder + description: Move project to the folder operationId: replace_project_folder parameters: - name: project_id @@ -4693,12 +5009,35 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_dict_Annotated_str__StringConstraints___ImageResources__' + /v0/projects/{project_id}/nodes/-/services: + get: + tags: + - projects + - nodes + summary: Get Project Services + operationId: get_project_services + parameters: + - name: project_id + in: path + required: true + schema: + type: string + format: uuid + title: Project Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_ProjectNodeServicesGet_' /v0/projects/{project_id}/nodes/-/services:access: get: tags: - projects - nodes - summary: Check whether provided group has access to the project services + summary: Get Project Services Access For Gid + description: Check whether provided group has access to the project services operationId: get_project_services_access_for_gid parameters: - name: project_id @@ -4728,7 +5067,8 @@ paths: tags: - projects - nodes - summary: Lists all previews in the node's project + summary: List Project Nodes Previews + description: Lists all previews in the node's project operationId: list_project_nodes_previews parameters: - name: project_id @@ -4750,7 +5090,8 @@ paths: tags: - projects - nodes - summary: Gets a give node's preview + summary: Get Project Node Preview + description: Gets a give node's preview operationId: get_project_node_preview parameters: - name: project_id @@ -4780,7 +5121,8 @@ paths: get: tags: - projects - summary: Get currently connected pricing unit to the project node. + summary: Get Project Node Pricing Unit + description: Get currently connected pricing unit to the project node. operationId: get_project_node_pricing_unit parameters: - name: project_id @@ -4808,8 +5150,9 @@ paths: put: tags: - projects - summary: Connect pricing unit to the project node (Project node can have only - one pricing unit) + summary: Connect Pricing Unit To Project Node + description: Connect pricing unit to the project node (Project node can have + only one pricing unit) operationId: connect_pricing_unit_to_project_node parameters: - name: project_id @@ -5107,7 +5450,8 @@ paths: get: tags: - projects - summary: Get current connected wallet to the project. + summary: Get Project Wallet + description: Get current connected wallet to the project. operationId: get_project_wallet parameters: - name: project_id @@ -5128,7 +5472,8 @@ paths: put: tags: - projects - summary: Connect wallet to the project (Project can have only one wallet) + summary: Connect Wallet To Project + description: Connect wallet to the project (Project can have only one wallet) operationId: connect_wallet_to_project parameters: - name: project_id @@ -5193,7 +5538,8 @@ paths: tags: - projects - workspaces - summary: Move project to the workspace + summary: Move Project To Workspace + description: Move project to the workspace operationId: move_project_to_workspace parameters: - name: project_id @@ -5236,8 +5582,9 @@ paths: get: tags: - usage - summary: Retrieve finished and currently running user services (user and product - are taken from context, optionally wallet_id parameter might be provided). + summary: List Resource Usage Services + description: Retrieve finished and currently running user services (user and + product are taken from context, optionally wallet_id parameter might be provided). operationId: list_resource_usage_services parameters: - name: order_by @@ -5294,7 +5641,8 @@ paths: get: tags: - usage - summary: Used credits based on aggregate by type, currently supported `services`. + summary: List Osparc Credits Aggregated Usages + description: Used credits based on aggregate by type, currently supported `services`. (user and product are taken from context, optionally wallet_id parameter might be provided). operationId: list_osparc_credits_aggregated_usages @@ -5340,7 +5688,8 @@ paths: get: tags: - usage - summary: Redirects to download CSV link. CSV obtains finished and currently + summary: Export Resource Usage Services + description: Redirects to download CSV link. CSV obtains finished and currently running user services (user and product are taken from context, optionally wallet_id parameter might be provided). operationId: export_resource_usage_services @@ -5730,9 +6079,9 @@ paths: get: tags: - storage - summary: Get available storage locations - description: Returns the list of available storage locations - operationId: get_storage_locations + summary: List Storage Locations + description: Get available storage locations + operationId: list_storage_locations responses: '200': description: Successful Response @@ -5740,17 +6089,16 @@ paths: application/json: schema: items: - $ref: '#/components/schemas/DatasetMetaData' + $ref: '#/components/schemas/FileLocation' type: array - title: Response Get Storage Locations - /v0/storage/locations/{location_id}:sync: - post: + title: Response List Storage Locations + /v0/storage/locations/{location_id}/paths: + get: tags: - storage - summary: Manually triggers the synchronisation of the file meta data table in - the database - description: Returns an object containing added, changed and removed paths - operationId: synchronise_meta_data_table + summary: List Storage Paths + description: Lists the files/directories in WorkingDirectory + operationId: list_storage_paths parameters: - name: location_id in: path @@ -5758,34 +6106,75 @@ paths: schema: type: integer title: Location Id - - name: dry_run + - name: size in: query required: false schema: - type: boolean - default: false - title: Dry Run - - name: fire_and_forget + type: integer + minimum: 1 + exclusiveMaximum: true + default: 50 + title: Size + maximum: 1000 + - name: cursor in: query required: false schema: - type: boolean - default: false - title: Fire And Forget + anyOf: + - type: string + - type: 'null' + title: Cursor + - name: fileFilter + in: query + required: false + schema: + anyOf: + - type: string + format: path + - type: 'null' + title: Filefilter responses: '200': description: Successful Response content: application/json: schema: - $ref: '#/components/schemas/Envelope_TableSynchronisation_' + $ref: '#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_' + /v0/storage/locations/{location_id}/paths/{path}:size: + post: + tags: + - storage + summary: Compute Path Size + description: Compute the size of a path + operationId: compute_path_size + parameters: + - name: location_id + in: path + required: true + schema: + type: integer + title: Location Id + - name: path + in: path + required: true + schema: + type: string + format: path + title: Path + responses: + '202': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_TaskGet_' /v0/storage/locations/{location_id}/datasets: get: tags: - storage - summary: Get datasets metadata - description: returns all the top level datasets a user has access to - operationId: get_datasets_metadata + summary: List Datasets Metadata + description: Get datasets metadata + operationId: list_datasets_metadata parameters: - name: location_id in: path @@ -5804,9 +6193,8 @@ paths: get: tags: - storage - summary: Get datasets metadata - description: returns all the file meta data a user has access to (uuid_filter - may be used) + summary: Get Files Metadata + description: Get datasets metadata operationId: get_files_metadata parameters: - name: location_id @@ -5844,9 +6232,9 @@ paths: get: tags: - storage - summary: Get Files Metadata - description: returns all the file meta data inside dataset with dataset_id - operationId: get_files_metadata_dataset + summary: List Dataset Files Metadata + description: Get Files Metadata + operationId: list_dataset_files_metadata parameters: - name: location_id in: path @@ -5883,8 +6271,7 @@ paths: tags: - storage summary: Get File Metadata - description: returns the file meta data of file_id if user_id has the rights - to + description: Get File Metadata operationId: get_file_metadata parameters: - name: location_id @@ -5913,8 +6300,8 @@ paths: get: tags: - storage - summary: Returns download link for requested file - description: creates a download file link if user has the rights to + summary: Download File + description: Returns download link for requested file operationId: download_file parameters: - name: location_id @@ -5945,9 +6332,8 @@ paths: put: tags: - storage - summary: Returns upload link - description: creates one or more upload file links if user has the rights to, - expects the client to complete/abort upload + summary: Upload File + description: Returns upload link operationId: upload_file parameters: - name: location_id @@ -5999,8 +6385,8 @@ paths: delete: tags: - storage - summary: Deletes File - description: deletes file if user has the rights to + summary: Delete File + description: Deletes File operationId: delete_file parameters: - name: location_id @@ -6080,8 +6466,8 @@ paths: post: tags: - storage - summary: Check for upload completion - description: Returns state of upload completion + summary: Is Completed Upload File + description: Check for upload completion operationId: is_completed_upload_file parameters: - name: location_id @@ -6109,80 +6495,123 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_FileUploadCompleteFutureResponse_' - /v0/trash: - delete: - tags: - - trash - summary: Empty Trash - operationId: empty_trash - responses: - '204': - description: Successful Response - /v0/projects/{project_id}:trash: + /v0/storage/locations/{location_id}/export-data: post: tags: - - trash - - projects - summary: Trash Project - operationId: trash_project + - storage + summary: Export Data + description: Export data + operationId: export_data parameters: - - name: project_id + - name: location_id in: path required: true schema: - type: string - format: uuid - title: Project Id - - name: force - in: query - required: false - schema: - type: boolean - default: false - title: Force + type: integer + title: Location Id + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DataExportPost' responses: - '204': + '200': description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_TaskGet_' '404': - description: Not such a project - '409': - description: Project is in use and cannot be trashed - '503': - description: Trash service error - /v0/projects/{project_id}:untrash: + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '410': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Gone + '500': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Internal Server Error + /v0/trash:empty: post: tags: - trash - - projects - summary: Untrash Project - operationId: untrash_project - parameters: - - name: project_id - in: path - required: true - schema: - type: string - format: uuid - title: Project Id + summary: Empty Trash + operationId: empty_trash responses: '204': description: Successful Response - /v0/folders/{folder_id}:trash: + '403': + description: Forbidden + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '404': + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '409': + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '503': + description: Service Unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '422': + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '402': + description: Payment Required + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + /v0/projects/{project_id}:trash: post: tags: - trash - - folders - summary: Trash Folder - operationId: trash_folder + - projects + summary: Trash Project + operationId: trash_project parameters: - - name: folder_id + - name: project_id in: path required: true schema: - type: integer - exclusiveMinimum: true - title: Folder Id - minimum: 0 + type: string + format: uuid + title: Project Id - name: force in: query required: false @@ -6193,47 +6622,123 @@ paths: responses: '204': description: Successful Response + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden '404': - description: Not such a folder + description: Not such a project + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' '409': - description: One or more projects in the folder are in use and cannot be - trashed + description: Project is in use and cannot be trashed + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' '503': description: Trash service error - /v0/folders/{folder_id}:untrash: + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/projects/{project_id}:untrash: post: tags: - trash - - folders - summary: Untrash Folder - operationId: untrash_folder + - projects + summary: Untrash Project + operationId: untrash_project parameters: - - name: folder_id + - name: project_id in: path required: true schema: - type: integer - exclusiveMinimum: true - title: Folder Id - minimum: 0 + type: string + format: uuid + title: Project Id responses: '204': description: Successful Response - /v0/workspaces/{workspace_id}:trash: + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/folders/{folder_id}:trash: post: tags: - trash - - workspaces - summary: Trash Workspace - operationId: trash_workspace + - folders + summary: Trash Folder + operationId: trash_folder parameters: - - name: workspace_id + - name: folder_id in: path required: true schema: type: integer exclusiveMinimum: true - title: Workspace Id + title: Folder Id minimum: 0 - name: force in: query @@ -6245,253 +6750,216 @@ paths: responses: '204': description: Successful Response + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden '404': - description: Not such a workspace + description: Not such a folder '409': - description: One or more projects in the workspace are in use and cannot - be trashed + description: One or more projects in the folder are in use and cannot be + trashed '503': description: Trash service error - /v0/workspaces/{workspace_id}:untrash: + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/folders/{folder_id}:untrash: post: tags: - trash - - workspaces - summary: Untrash Workspace - operationId: untrash_workspace + - folders + summary: Untrash Folder + operationId: untrash_folder parameters: - - name: workspace_id + - name: folder_id in: path required: true schema: type: integer exclusiveMinimum: true - title: Workspace Id + title: Folder Id minimum: 0 responses: '204': description: Successful Response - /v0/repos/projects: - get: - tags: - - repository - summary: List Repos - operationId: list_repos - parameters: - - name: limit - in: query - required: false - schema: - type: integer - minimum: 1 - exclusiveMaximum: true - default: 20 - title: Limit - maximum: 50 - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response + '403': content: application/json: schema: - $ref: '#/components/schemas/Page_RepoApiModel_' - /v0/repos/projects/{project_uuid}/checkpoints: - get: - tags: - - repository - summary: List Checkpoints - operationId: list_checkpoints - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: limit - in: query - required: false - schema: - type: integer - minimum: 1 - exclusiveMaximum: true - default: 20 - title: Limit - maximum: 50 - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': content: application/json: schema: - $ref: '#/components/schemas/Page_CheckpointApiModel_' - post: - tags: - - repository - summary: Create Checkpoint - operationId: create_checkpoint - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CheckpointNew' - responses: - '200': - description: Successful Response + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': content: application/json: schema: - $ref: '#/components/schemas/Envelope_CheckpointApiModel_' - /v0/repos/projects/{project_uuid}/checkpoints/{ref_id}: - get: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/workspaces/{workspace_id}:trash: + post: tags: - - repository - summary: Get Checkpoint - operationId: get_checkpoint + - trash + - workspaces + summary: Trash Workspace + operationId: trash_workspace parameters: - - name: ref_id + - name: workspace_id in: path required: true schema: - anyOf: - - type: integer - - type: string - - const: HEAD - type: string - title: Ref Id - - name: project_uuid - in: path - required: true + type: integer + exclusiveMinimum: true + title: Workspace Id + minimum: 0 + - name: force + in: query + required: false schema: - type: string - format: uuid - title: Project Uuid + type: boolean + default: false + title: Force responses: - '200': + '204': description: Successful Response + '403': content: application/json: schema: - $ref: '#/components/schemas/Envelope_CheckpointApiModel_' - patch: - tags: - - repository - summary: Update Checkpoint - description: Update Checkpoint Annotations - operationId: update_checkpoint - parameters: - - name: ref_id - in: path - required: true - schema: - anyOf: - - type: integer - - type: string - title: Ref Id - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CheckpointAnnotations' - responses: - '200': - description: Successful Response + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + description: Not such a workspace + '409': + description: One or more projects in the workspace are in use and cannot + be trashed + '503': + description: Trash service error + '422': content: application/json: schema: - $ref: '#/components/schemas/Envelope_CheckpointApiModel_' - /v0/repos/projects/{project_uuid}/checkpoints/{ref_id}/workbench/view: - get: - tags: - - repository - summary: View Project Workbench - operationId: view_project_workbench - parameters: - - name: ref_id - in: path - required: true - schema: - anyOf: - - type: integer - - type: string - title: Ref Id - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - responses: - '200': - description: Successful Response + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': content: application/json: schema: - $ref: '#/components/schemas/Envelope_WorkbenchViewApiModel_' - /v0/repos/projects/{project_uuid}/checkpoints/{ref_id}:checkout: + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required + /v0/workspaces/{workspace_id}:untrash: post: tags: - - repository - summary: Checkout - operationId: checkout + - trash + - workspaces + summary: Untrash Workspace + operationId: untrash_workspace parameters: - - name: ref_id - in: path - required: true - schema: - anyOf: - - type: integer - - type: string - title: Ref Id - - name: project_uuid + - name: workspace_id in: path required: true schema: - type: string - format: uuid - title: Project Uuid + type: integer + exclusiveMinimum: true + title: Workspace Id + minimum: 0 responses: - '200': + '204': description: Successful Response + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + '409': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Conflict + '503': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Service Unavailable + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Unprocessable Entity + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Bad Request + '402': content: application/json: schema: - $ref: '#/components/schemas/Envelope_CheckpointApiModel_' + $ref: '#/components/schemas/EnvelopedError' + description: Payment Required /v0/workspaces: post: tags: @@ -7021,8 +7489,8 @@ paths: get: tags: - maintenance - summary: Front end runtime configuration - description: Returns app and products configs + summary: Get Config + description: Front end runtime configuration operationId: get_config responses: '200': @@ -7048,7 +7516,8 @@ paths: get: tags: - maintenance - summary: checks status of self and connected services + summary: Get App Status + description: checks status of self and connected services operationId: get_app_status responses: '200': @@ -7175,7 +7644,7 @@ components: - stats - limits title: Activity - Annotation: + AnnotationUI: properties: type: type: string @@ -7198,7 +7667,7 @@ components: - type - color - attributes - title: Annotation + title: AnnotationUI Announcement: properties: id: @@ -7432,6 +7901,225 @@ components: - default - items title: BootOption + CatalogLatestServiceGet: + properties: + key: + type: string + pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ + title: Key + version: + type: string + pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ + title: Version + name: + type: string + title: Name + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + description: + type: string + title: Description + descriptionUi: + type: boolean + title: Descriptionui + default: false + versionDisplay: + anyOf: + - type: string + - type: 'null' + title: Versiondisplay + type: + $ref: '#/components/schemas/ServiceType' + contact: + anyOf: + - type: string + format: email + - type: 'null' + title: Contact + authors: + items: + $ref: '#/components/schemas/Author' + type: array + minItems: 1 + title: Authors + owner: + anyOf: + - type: string + format: email + - type: 'null' + title: Owner + description: None when the owner email cannot be found in the database + inputs: + type: object + title: Inputs + outputs: + type: object + title: Outputs + bootOptions: + anyOf: + - type: object + - type: 'null' + title: Bootoptions + minVisibleInputs: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + title: Minvisibleinputs + accessRights: + anyOf: + - additionalProperties: + $ref: '#/components/schemas/ServiceGroupAccessRightsV2' + type: object + - type: 'null' + title: Accessrights + classifiers: + anyOf: + - items: + type: string + type: array + - type: 'null' + title: Classifiers + quality: + type: object + title: Quality + release: + $ref: '#/components/schemas/ServiceRelease' + description: release information of current (latest) service + type: object + required: + - key + - version + - name + - description + - type + - contact + - authors + - owner + - inputs + - outputs + - accessRights + - release + title: CatalogLatestServiceGet + example: + accessRights: + '1': + execute: true + write: false + authors: + - affiliation: ACME + email: author@acme.com + name: Author Bar + classifiers: [] + contact: contact@acme.com + description: A service which awaits for time to pass, two times. + description_ui: true + icon: https://cdn-icons-png.flaticon.com/512/25/25231.png + inputs: + input_1: + description: Pick a file containing only one integer + displayOrder: 1 + fileToKeyMap: + single_number.txt: input_1 + keyId: input_1 + label: File with int number + type: data:text/plain + input_2: + contentSchema: + minimum: 0 + title: Sleep interval + type: integer + x_unit: second + defaultValue: 2 + description: Choose an amount of time to sleep in range [0:] + displayOrder: 2 + keyId: input_2 + label: Sleep interval + type: ref_contentSchema + unitLong: second + unitShort: s + input_3: + defaultValue: false + description: If set to true will cause service to fail after it sleeps + displayOrder: 3 + keyId: input_3 + label: Fail after sleep + type: boolean + input_4: + contentSchema: + title: Distance to bed + type: integer + x_unit: meter + defaultValue: 0 + description: It will first walk the distance to bed + displayOrder: 4 + keyId: input_4 + label: Distance to bed + type: ref_contentSchema + unitLong: meter + unitShort: m + input_5: + contentSchema: + minimum: 0 + title: Dream of the night + type: integer + x_unit: byte + defaultValue: 0 + description: Defines the size of the dream that will be generated [0:] + displayOrder: 5 + keyId: input_5 + label: Dream (or nightmare) of the night + type: ref_contentSchema + unitLong: byte + unitShort: B + key: simcore/services/comp/itis/sleeper + name: sleeper + outputs: + output_1: + description: Integer is generated in range [1-9] + displayOrder: 1 + fileToKeyMap: + single_number.txt: output_1 + keyId: output_1 + label: File containing one random integer + type: data:text/plain + output_2: + contentSchema: + title: Random sleep interval + type: integer + x_unit: second + description: Interval is generated in range [1-9] + displayOrder: 2 + keyId: output_2 + label: Random sleep interval + type: ref_contentSchema + unitLong: second + unitShort: s + output_3: + description: Contains some random data representing a dream + displayOrder: 3 + fileToKeyMap: + dream.txt: output_3 + keyId: output_3 + label: Dream output + type: data:text/plain + owner: owner@acme.com + quality: {} + release: + released: '2025-07-20T15:00:00' + version: 2.2.1 + version_display: Summer Release + type: computational + version: 2.2.1 + version_display: 2 Xtreme CatalogServiceGet: properties: key: @@ -7450,6 +8138,11 @@ components: - type: string - type: 'null' title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon description: type: string title: Description @@ -7516,11 +8209,9 @@ components: type: array - type: 'null' title: Classifiers - default: [] quality: type: object title: Quality - default: {} history: items: $ref: '#/components/schemas/ServiceRelease' @@ -7557,7 +8248,7 @@ components: description: A service which awaits for time to pass, two times. description_ui: true history: - - released: '2024-07-20T15:00:00' + - released: '2024-07-21T15:00:00' version: 2.2.1 version_display: Summer Release - compatibility: @@ -7576,10 +8267,11 @@ components: released: '2024-01-20T18:49:17' version: 0.9.1 versionDisplay: Matterhorn - - retired: '2024-07-20T15:00:00' + - retired: '2024-07-20T16:00:00' version: 0.9.0 - version: 0.8.0 - version: 0.1.0 + icon: https://cdn-icons-png.flaticon.com/512/25/25231.png inputs: input0: contentSchema: @@ -7624,6 +8316,14 @@ components: format: uri - type: 'null' title: Thumbnail + icon: + anyOf: + - type: string + maxLength: 2083 + minLength: 1 + format: uri + - type: 'null' + title: Icon description: anyOf: - type: string @@ -7688,79 +8388,6 @@ components: - new - confirm title: ChangePasswordBody - CheckpointAnnotations: - properties: - tag: - anyOf: - - type: string - - type: 'null' - title: Tag - message: - anyOf: - - type: string - - type: 'null' - title: Message - type: object - title: CheckpointAnnotations - CheckpointApiModel: - properties: - id: - type: integer - exclusiveMinimum: true - title: Id - minimum: 0 - checksum: - type: string - pattern: ^[a-fA-F0-9]{40}$ - title: Checksum - created_at: - type: string - format: date-time - title: Created At - tags: - items: - type: string - type: array - title: Tags - message: - anyOf: - - type: string - - type: 'null' - title: Message - parents_ids: - anyOf: - - items: - type: integer - exclusiveMinimum: true - minimum: 0 - type: array - - type: 'null' - title: Parents Ids - url: - type: string - title: Url - type: object - required: - - id - - checksum - - created_at - - tags - - url - title: CheckpointApiModel - CheckpointNew: - properties: - tag: - type: string - title: Tag - message: - anyOf: - - type: string - - type: 'null' - title: Message - type: object - required: - - tag - title: CheckpointNew CodePageParams: properties: message: @@ -7951,7 +8578,10 @@ components: type: string title: Unitname unitExtraInfo: - $ref: '#/components/schemas/UnitExtraInfo-Input' + anyOf: + - $ref: '#/components/schemas/UnitExtraInfoTier-Input' + - $ref: '#/components/schemas/UnitExtraInfoLicense' + title: Unitextrainfo default: type: boolean title: Default @@ -8014,6 +8644,83 @@ components: required: - priceDollars title: CreateWalletPayment + CreditPriceGet: + properties: + productName: + type: string + title: Productname + usdPerCredit: + anyOf: + - type: number + minimum: 0.0 + - type: 'null' + title: Usdpercredit + description: Price of a credit in USD. If None, then this product's price + is UNDEFINED + minPaymentAmountUsd: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + title: Minpaymentamountusd + description: Minimum amount (included) in USD that can be paid for this + productCan be None if this product's price is UNDEFINED + type: object + required: + - productName + - usdPerCredit + - minPaymentAmountUsd + title: CreditPriceGet + CreditTransactionStatus: + type: string + enum: + - PENDING + - BILLED + - IN_DEBT + - NOT_BILLED + - REQUIRES_MANUAL_REVIEW + title: CreditTransactionStatus + CursorPage___T_Customized_PathMetaDataGet_: + properties: + items: + items: + $ref: '#/components/schemas/PathMetaDataGet' + type: array + title: Items + total: + anyOf: + - type: integer + - type: 'null' + title: Total + description: Total items + current_page: + anyOf: + - type: string + - type: 'null' + title: Current Page + description: Cursor to refetch the current page + current_page_backwards: + anyOf: + - type: string + - type: 'null' + title: Current Page Backwards + description: Cursor to refetch the current page starting from the last item + previous_page: + anyOf: + - type: string + - type: 'null' + title: Previous Page + description: Cursor for the previous page + next_page: + anyOf: + - type: string + - type: 'null' + title: Next Page + description: Cursor for the next page + type: object + required: + - items + title: CursorPage[~_T]Customized[PathMetaDataGet] DatCoreFileLink: properties: store: @@ -8053,6 +8760,21 @@ components: - dataset title: DatCoreFileLink description: I/O port type to hold a link to a file in DATCORE storage + DataExportPost: + properties: + paths: + items: + anyOf: + - type: string + pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ + - type: string + pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ + type: array + title: Paths + type: object + required: + - paths + title: DataExportPost DatasetMetaData: properties: dataset_id: @@ -8193,11 +8915,11 @@ components: title: Error type: object title: Envelope[CatalogServiceGet] - Envelope_CheckpointApiModel_: + Envelope_ComputationGet_: properties: data: anyOf: - - $ref: '#/components/schemas/CheckpointApiModel' + - $ref: '#/components/schemas/ComputationGet' - type: 'null' error: anyOf: @@ -8205,12 +8927,12 @@ components: - type: 'null' title: Error type: object - title: Envelope[CheckpointApiModel] - Envelope_ComputationGet_: + title: Envelope[ComputationGet] + Envelope_CreditPriceGet_: properties: data: anyOf: - - $ref: '#/components/schemas/ComputationGet' + - $ref: '#/components/schemas/CreditPriceGet' - type: 'null' error: anyOf: @@ -8218,7 +8940,7 @@ components: - type: 'null' title: Error type: object - title: Envelope[ComputationGet] + title: Envelope[CreditPriceGet] Envelope_FileMetaDataGet_: properties: data: @@ -8297,32 +9019,6 @@ components: title: Error type: object title: Envelope[FolderGet] - Envelope_GetCreditPrice_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/GetCreditPrice' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[GetCreditPrice] - Envelope_GetProduct_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/GetProduct' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[GetProduct] Envelope_GetProjectInactivityResponse_: properties: data: @@ -8427,19 +9123,6 @@ components: title: Error type: object title: Envelope[LicensedItemPurchaseGet] - Envelope_LicensedItemRestGet_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/LicensedItemRestGet' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[LicensedItemRestGet] Envelope_Log_: properties: data: @@ -8635,6 +9318,32 @@ components: title: Error type: object title: Envelope[PricingUnitGet] + Envelope_ProductGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/ProductGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[ProductGet] + Envelope_ProductUIGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/ProductUIGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[ProductUIGet] Envelope_ProjectGet_: properties: data: @@ -8674,6 +9383,19 @@ components: title: Error type: object title: Envelope[ProjectMetadataGet] + Envelope_ProjectNodeServicesGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/ProjectNodeServicesGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[ProjectNodeServicesGet] Envelope_ProjectState_: properties: data: @@ -8739,24 +9461,11 @@ components: title: Error type: object title: Envelope[ServiceInputGet] - Envelope_ServicePricingPlanGet_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/ServicePricingPlanGet' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[ServicePricingPlanGet] - Envelope_StatusDiagnosticsGet_: + Envelope_ServicePricingPlanGet_: properties: data: anyOf: - - $ref: '#/components/schemas/StatusDiagnosticsGet' + - $ref: '#/components/schemas/ServicePricingPlanGet' - type: 'null' error: anyOf: @@ -8764,12 +9473,12 @@ components: - type: 'null' title: Error type: object - title: Envelope[StatusDiagnosticsGet] - Envelope_TableSynchronisation_: + title: Envelope[ServicePricingPlanGet] + Envelope_StatusDiagnosticsGet_: properties: data: anyOf: - - $ref: '#/components/schemas/TableSynchronisation' + - $ref: '#/components/schemas/StatusDiagnosticsGet' - type: 'null' error: anyOf: @@ -8777,7 +9486,7 @@ components: - type: 'null' title: Error type: object - title: Envelope[TableSynchronisation] + title: Envelope[StatusDiagnosticsGet] Envelope_TagGet_: properties: data: @@ -8941,19 +9650,6 @@ components: title: Error type: object title: Envelope[WalletPaymentInitiated] - Envelope_WorkbenchViewApiModel_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/WorkbenchViewApiModel' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[WorkbenchViewApiModel] Envelope_WorkspaceGet_: properties: data: @@ -9645,8 +10341,6 @@ components: status: type: integer title: Status - default: 400 - deprecated: true errors: items: $ref: '#/components/schemas/ErrorItemType' @@ -9664,6 +10358,7 @@ components: type: object required: - message + - status title: ErrorGet ErrorItemType: properties: @@ -9690,47 +10385,73 @@ components: - resource - field title: ErrorItemType - ExtractedResults: + ExecutableAccessRights: properties: - progress: - type: object - title: Progress - description: Progress in each computational node - labels: - type: object - title: Labels - description: Maps captured node with a label - values: - type: object - title: Values - description: Captured outputs per node + write: + type: boolean + title: Write + description: can change executable settings + execute: + type: boolean + title: Execute + description: can run executable + additionalProperties: false type: object required: - - progress - - labels - - values - title: ExtractedResults - example: - labels: - 0f1e38c9-dcb7-443c-a745-91b97ac28ccc: Integer iterator - 2d0ce8b9-c9c3-43ce-ad2f-ad493898de37: Probe Sensor - Integer - 445b44d1-59b3-425c-ac48-7c13e0f2ea5b: Probe Sensor - Integer_2 - d76fca06-f050-4790-88a8-0aac10c87b39: Boolean Parameter - progress: - 4c08265a-427b-4ac3-9eab-1d11c822ada4: 0 - e33c6880-1b1d-4419-82d7-270197738aa9: 100 - values: - 0f1e38c9-dcb7-443c-a745-91b97ac28ccc: - out_1: 1 - out_2: - - 3 - - 4 - 2d0ce8b9-c9c3-43ce-ad2f-ad493898de37: - in_1: 7 - 445b44d1-59b3-425c-ac48-7c13e0f2ea5b: - in_1: 1 - d76fca06-f050-4790-88a8-0aac10c87b39: - out_1: true + - write + - execute + title: ExecutableAccessRights + FeaturesDict: + properties: + age: + type: string + title: Age + date: + type: string + format: date + title: Date + ethnicity: + type: string + title: Ethnicity + functionality: + type: string + title: Functionality + height: + type: string + title: Height + name: + type: string + title: Name + sex: + type: string + title: Sex + species: + type: string + title: Species + version: + type: string + title: Version + weight: + type: string + title: Weight + type: object + required: + - date + title: FeaturesDict + FileLocation: + properties: + name: + type: string + title: Name + id: + type: integer + title: Id + additionalProperties: false + type: object + required: + - name + - id + title: FileLocation FileMetaData: properties: file_uuid: @@ -9963,6 +10684,7 @@ components: title: Urls links: $ref: '#/components/schemas/FileUploadLinks' + additionalProperties: false type: object required: - chunk_size @@ -10079,151 +10801,6 @@ components: required: - name title: FolderReplaceBodyParams - GenerateInvitation: - properties: - guest: - type: string - format: email - title: Guest - trialAccountDays: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Trialaccountdays - extraCreditsInUsd: - anyOf: - - type: integer - exclusiveMaximum: true - minimum: 0 - maximum: 500 - - type: 'null' - title: Extracreditsinusd - type: object - required: - - guest - title: GenerateInvitation - GetCreditPrice: - properties: - productName: - type: string - title: Productname - usdPerCredit: - anyOf: - - type: number - minimum: 0.0 - - type: 'null' - title: Usdpercredit - description: Price of a credit in USD. If None, then this product's price - is UNDEFINED - minPaymentAmountUsd: - anyOf: - - type: integer - minimum: 0 - - type: 'null' - title: Minpaymentamountusd - description: Minimum amount (included) in USD that can be paid for this - productCan be None if this product's price is UNDEFINED - type: object - required: - - productName - - usdPerCredit - - minPaymentAmountUsd - title: GetCreditPrice - GetProduct: - properties: - name: - type: string - title: Name - displayName: - type: string - title: Displayname - shortName: - anyOf: - - type: string - - type: 'null' - title: Shortname - description: Short display name for SMS - vendor: - anyOf: - - type: object - - type: 'null' - title: Vendor - description: vendor attributes - issues: - anyOf: - - items: - type: object - type: array - - type: 'null' - title: Issues - description: Reference to issues tracker - manuals: - anyOf: - - items: - type: object - type: array - - type: 'null' - title: Manuals - description: List of manuals - support: - anyOf: - - items: - type: object - type: array - - type: 'null' - title: Support - description: List of support resources - loginSettings: - type: object - title: Loginsettings - maxOpenStudiesPerUser: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Maxopenstudiesperuser - isPaymentEnabled: - type: boolean - title: Ispaymentenabled - creditsPerUsd: - anyOf: - - type: string - - type: 'null' - title: Creditsperusd - templates: - items: - $ref: '#/components/schemas/GetProductTemplate' - type: array - title: Templates - description: List of templates available to this product for communications - (e.g. emails, sms, etc) - type: object - required: - - name - - displayName - - loginSettings - - maxOpenStudiesPerUser - - isPaymentEnabled - - creditsPerUsd - title: GetProduct - GetProductTemplate: - properties: - id: - type: string - maxLength: 100 - minLength: 1 - title: Id - content: - type: string - title: Content - type: object - required: - - id - - content - title: GetProductTemplate GetProjectInactivityResponse: properties: is_inactive: @@ -10408,10 +10985,13 @@ components: title: Id description: the user's id userName: - type: string - maxLength: 100 - minLength: 1 + anyOf: + - type: string + maxLength: 100 + minLength: 1 + - type: 'null' title: Username + description: None if private gid: anyOf: - type: integer @@ -10426,19 +11006,19 @@ components: format: email - type: 'null' title: Login - description: the user's email, if privacy settings allows + description: the user's email or None if private first_name: anyOf: - type: string - type: 'null' title: First Name - description: If privacy settings allows + description: None if private last_name: anyOf: - type: string - type: 'null' title: Last Name - description: If privacy settings allows + description: None if private gravatar_id: anyOf: - type: string @@ -10453,8 +11033,6 @@ components: description: If group is standard, these are these are the access rights of the user to it.None if primary group. type: object - required: - - userName title: GroupUserGet example: accessRights: @@ -10566,6 +11144,31 @@ components: required: - invitation title: InvitationCheck + InvitationGenerate: + properties: + guest: + type: string + format: email + title: Guest + trialAccountDays: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Trialaccountdays + extraCreditsInUsd: + anyOf: + - type: integer + exclusiveMaximum: true + minimum: 0 + maximum: 500 + - type: 'null' + title: Extracreditsinusd + type: object + required: + - guest + title: InvitationGenerate InvitationGenerated: properties: productName: @@ -10632,6 +11235,13 @@ components: type: string format: uuid title: Licenseditemid + key: + type: string + title: Key + version: + type: string + pattern: ^\d+\.\d+\.\d+$ + title: Version walletId: type: integer exclusiveMinimum: true @@ -10678,6 +11288,8 @@ components: - licensedItemPurchaseId - productName - licensedItemId + - key + - version - walletId - pricingUnitCostId - pricingUnitCost @@ -10695,19 +11307,46 @@ components: type: string format: uuid title: Licenseditemid + key: + type: string + title: Key + version: + type: string + pattern: ^\d+\.\d+\.\d+$ + title: Version displayName: type: string title: Displayname licensedResourceType: $ref: '#/components/schemas/LicensedResourceType' - licensedResourceData: - type: object - title: Licensedresourcedata + licensedResources: + items: + $ref: '#/components/schemas/_ItisVipResourceRestData' + type: array + title: Licensedresources pricingPlanId: type: integer exclusiveMinimum: true title: Pricingplanid minimum: 0 + categoryId: + type: string + maxLength: 100 + minLength: 1 + title: Categoryid + categoryDisplay: + type: string + title: Categorydisplay + categoryIcon: + anyOf: + - type: string + - type: 'null' + title: Categoryicon + termsOfUseUrl: + anyOf: + - type: string + - type: 'null' + title: Termsofuseurl createdAt: type: string format: date-time @@ -10719,10 +11358,14 @@ components: type: object required: - licensedItemId + - key + - version - displayName - licensedResourceType - - licensedResourceData + - licensedResources - pricingPlanId + - categoryId + - categoryDisplay - createdAt - modifiedAt title: LicensedItemRestGet @@ -10891,7 +11534,7 @@ components: additionalProperties: false type: object title: LogoutBody - Marker: + MarkerUI: properties: color: type: string @@ -10901,7 +11544,7 @@ components: type: object required: - color - title: Marker + title: MarkerUI MyGroupsGet: properties: me: @@ -11061,7 +11704,7 @@ components: anyOf: - type: string maxLength: 100 - minLength: 1 + minLength: 4 - type: 'null' title: Username privacy: @@ -11075,6 +11718,9 @@ components: last_name: Crespo MyProfilePrivacyGet: properties: + hideUsername: + type: boolean + title: Hideusername hideFullname: type: boolean title: Hidefullname @@ -11083,11 +11729,17 @@ components: title: Hideemail type: object required: + - hideUsername - hideFullname - hideEmail title: MyProfilePrivacyGet MyProfilePrivacyPatch: properties: + hideUsername: + anyOf: + - type: boolean + - type: 'null' + title: Hideusername hideFullname: anyOf: - type: boolean @@ -11670,6 +12322,32 @@ components: - thumbnail_url - file_url title: NodeScreenshot + NodeServiceGet: + properties: + key: + type: string + pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ + title: Key + release: + $ref: '#/components/schemas/ServiceRelease' + owner: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Owner + description: Service owner primary group id or None if ownership still not + defined + myAccessRights: + $ref: '#/components/schemas/ExecutableAccessRights' + type: object + required: + - key + - release + - owner + - myAccessRights + title: NodeServiceGet NodeState: properties: modified: @@ -11816,25 +12494,7 @@ components: - total - count title: PageMetaInfoLimitOffset - Page_CatalogServiceGet_: - properties: - _meta: - $ref: '#/components/schemas/PageMetaInfoLimitOffset' - _links: - $ref: '#/components/schemas/PageLinks' - data: - items: - $ref: '#/components/schemas/CatalogServiceGet' - type: array - title: Data - additionalProperties: false - type: object - required: - - _meta - - _links - - data - title: Page[CatalogServiceGet] - Page_CheckpointApiModel_: + Page_CatalogLatestServiceGet_: properties: _meta: $ref: '#/components/schemas/PageMetaInfoLimitOffset' @@ -11842,7 +12502,7 @@ components: $ref: '#/components/schemas/PageLinks' data: items: - $ref: '#/components/schemas/CheckpointApiModel' + $ref: '#/components/schemas/CatalogLatestServiceGet' type: array title: Data additionalProperties: false @@ -11851,7 +12511,7 @@ components: - _meta - _links - data - title: Page[CheckpointApiModel] + title: Page[CatalogLatestServiceGet] Page_LicensedItemPurchaseGet_: properties: _meta: @@ -11960,42 +12620,6 @@ components: - _links - data title: Page[PricingPlanGet] - Page_ProjectIterationItem_: - properties: - _meta: - $ref: '#/components/schemas/PageMetaInfoLimitOffset' - _links: - $ref: '#/components/schemas/PageLinks' - data: - items: - $ref: '#/components/schemas/ProjectIterationItem' - type: array - title: Data - additionalProperties: false - type: object - required: - - _meta - - _links - - data - title: Page[ProjectIterationItem] - Page_ProjectIterationResultItem_: - properties: - _meta: - $ref: '#/components/schemas/PageMetaInfoLimitOffset' - _links: - $ref: '#/components/schemas/PageLinks' - data: - items: - $ref: '#/components/schemas/ProjectIterationResultItem' - type: array - title: Data - additionalProperties: false - type: object - required: - - _meta - - _links - - data - title: Page[ProjectIterationResultItem] Page_ProjectListItem_: properties: _meta: @@ -12014,24 +12638,6 @@ components: - _links - data title: Page[ProjectListItem] - Page_RepoApiModel_: - properties: - _meta: - $ref: '#/components/schemas/PageMetaInfoLimitOffset' - _links: - $ref: '#/components/schemas/PageLinks' - data: - items: - $ref: '#/components/schemas/RepoApiModel' - type: array - title: Data - additionalProperties: false - type: object - required: - - _meta - - _links - - data - title: Page[RepoApiModel] Page_ServiceRunGet_: properties: _meta: @@ -12050,22 +12656,6 @@ components: - _links - data title: Page[ServiceRunGet] - ParentMetaProjectRef: - properties: - project_id: - type: string - format: uuid - title: Project Id - ref_id: - type: integer - exclusiveMinimum: true - title: Ref Id - minimum: 0 - type: object - required: - - project_id - - ref_id - title: ParentMetaProjectRef PatchRequestBody: properties: value: @@ -12074,6 +12664,30 @@ components: required: - value title: PatchRequestBody + PathMetaDataGet: + properties: + path: + type: string + format: path + title: Path + description: the path to the current path + display_path: + type: string + format: path + title: Display Path + description: the path to display with UUID replaced (URL Encoded by parts + as names may contain '/') + file_meta_data: + anyOf: + - $ref: '#/components/schemas/FileMetaDataGet' + - type: 'null' + description: if filled, this is the file meta data of the s3 object + additionalProperties: false + type: object + required: + - path + - display_path + title: PathMetaDataGet PaymentMethodGet: properties: idr: @@ -12510,7 +13124,10 @@ components: type: string title: Unitname unitExtraInfo: - $ref: '#/components/schemas/UnitExtraInfo-Output' + anyOf: + - $ref: '#/components/schemas/UnitExtraInfoTier-Output' + - $ref: '#/components/schemas/UnitExtraInfoLicense' + title: Unitextrainfo currentCostPerUnit: type: string title: Currentcostperunit @@ -12554,7 +13171,10 @@ components: type: string title: Unitname unitExtraInfo: - $ref: '#/components/schemas/UnitExtraInfo-Output' + anyOf: + - $ref: '#/components/schemas/UnitExtraInfoTier-Output' + - $ref: '#/components/schemas/UnitExtraInfoLicense' + title: Unitextrainfo currentCostPerUnit: type: string title: Currentcostperunit @@ -12569,6 +13189,113 @@ components: - currentCostPerUnit - default title: PricingUnitGet + ProductGet: + properties: + name: + type: string + title: Name + displayName: + type: string + title: Displayname + shortName: + anyOf: + - type: string + - type: 'null' + title: Shortname + description: Short display name for SMS + vendor: + anyOf: + - type: object + - type: 'null' + title: Vendor + description: vendor attributes + issues: + anyOf: + - items: + type: object + type: array + - type: 'null' + title: Issues + description: Reference to issues tracker + manuals: + anyOf: + - items: + type: object + type: array + - type: 'null' + title: Manuals + description: List of manuals + support: + anyOf: + - items: + type: object + type: array + - type: 'null' + title: Support + description: List of support resources + loginSettings: + type: object + title: Loginsettings + maxOpenStudiesPerUser: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Maxopenstudiesperuser + isPaymentEnabled: + type: boolean + title: Ispaymentenabled + creditsPerUsd: + anyOf: + - type: string + - type: 'null' + title: Creditsperusd + templates: + items: + $ref: '#/components/schemas/ProductTemplateGet' + type: array + title: Templates + description: List of templates available to this product for communications + (e.g. emails, sms, etc) + type: object + required: + - name + - displayName + - loginSettings + - maxOpenStudiesPerUser + - isPaymentEnabled + - creditsPerUsd + title: ProductGet + ProductTemplateGet: + properties: + id: + type: string + maxLength: 100 + minLength: 1 + title: Id + content: + type: string + title: Content + type: object + required: + - id + - content + title: ProductTemplateGet + ProductUIGet: + properties: + productName: + type: string + title: Productname + ui: + type: object + title: Ui + description: Front-end owned ui product configuration + type: object + required: + - productName + - ui + title: ProductUIGet ProjectCopyOverride: properties: name: @@ -12683,14 +13410,6 @@ components: - type: string const: '' title: Thumbnail - creationDate: - type: string - pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z - title: Creationdate - lastChangeDate: - type: string - pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z - title: Lastchangedate workbench: type: object title: Workbench @@ -12706,6 +13425,32 @@ components: minLength: 1 type: object title: Accessrights + creationDate: + type: string + pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z + title: Creationdate + lastChangeDate: + type: string + pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z + title: Lastchangedate + state: + anyOf: + - $ref: '#/components/schemas/ProjectState' + - type: 'null' + trashedAt: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Trashedat + trashedBy: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Trashedby + description: The primary gid of the user who trashed tags: items: type: integer @@ -12716,21 +13461,17 @@ components: type: string type: array title: Classifiers - default: [] - state: - anyOf: - - $ref: '#/components/schemas/ProjectState' - - type: 'null' + default: [] + quality: + type: object + title: Quality + default: {} ui: anyOf: - $ref: '#/components/schemas/EmptyModel' - $ref: '#/components/schemas/StudyUI-Output' - type: 'null' title: Ui - quality: - type: object - title: Quality - default: {} dev: anyOf: - type: object @@ -12754,37 +13495,23 @@ components: minimum: 0 - type: 'null' title: Folderid - trashedAt: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Trashedat - trashedBy: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Trashedby - description: The primary gid of the user who trashed type: object required: - uuid - name - description - thumbnail - - creationDate - - lastChangeDate - workbench - prjOwner - accessRights + - creationDate + - lastChangeDate + - trashedAt + - trashedBy - tags - dev - workspaceId - folderId - - trashedAt - - trashedBy title: ProjectGet ProjectGroupGet: properties: @@ -12855,73 +13582,6 @@ components: - key - value title: ProjectInputUpdate - ProjectIterationItem: - properties: - name: - type: string - title: Name - description: Iteration's resource API name - parent: - $ref: '#/components/schemas/ParentMetaProjectRef' - description: Reference to the the meta-project that created this iteration - iteration_index: - type: integer - exclusiveMinimum: true - title: Iteration Index - minimum: 0 - workcopy_project_id: - type: string - format: uuid - title: Workcopy Project Id - description: ID to this iteration's working copy.A working copy is a real - project where this iteration is run - workcopy_project_url: - type: string - title: Workcopy Project Url - description: reference to a working copy project - type: object - required: - - name - - parent - - iteration_index - - workcopy_project_id - - workcopy_project_url - title: ProjectIterationItem - ProjectIterationResultItem: - properties: - name: - type: string - title: Name - description: Iteration's resource API name - parent: - $ref: '#/components/schemas/ParentMetaProjectRef' - description: Reference to the the meta-project that created this iteration - iteration_index: - type: integer - exclusiveMinimum: true - title: Iteration Index - minimum: 0 - workcopy_project_id: - type: string - format: uuid - title: Workcopy Project Id - description: ID to this iteration's working copy.A working copy is a real - project where this iteration is run - workcopy_project_url: - type: string - title: Workcopy Project Url - description: reference to a working copy project - results: - $ref: '#/components/schemas/ExtractedResults' - type: object - required: - - name - - parent - - iteration_index - - workcopy_project_id - - workcopy_project_url - - results - title: ProjectIterationResultItem ProjectListItem: properties: uuid: @@ -12940,14 +13600,6 @@ components: - type: string const: '' title: Thumbnail - creationDate: - type: string - pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z - title: Creationdate - lastChangeDate: - type: string - pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z - title: Lastchangedate workbench: type: object title: Workbench @@ -12963,6 +13615,32 @@ components: minLength: 1 type: object title: Accessrights + creationDate: + type: string + pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z + title: Creationdate + lastChangeDate: + type: string + pattern: \d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z + title: Lastchangedate + state: + anyOf: + - $ref: '#/components/schemas/ProjectState' + - type: 'null' + trashedAt: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Trashedat + trashedBy: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Trashedby + description: The primary gid of the user who trashed tags: items: type: integer @@ -12974,20 +13652,16 @@ components: type: array title: Classifiers default: [] - state: - anyOf: - - $ref: '#/components/schemas/ProjectState' - - type: 'null' + quality: + type: object + title: Quality + default: {} ui: anyOf: - $ref: '#/components/schemas/EmptyModel' - $ref: '#/components/schemas/StudyUI-Output' - type: 'null' title: Ui - quality: - type: object - title: Quality - default: {} dev: anyOf: - type: object @@ -13011,37 +13685,23 @@ components: minimum: 0 - type: 'null' title: Folderid - trashedAt: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Trashedat - trashedBy: - anyOf: - - type: integer - exclusiveMinimum: true - minimum: 0 - - type: 'null' - title: Trashedby - description: The primary gid of the user who trashed type: object required: - uuid - name - description - thumbnail - - creationDate - - lastChangeDate - workbench - prjOwner - accessRights + - creationDate + - lastChangeDate + - trashedAt + - trashedBy - tags - dev - workspaceId - folderId - - trashedAt - - trashedBy title: ProjectListItem ProjectLocked: properties: @@ -13123,6 +13783,22 @@ components: required: - custom title: ProjectMetadataUpdate + ProjectNodeServicesGet: + properties: + projectUuid: + type: string + format: uuid + title: Projectuuid + services: + items: + $ref: '#/components/schemas/NodeServiceGet' + type: array + title: Services + type: object + required: + - projectUuid + - services + title: ProjectNodeServicesGet ProjectOutputGet: properties: key: @@ -13426,20 +14102,6 @@ components: - topUpAmountInUsd - monthlyLimitInUsd title: ReplaceWalletAutoRecharge - RepoApiModel: - properties: - project_uuid: - type: string - format: uuid - title: Project Uuid - url: - type: string - title: Url - type: object - required: - - project_uuid - - url - title: RepoApiModel ResearchResource: properties: rrid: @@ -13483,6 +14145,7 @@ components: properties: email: type: string + format: email title: Email additionalProperties: false type: object @@ -13989,9 +14652,9 @@ components: format: date-time - type: 'null' title: Retired - description: 'whether this service is planned to be retired. If None, the + description: whether this service is planned to be retired. If None, the service is still active. If now
| " + " | ".join(column_headers) + " |
|---|---|
| " + converted_output += " | ".join( + [ + self.convert_json_node(list_entry[column_header]) + for column_header in column_headers + ] + ) + converted_output += " |
- " + converted_output += "
- ".join( + [self.convert_json_node(child) for child in list_input] + ) + converted_output += "
Dear {{ name }},
-A request to reset your {{ host }} password has been made.
-It could not be completed due to the following reason:
-{{ reason }}
-If you did not request this, please contact us immediatly at {{ product.support_email }} for security reasons.
-Best Regards,
-The {{ product.display_name }} Team
diff --git a/services/web/server/src/simcore_service_webserver/tracing.py b/services/web/server/src/simcore_service_webserver/tracing.py index 23041d95238..d07757106e8 100644 --- a/services/web/server/src/simcore_service_webserver/tracing.py +++ b/services/web/server/src/simcore_service_webserver/tracing.py @@ -5,8 +5,8 @@ from servicelib.aiohttp.tracing import setup_tracing from settings_library.tracing import TracingSettings -from ._constants import APP_SETTINGS_KEY from ._meta import APP_NAME +from .constants import APP_SETTINGS_KEY log = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/trash/_rest.py b/services/web/server/src/simcore_service_webserver/trash/_rest.py index f5912b042fe..d6971984086 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_rest.py +++ b/services/web/server/src/simcore_service_webserver/trash/_rest.py @@ -1,7 +1,10 @@ +import asyncio import logging from aiohttp import web from servicelib.aiohttp import status +from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY +from servicelib.utils import fire_and_forget_task from .._meta import API_VTAG as VTAG from ..exception_handling import ( @@ -11,17 +14,13 @@ to_exceptions_handlers_map, ) from ..login.decorators import get_user_id, login_required -from ..products.api import get_product_name +from ..products import products_web from ..projects.exceptions import ProjectRunningConflictError, ProjectStoppingError from ..security.decorators import permission_required from . import _service _logger = logging.getLogger(__name__) -# -# EXCEPTIONS HANDLING -# - _TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { ProjectRunningConflictError: HttpErrorInfo( @@ -40,21 +39,34 @@ ) -# -# ROUTES -# - routes = web.RouteTableDef() -@routes.delete(f"/{VTAG}/trash", name="empty_trash") +@routes.post(f"/{VTAG}/trash:empty", name="empty_trash") @login_required @permission_required("project.delete") @_handle_exceptions async def empty_trash(request: web.Request): user_id = get_user_id(request) - product_name = get_product_name(request) + product_name = products_web.get_product_name(request) + + explicitly_trashed_project_deleted = asyncio.Event() + + fire_and_forget_task( + _service.safe_empty_trash( + request.app, + product_name=product_name, + user_id=user_id, + on_explicitly_trashed_projects_deleted=explicitly_trashed_project_deleted, + ), + task_suffix_name="rest.empty_trash", + fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], + ) - await _service.empty_trash(request.app, product_name=product_name, user_id=user_id) + # NOTE: Ensures `fire_and_forget_task` is triggered and deletes explicit projects; + # otherwise, when the front-end requests the trash item list, + # it may still display items, misleading the user into + # thinking the `empty trash` operation failed. + await explicitly_trashed_project_deleted.wait() return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py index cc94d680d64..1f38043faf5 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_service.py +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -1,36 +1,179 @@ import asyncio import logging from datetime import timedelta +from typing import Final +import arrow from aiohttp import web from models_library.products import ProductName from models_library.users import UserID +from servicelib.logging_errors import create_troubleshotting_log_kwargs +from servicelib.logging_utils import log_context +from ..folders import folders_trash_service +from ..products import products_service +from ..projects import projects_trash_service from .settings import get_plugin_settings _logger = logging.getLogger(__name__) +_TIP: Final[str] = ( + "`empty_trash_safe` is set `fail_fast=False`." + "\nErrors while deletion are ignored." + "\nNew runs might resolve them" +) -async def empty_trash(app: web.Application, product_name: ProductName, user_id: UserID): - assert app # nosec - # filter trashed=True and set them to False - _logger.debug( - "CODE PLACEHOLDER: all projects marked as trashed of %s in %s are deleted", - f"{user_id=}", - f"{product_name=}", + +async def _empty_explicitly_trashed_projects( + app: web.Application, product_name: ProductName, user_id: UserID +): + trashed_projects_ids = ( + await projects_trash_service.list_explicitly_trashed_projects( + app=app, product_name=product_name, user_id=user_id + ) ) - raise NotImplementedError + with log_context( + _logger, + logging.DEBUG, + "Deleting %s explicitly trashed projects", + len(trashed_projects_ids), + ): + for project_id in trashed_projects_ids: + try: + + await projects_trash_service.delete_explicitly_trashed_project( + app, + user_id=user_id, + project_id=project_id, + ) + + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error deleting a trashed project while emptying trash.", + error=exc, + error_context={ + "project_id": project_id, + "product_name": product_name, + "user_id": user_id, + }, + tip=_TIP, + ) + ) + + +async def _empty_explicitly_trashed_folders_and_content( + app: web.Application, product_name: ProductName, user_id: UserID +): + trashed_folders_ids = await folders_trash_service.list_explicitly_trashed_folders( + app=app, product_name=product_name, user_id=user_id + ) + + with log_context( + _logger, + logging.DEBUG, + "Deleting %s trashed folders (and all its content)", + len(trashed_folders_ids), + ): + for folder_id in trashed_folders_ids: + try: + await folders_trash_service.delete_trashed_folder( + app, + product_name=product_name, + user_id=user_id, + folder_id=folder_id, + ) + + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error deleting a trashed folders (and content) while emptying trash.", + error=exc, + error_context={ + "folder_id": folder_id, + "product_name": product_name, + "user_id": user_id, + }, + tip=_TIP, + ) + ) + + +async def safe_empty_trash( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + on_explicitly_trashed_projects_deleted: asyncio.Event | None = None +): + # Delete explicitly trashed projects & notify + await _empty_explicitly_trashed_projects(app, product_name, user_id) + if on_explicitly_trashed_projects_deleted: + on_explicitly_trashed_projects_deleted.set() + + # Delete explicitly trashed folders (and all implicitly trashed sub-folders and projects) + await _empty_explicitly_trashed_folders_and_content(app, product_name, user_id) -async def prune_trash(app: web.Application) -> list[str]: - """Deletes expired items in the trash""" + +async def safe_delete_expired_trash_as_admin(app: web.Application) -> None: settings = get_plugin_settings(app) retention = timedelta(days=settings.TRASH_RETENTION_DAYS) + delete_until = arrow.now().datetime - retention - _logger.debug( - "CODE PLACEHOLDER: **ALL** projects marked as trashed during %s days are deleted", - retention, - ) - await asyncio.sleep(5) + app_products_names = await products_service.list_products_names(app) + + for product_name in app_products_names: + + ctx = { + "delete_until": delete_until, + "retention": retention, + "product_name": product_name, + } + + with log_context( + _logger, + logging.DEBUG, + "Deleting items marked as trashed before %s in %s [trashed_at < %s will be deleted]", + retention, + product_name, + delete_until, + ): + try: + + await folders_trash_service.batch_delete_trashed_folders_as_admin( + app, + trashed_before=delete_until, + product_name=product_name, + fail_fast=False, + ) + + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error batch deleting expired trashed folders as admin.", + error=exc, + error_context=ctx, + ) + ) + + try: + + deleted_project_ids = ( + await projects_trash_service.batch_delete_trashed_projects_as_admin( + app, + trashed_before=delete_until, + fail_fast=False, + ) + ) + + _logger.info("Deleted %d trashed projects", len(deleted_project_ids)) - return [] + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error batch deleting expired projects as admin.", + error=exc, + error_context=ctx, + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/trash/plugin.py b/services/web/server/src/simcore_service_webserver/trash/plugin.py index a4cde641596..977a1c74884 100644 --- a/services/web/server/src/simcore_service_webserver/trash/plugin.py +++ b/services/web/server/src/simcore_service_webserver/trash/plugin.py @@ -8,7 +8,7 @@ from aiohttp import web from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from .._constants import APP_SETTINGS_KEY +from ..constants import APP_SETTINGS_KEY from ..folders.plugin import setup_folders from ..projects.plugin import setup_projects from ..workspaces.plugin import setup_workspaces diff --git a/services/web/server/src/simcore_service_webserver/trash/settings.py b/services/web/server/src/simcore_service_webserver/trash/settings.py index 38d4f91fdcb..f51832b9aa7 100644 --- a/services/web/server/src/simcore_service_webserver/trash/settings.py +++ b/services/web/server/src/simcore_service_webserver/trash/settings.py @@ -2,7 +2,7 @@ from pydantic import Field, NonNegativeInt from settings_library.base import BaseCustomSettings -from .._constants import APP_SETTINGS_KEY +from ..constants import APP_SETTINGS_KEY class TrashSettings(BaseCustomSettings): diff --git a/services/web/server/src/simcore_service_webserver/trash/trash_service.py b/services/web/server/src/simcore_service_webserver/trash/trash_service.py new file mode 100644 index 00000000000..3cd438f4e1c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/trash/trash_service.py @@ -0,0 +1,4 @@ +from ._service import safe_delete_expired_trash_as_admin + +__all__: tuple[str, ...] = ("safe_delete_expired_trash_as_admin",) +# nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/users/_common/models.py b/services/web/server/src/simcore_service_webserver/users/_common/models.py index 513d8bed102..967f010d0b0 100644 --- a/services/web/server/src/simcore_service_webserver/users/_common/models.py +++ b/services/web/server/src/simcore_service_webserver/users/_common/models.py @@ -55,6 +55,7 @@ class ToUserUpdateDB(BaseModel): first_name: str | None = None last_name: str | None = None + privacy_hide_username: bool | None = None privacy_hide_fullname: bool | None = None privacy_hide_email: bool | None = None diff --git a/services/web/server/src/simcore_service_webserver/users/_common/schemas.py b/services/web/server/src/simcore_service_webserver/users/_common/schemas.py index 04946e21fcc..a76326182ae 100644 --- a/services/web/server/src/simcore_service_webserver/users/_common/schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/_common/schemas.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from servicelib.request_keys import RQT_USERID_KEY -from ..._constants import RQ_PRODUCT_KEY +from ...constants import RQ_PRODUCT_KEY class UsersRequestContext(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py index 2e243d4da90..65c427bf7b0 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py @@ -15,7 +15,7 @@ from .._meta import API_VTAG from ..login.decorators import login_required -from ..products.api import get_product_name +from ..products import products_web from ..redis import get_redis_user_notifications_client from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response @@ -62,7 +62,7 @@ async def _get_user_notifications( async def list_user_notifications(request: web.Request) -> web.Response: redis_client = get_redis_user_notifications_client(request.app) req_ctx = UsersRequestContext.model_validate(request) - product_name = get_product_name(request) + product_name = products_web.get_product_name(request) notifications = await _get_user_notifications( redis_client, req_ctx.user_id, product_name ) diff --git a/services/web/server/src/simcore_service_webserver/users/_users_repository.py b/services/web/server/src/simcore_service_webserver/users/_users_repository.py index 16730437394..8f13169e147 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_repository.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_repository.py @@ -5,6 +5,7 @@ from aiohttp import web from common_library.users_enums import UserRole from models_library.groups import GroupID +from models_library.products import ProductName from models_library.users import ( MyProfile, UserBillingDetails, @@ -58,8 +59,7 @@ def _public_user_cols(caller_id: int): return ( # Fits PublicUser model users.c.id.label("user_id"), - users.c.name.label("user_name"), - *visible_user_profile_cols(caller_id), + *visible_user_profile_cols(caller_id, username_label="user_name"), users.c.primary_gid.label("group_id"), ) @@ -102,7 +102,10 @@ async def search_public_user( query = ( sa.select(*_public_user_cols(caller_id=caller_id)) .where( - users.c.name.ilike(_pattern) + ( + is_public(users.c.privacy_hide_username, caller_id) + & users.c.name.ilike(_pattern) + ) | ( is_public(users.c.privacy_hide_email, caller_id) & users.c.email.ilike(_pattern) @@ -151,7 +154,10 @@ async def get_user_or_raise( async def get_user_primary_group_id( - engine: AsyncEngine, connection: AsyncConnection | None = None, *, user_id: UserID + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + user_id: UserID, ) -> GroupID: async with pass_or_acquire_connection(engine, connection) as conn: primary_gid: GroupID | None = await conn.scalar( @@ -179,7 +185,7 @@ async def get_users_ids_in_group( return {row.uid async for row in result} -async def get_user_id_from_pgid(app: web.Application, primary_gid: int) -> UserID: +async def get_user_id_from_pgid(app: web.Application, *, primary_gid: int) -> UserID: async with pass_or_acquire_connection(engine=get_asyncpg_engine(app)) as conn: user_id: UserID = await conn.scalar( sa.select( @@ -386,13 +392,9 @@ async def get_user_products( .where(products.c.group_id == groups.c.gid) .label("product_name") ) - products_gis_subq = ( - sa.select( - products.c.group_id, - ) - .distinct() - .subquery() - ) + products_group_ids_subq = sa.select( + products.c.group_id, + ).distinct() query = ( sa.select( groups.c.gid, @@ -402,7 +404,7 @@ async def get_user_products( users.join(user_to_groups, user_to_groups.c.uid == users.c.id).join( groups, (groups.c.gid == user_to_groups.c.gid) - & groups.c.gid.in_(products_gis_subq), + & groups.c.gid.in_(products_group_ids_subq), ) ) .where(users.c.id == user_id) @@ -461,6 +463,32 @@ async def delete_user_by_id( return bool(deleted_user) +async def is_user_in_product_name( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + user_id: UserID, + product_name: ProductName, +) -> bool: + query = ( + sa.select(users.c.id) + .select_from( + users.join( + user_to_groups, + user_to_groups.c.uid == users.c.id, + ).join( + products, + products.c.group_id == user_to_groups.c.gid, + ) + ) + .where((users.c.id == user_id) & (products.c.name == product_name)) + ) + async with pass_or_acquire_connection(engine, connection) as conn: + value = await conn.scalar(query) + assert value is None or value == user_id # nosec + return value is not None + + # # USER PROFILE # @@ -480,6 +508,8 @@ async def get_my_profile(app: web.Application, *, user_id: UserID) -> MyProfile: users.c.email, users.c.role, sa.func.json_build_object( + "hide_username", + users.c.privacy_hide_username, "hide_fullname", users.c.privacy_hide_fullname, "hide_email", @@ -530,11 +560,12 @@ async def update_user_profile( ) except IntegrityError as err: - user_name = updated_values.get("name") - - raise UserNameDuplicateError( - user_name=user_name, - alternative_user_name=generate_alternative_username(user_name), - user_id=user_id, - updated_values=updated_values, - ) from err + if user_name := updated_values.get("name"): + raise UserNameDuplicateError( + user_name=user_name, + alternative_user_name=generate_alternative_username(user_name), + user_id=user_id, + updated_values=updated_values, + ) from err + + raise # not due to name duplication diff --git a/services/web/server/src/simcore_service_webserver/users/_users_rest.py b/services/web/server/src/simcore_service_webserver/users/_users_rest.py index d2ece688514..e89814e5e2d 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_rest.py @@ -15,8 +15,6 @@ parse_request_query_parameters_as, ) from servicelib.rest_constants import RESPONSE_MODEL_POLICY -from simcore_service_webserver.products._api import get_current_product -from simcore_service_webserver.products._model import Product from .._meta import API_VTAG from ..exception_handling import ( @@ -28,6 +26,8 @@ from ..groups import api as groups_api from ..groups.exceptions import GroupNotFoundError from ..login.decorators import login_required +from ..products import products_web +from ..products.models import Product from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _users_service @@ -59,8 +59,7 @@ MissingGroupExtraPropertiesForProductError: HttpErrorInfo( status.HTTP_503_SERVICE_UNAVAILABLE, "The product is not ready for use until the configuration is fully completed. " - "Please wait and try again. " - "If this issue persists, contact support indicating this support code: {error_code}.", + "Please wait and try again. ", ), } @@ -81,7 +80,7 @@ @login_required @_handle_users_exceptions async def get_my_profile(request: web.Request) -> web.Response: - product: Product = get_current_product(request) + product: Product = products_web.get_current_product(request) req_ctx = UsersRequestContext.model_validate(request) groups_by_type = await groups_api.list_user_groups_with_read_access( @@ -114,9 +113,6 @@ async def get_my_profile(request: web.Request) -> web.Response: @routes.patch(f"/{API_VTAG}/me", name="update_my_profile") -@routes.put( - f"/{API_VTAG}/me", name="replace_my_profile" # deprecated. Use patch instead -) @login_required @permission_required("user.profile.update") @_handle_users_exceptions @@ -181,7 +177,10 @@ async def search_users_for_admin(request: web.Request) -> web.Response: ) return envelope_json_response( - [_.model_dump(**_RESPONSE_MODEL_MINIMAL_POLICY) for _ in found] + [ + user_for_admin.model_dump(**_RESPONSE_MODEL_MINIMAL_POLICY) + for user_for_admin in found + ] ) diff --git a/services/web/server/src/simcore_service_webserver/users/_users_service.py b/services/web/server/src/simcore_service_webserver/users/_users_service.py index 2bb52b85d57..5d71423646d 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_service.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_service.py @@ -122,7 +122,7 @@ async def get_user_primary_group_id(app: web.Application, user_id: UserID) -> Gr async def get_user_id_from_gid(app: web.Application, primary_gid: GroupID) -> UserID: - return await _users_repository.get_user_id_from_pgid(app, primary_gid) + return await _users_repository.get_user_id_from_pgid(app, primary_gid=primary_gid) async def search_users( @@ -180,6 +180,14 @@ async def get_users_in_group(app: web.Application, *, gid: GroupID) -> set[UserI get_guest_user_ids_and_names = _users_repository.get_guest_user_ids_and_names +async def is_user_in_product( + app: web.Application, *, user_id: UserID, product_name: ProductName +) -> bool: + return await _users_repository.is_user_in_product_name( + get_asyncpg_engine(app), user_id=user_id, product_name=product_name + ) + + # # GET USER PROPERTIES # diff --git a/services/web/server/src/simcore_service_webserver/users/api.py b/services/web/server/src/simcore_service_webserver/users/api.py index 09ca7b757e6..0d00834c02d 100644 --- a/services/web/server/src/simcore_service_webserver/users/api.py +++ b/services/web/server/src/simcore_service_webserver/users/api.py @@ -14,13 +14,17 @@ get_user_primary_group_id, get_user_role, get_users_in_group, + is_user_in_product, set_user_as_deleted, update_expired_users, ) __all__: tuple[str, ...] = ( + "FullNameDict", + "UserDisplayAndIdNamesTuple", "delete_user_without_projects", "get_guest_user_ids_and_names", + "get_user", "get_user_credentials", "get_user_display_and_id_names", "get_user_fullname", @@ -29,11 +33,9 @@ "get_user_name_and_email", "get_user_primary_group_id", "get_user_role", - "get_user", "get_users_in_group", + "is_user_in_product", "set_user_as_deleted", "update_expired_users", - "FullNameDict", - "UserDisplayAndIdNamesTuple", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/users/settings.py b/services/web/server/src/simcore_service_webserver/users/settings.py index 2b6b9f101ac..3800f55d635 100644 --- a/services/web/server/src/simcore_service_webserver/users/settings.py +++ b/services/web/server/src/simcore_service_webserver/users/settings.py @@ -3,7 +3,7 @@ from settings_library.base import BaseCustomSettings from settings_library.utils_service import MixinServiceSettings -from .._constants import APP_SETTINGS_KEY +from ..constants import APP_SETTINGS_KEY class UsersSettings(BaseCustomSettings, MixinServiceSettings): diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index bb60b8a1b8f..5a13e108201 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -14,7 +14,7 @@ from servicelib.rest_constants import RESPONSE_MODEL_POLICY from yarl import URL -from ._constants import INDEX_RESOURCE_NAME +from .constants import INDEX_RESOURCE_NAME _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py b/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py index 8c1117300ca..2266170c5ac 100644 --- a/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py +++ b/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py @@ -1,11 +1,13 @@ +from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from functools import wraps from math import ceil -from typing import Callable, NamedTuple +from typing import Final, NamedTuple from aiohttp.web_exceptions import HTTPTooManyRequests -from common_library.json_serialization import json_dumps +from models_library.rest_error import EnvelopedError, ErrorGet +from servicelib.aiohttp import status class RateLimitSetup(NamedTuple): @@ -13,7 +15,16 @@ class RateLimitSetup(NamedTuple): interval_seconds: float -def global_rate_limit_route(number_of_requests: int, interval_seconds: float): +MSG_TOO_MANY_REQUESTS: Final[str] = ( + "Requests are being made too frequently. Please wait a moment before trying again." +) + + +def global_rate_limit_route( + number_of_requests: int, + interval_seconds: float, + error_msg: str = MSG_TOO_MANY_REQUESTS, +): """ Limits the requests per given interval to this endpoint from all incoming sources. @@ -41,7 +52,7 @@ class _Context: @wraps(decorated_function) async def _wrapper(*args, **kwargs): - utc_now = datetime.utcnow() + utc_now = datetime.now(UTC) utc_now_timestamp = datetime.timestamp(utc_now) # reset counter & first time initialization @@ -61,16 +72,18 @@ async def _wrapper(*args, **kwargs): "Content-Type": "application/json", "Retry-After": f"{retry_after_sec}", }, - text=json_dumps( - { - "error": { - "logs": [{"message": "API rate limit exceeded."}], - "status": HTTPTooManyRequests.status_code, - } - } - ), + text=EnvelopedError( + error=ErrorGet( + message=error_msg, + status=status.HTTP_429_TOO_MANY_REQUESTS, + ) + ).model_dump_json(), ) + assert ( # nosec + HTTPTooManyRequests.status_code == status.HTTP_429_TOO_MANY_REQUESTS + ) + # increase counter and return original function call context.remaining -= 1 return await decorated_function(*args, **kwargs) diff --git a/services/web/server/src/simcore_service_webserver/version_control/_core.py b/services/web/server/src/simcore_service_webserver/version_control/_core.py deleted file mode 100644 index 860d124ce48..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/_core.py +++ /dev/null @@ -1,155 +0,0 @@ -""" - A checkpoint is equivalent to a commit and can be tagged at the same time (*) - - Working copy - - HEAD revision - - (*) This is a concept introduced for the front-end to avoid using - more fine grained concepts as tags and commits directly -""" -import logging -from uuid import UUID - -from aiopg.sa.result import RowProxy -from pydantic import NonNegativeInt, PositiveInt, validate_call - -from .db import VersionControlRepository -from .errors import CleanRequiredError -from .models import Checkpoint, CommitLog, RefID, WorkbenchView - -_logger = logging.getLogger(__name__) - - -async def list_repos( - vc_repo: VersionControlRepository, - *, - offset: NonNegativeInt = 0, - limit: PositiveInt | None = None, -) -> tuple[list[RowProxy], PositiveInt]: - # NOTE: this layer does NOT add much .. why not use vc_repo directly? - repos_rows, total_number_of_repos = await vc_repo.list_repos(offset, limit) - - assert len(repos_rows) <= total_number_of_repos # nosec - return repos_rows, total_number_of_repos - - -async def list_checkpoints( - vc_repo: VersionControlRepository, - project_uuid: UUID, - *, - offset: NonNegativeInt = 0, - limit: PositiveInt | None = None, -) -> tuple[list[Checkpoint], PositiveInt]: - repo_id = await vc_repo.get_repo_id(project_uuid) - if not repo_id: - return [], 0 - - logs: list[CommitLog] - logs, total_number_of_commits = await vc_repo.log( - repo_id, offset=offset, limit=limit - ) - - checkpoints = [Checkpoint.from_commit_log(commit, tags) for commit, tags in logs] - assert len(checkpoints) <= limit if limit else True # nosec - assert total_number_of_commits > 0 # nosec - - return checkpoints, total_number_of_commits - - -async def create_checkpoint( - vc_repo: VersionControlRepository, - project_uuid: UUID, - *, - tag: str, - message: str | None = None, -) -> Checkpoint: - repo_id = await vc_repo.get_repo_id(project_uuid) - if repo_id is None: - repo_id = await vc_repo.init_repo(project_uuid) - - commit_id = await vc_repo.commit(repo_id, tag=tag, message=message) - commit, tags = await vc_repo.get_commit_log(commit_id) - assert commit # nosec - - return Checkpoint.from_commit_log(commit, tags) - - -async def get_checkpoint( - vc_repo: VersionControlRepository, - project_uuid: UUID, - ref_id: RefID, -) -> Checkpoint: - repo_id, commit_id = await vc_repo.as_repo_and_commit_ids(project_uuid, ref_id) - assert repo_id # nosec - - commit, tags = await vc_repo.get_commit_log(commit_id) - return Checkpoint.from_commit_log(commit, tags) - - -async def update_checkpoint( - vc_repo: VersionControlRepository, - project_uuid: UUID, - ref_id: RefID, - *, - message: str | None = None, - tag: str | None = None, -) -> Checkpoint: - repo_id, commit_id = await vc_repo.as_repo_and_commit_ids(project_uuid, ref_id) - - if message is None and tag is None: - _logger.warning( - "Nothing to update. Skipping updating ref %s of %s", ref_id, project_uuid - ) - else: - await vc_repo.update_annotations(repo_id, commit_id, message, tag) - - commit, tags = await vc_repo.get_commit_log(commit_id) - return Checkpoint.from_commit_log(commit, tags) - - -async def checkout_checkpoint( - vc_repo: VersionControlRepository, - project_uuid: UUID, - ref_id: RefID, -) -> Checkpoint: - repo_id, commit_id = await vc_repo.as_repo_and_commit_ids(project_uuid, ref_id) - - # check if working copy has changes, if so, auto commit it - try: - commit_id = await vc_repo.checkout(repo_id, commit_id) - except CleanRequiredError: - _logger.info("Local changes found. Auto-commiting project %s", project_uuid) - await vc_repo.commit(repo_id, message="auto commit") - commit_id = await vc_repo.checkout(repo_id, commit_id) - - commit, tags = await vc_repo.get_commit_log(commit_id) - return Checkpoint.from_commit_log(commit, tags) - - -async def get_workbench( - vc_repo: VersionControlRepository, - project_uuid: UUID, - ref_id: RefID, -) -> WorkbenchView: - repo_id, commit_id = await vc_repo.as_repo_and_commit_ids(project_uuid, ref_id) - - # prefer actual project to snapshot - content = await vc_repo.get_workbench_view(repo_id, commit_id) - return WorkbenchView.model_validate(content) - - -# -# All above with validated arguments -# - -_CONFIG = {"arbitrary_types_allowed": True} - - -list_repos_safe = validate_call(list_repos, config=_CONFIG) # type: ignore -list_checkpoints_safe = validate_call(list_checkpoints, config=_CONFIG) # type: ignore -create_checkpoint_safe = validate_call(create_checkpoint, config=_CONFIG) # type: ignore -get_checkpoint_safe = validate_call(get_checkpoint, config=_CONFIG) # type: ignore -update_checkpoint_safe = validate_call(update_checkpoint, config=_CONFIG) # type: ignore -checkout_checkpoint_safe = validate_call(checkout_checkpoint, config=_CONFIG) # type: ignore -get_workbench_safe = validate_call(get_workbench, config=_CONFIG) # type: ignore diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py deleted file mode 100644 index a0847ea34ea..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py +++ /dev/null @@ -1,337 +0,0 @@ -import logging - -from aiohttp import web -from models_library.projects import ProjectID -from models_library.rest_pagination import Page, PageQueryParameters -from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel, field_validator -from servicelib.aiohttp.requests_validation import ( - parse_request_body_as, - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) -from servicelib.rest_constants import RESPONSE_MODEL_POLICY - -from .._meta import API_VTAG as VTAG -from ..login.decorators import login_required -from ..security.decorators import permission_required -from ..utils_aiohttp import create_url_for_function, envelope_json_response -from ._core import ( - checkout_checkpoint, - create_checkpoint, - get_checkpoint, - get_workbench, - list_checkpoints, - list_repos, - update_checkpoint, -) -from ._handlers_base import handle_request_errors -from .db import VersionControlRepository -from .models import ( - HEAD, - Checkpoint, - CheckpointAnnotations, - CheckpointApiModel, - CheckpointNew, - RefID, - RepoApiModel, - WorkbenchView, - WorkbenchViewApiModel, -) - -_logger = logging.getLogger(__name__) - - -class _CheckpointsPathParam(BaseModel): - project_uuid: ProjectID - ref_id: RefID - - @field_validator("ref_id", mode="before") - @classmethod - def _normalize_refid(cls, v): - if v and v == "HEAD": - return HEAD - return v - - -class _ProjectPathParam(BaseModel): - project_uuid: ProjectID - - -routes = web.RouteTableDef() - - -@routes.get(f"/{VTAG}/repos/projects", name="list_repos") -@login_required -@permission_required("project.read") -@handle_request_errors -async def _list_repos_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - query_params: PageQueryParameters = parse_request_query_parameters_as( - PageQueryParameters, request - ) - - repos_rows, total_number_of_repos = await list_repos( - vc_repo, offset=query_params.offset, limit=query_params.limit - ) - - assert len(repos_rows) <= query_params.limit # nosec - - # parse and validate - repos_list = [ - RepoApiModel.model_validate( - { - "url": url_for("list_repos"), - **dict(row.items()), - } - ) - for row in repos_rows - ] - - page = Page[RepoApiModel].model_validate( - paginate_data( - chunk=repos_list, - request_url=request.url, - total=total_number_of_repos, - limit=query_params.limit, - offset=query_params.offset, - ) - ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type="application/json", - ) - - -@routes.post( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints", name="create_checkpoint" -) -@login_required -@permission_required("project.create") -@handle_request_errors -async def _create_checkpoint_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_ProjectPathParam, request) - _body = CheckpointNew.model_validate(await request.json()) - - checkpoint: Checkpoint = await create_checkpoint( - vc_repo, - project_uuid=path_params.project_uuid, - **_body.model_dump(include={"tag", "message"}), - ) - - data = CheckpointApiModel.model_validate( - { - "url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **checkpoint.model_dump(), - } - ) - return envelope_json_response(data, status_cls=web.HTTPCreated) - - -@routes.get( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints", name="list_checkpoints" -) -@login_required -@permission_required("project.read") -@handle_request_errors -async def _list_checkpoints_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_ProjectPathParam, request) - query_params: PageQueryParameters = parse_request_query_parameters_as( - PageQueryParameters, request - ) - - checkpoints: list[Checkpoint] - - checkpoints, total = await list_checkpoints( - vc_repo, - project_uuid=path_params.project_uuid, - offset=query_params.offset, - limit=query_params.limit, - ) - - # parse and validate - checkpoints_list = [ - CheckpointApiModel.model_validate( - { - "url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **checkpoint.model_dump(), - } - ) - for checkpoint in checkpoints - ] - - page = Page[CheckpointApiModel].model_validate( - paginate_data( - chunk=checkpoints_list, - request_url=request.url, - total=total, - limit=query_params.limit, - offset=query_params.offset, - ) - ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type="application/json", - ) - - -# includes repos/projects/{project_uuid}/checkpoints/HEAD -@routes.get( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints/{{ref_id}}", - name="get_checkpoint", -) -@login_required -@permission_required("project.read") -@handle_request_errors -async def _get_checkpoint_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_CheckpointsPathParam, request) - - checkpoint: Checkpoint = await get_checkpoint( - vc_repo, - project_uuid=path_params.project_uuid, - ref_id=path_params.ref_id, - ) - - data = CheckpointApiModel.model_validate( - { - "url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), - } - ) - return envelope_json_response(data) - - -@routes.patch( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints/{{ref_id}}", - name="update_checkpoint", -) -@login_required -@permission_required("project.update") -@handle_request_errors -async def _update_checkpoint_annotations_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_CheckpointsPathParam, request) - update = await parse_request_body_as(CheckpointAnnotations, request) - - assert isinstance(path_params.ref_id, int) - - checkpoint: Checkpoint = await update_checkpoint( - vc_repo, - project_uuid=path_params.project_uuid, - ref_id=path_params.ref_id, - **update.model_dump(include={"tag", "message"}, exclude_none=True), - ) - - data = CheckpointApiModel.model_validate( - { - "url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), - } - ) - return envelope_json_response(data) - - -@routes.post( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints/{{ref_id}}:checkout", - name="checkout", -) -@login_required -@permission_required("project.create") -@handle_request_errors -async def _checkout_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_CheckpointsPathParam, request) - - checkpoint: Checkpoint = await checkout_checkpoint( - vc_repo, - project_uuid=path_params.project_uuid, - ref_id=path_params.ref_id, - ) - - data = CheckpointApiModel.model_validate( - { - "url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), - } - ) - return envelope_json_response(data) - - -@routes.get( - f"/{VTAG}/repos/projects/{{project_uuid}}/checkpoints/{{ref_id}}/workbench/view", - name="view_project_workbench", -) -@login_required -@permission_required("project.read") -@handle_request_errors -async def _view_project_workbench_handler(request: web.Request): - url_for = create_url_for_function(request) - vc_repo = VersionControlRepository.create_from_request(request) - - path_params = parse_request_path_parameters_as(_CheckpointsPathParam, request) - - checkpoint: Checkpoint = await get_checkpoint( - vc_repo, - project_uuid=path_params.project_uuid, - ref_id=path_params.ref_id, - ) - - view: WorkbenchView = await get_workbench( - vc_repo, - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ) - - data = WorkbenchViewApiModel.model_validate( - { - # = request.url?? - "url": url_for( - "view_project_workbench", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - "checkpoint_url": url_for( - "get_checkpoint", - project_uuid=path_params.project_uuid, - ref_id=checkpoint.id, - ), - **view.model_dump(**RESPONSE_MODEL_POLICY), - } - ) - - return envelope_json_response(data) diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py deleted file mode 100644 index 3424788fafa..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -from functools import wraps -from typing import Any - -from aiohttp import web -from common_library.json_serialization import json_dumps -from pydantic import ValidationError -from servicelib.aiohttp.typing_extension import Handler - -from ..projects.exceptions import ProjectNotFoundError -from .errors import InvalidParameterError, NoCommitError, NotFoundError - -_logger = logging.getLogger(__name__) - - -def handle_request_errors(handler: Handler) -> Handler: - """ - - required and type validation of path and query parameters - """ - - @wraps(handler) - async def wrapped(request: web.Request): - try: - response: Any = await handler(request) - return response - - except KeyError as err: - # NOTE: handles required request.match_info[*] or request.query[*] - _logger.debug(err, exc_info=True) - raise web.HTTPBadRequest(reason=f"Expected parameter {err}") from err - - except ValidationError as err: - # NOTE: pydantic.validate_arguments parses and validates -> ValidationError - _logger.debug(err, exc_info=True) - raise web.HTTPUnprocessableEntity( - text=json_dumps({"error": err.errors()}), - content_type="application/json", - ) from err - - except (InvalidParameterError, NoCommitError) as err: - raise web.HTTPUnprocessableEntity(reason=str(err)) from err - - except NotFoundError as err: - raise web.HTTPNotFound(reason=str(err)) from err - - except ProjectNotFoundError as err: - _logger.debug(err, exc_info=True) - raise web.HTTPNotFound( - reason=f"Project not found {err.project_uuid} or not accessible. Skipping snapshot" - ) from err - - return wrapped diff --git a/services/web/server/src/simcore_service_webserver/version_control/db.py b/services/web/server/src/simcore_service_webserver/version_control/db.py deleted file mode 100644 index ee884df6e9c..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/db.py +++ /dev/null @@ -1,551 +0,0 @@ -import json -import logging -from types import SimpleNamespace -from typing import Any, cast -from uuid import UUID - -import sqlalchemy as sa -from aiopg.sa import SAConnection -from aiopg.sa.result import RowProxy -from common_library.json_serialization import json_dumps -from models_library.basic_types import SHA1Str -from models_library.projects import ProjectIDStr -from pydantic.types import NonNegativeInt, PositiveInt -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.projects_version_control import ( - projects_vc_branches, - projects_vc_commits, - projects_vc_heads, - projects_vc_repos, - projects_vc_snapshots, - projects_vc_tags, -) -from simcore_postgres_database.utils_aiopg_orm import BaseOrm -from sqlalchemy.dialects.postgresql import insert as pg_insert - -from ..db.base_repository import BaseRepository -from ..projects.models import ProjectProxy -from .errors import ( - CleanRequiredError, - InvalidParameterError, - NoCommitError, - NotFoundError, -) -from .models import HEAD, CommitID, CommitLog, CommitProxy, RefID, RepoProxy, TagProxy -from .vc_changes import compute_workbench_checksum -from .vc_tags import parse_workcopy_project_tag_name - -_logger = logging.getLogger(__name__) - - -class VersionControlRepository(BaseRepository): - """ - db layer to access multiple tables within projects_version_control - """ - - class ReposOrm(BaseOrm[int]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_repos, - connection, - readonly={"id", "created", "modified"}, - ) - - class BranchesOrm(BaseOrm[int]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_branches, - connection, - readonly={"id", "created", "modified"}, - ) - - class CommitsOrm(BaseOrm[int]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_commits, - connection, - readonly={"id", "created", "modified"}, - # pylint: disable=no-member - writeonce={ - c for c in projects_vc_commits.columns.keys() if c != "message" - }, - ) - - class TagsOrm(BaseOrm[int]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_tags, - connection, - readonly={"id", "created", "modified"}, - ) - - class ProjectsOrm(BaseOrm[str]): - def __init__(self, connection: SAConnection): - super().__init__( - projects, - connection, - readonly={"id", "creation_date", "last_change_date"}, - writeonce={"uuid"}, - ) - - class SnapshotsOrm(BaseOrm[str]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_snapshots, - connection, - writeonce={"checksum"}, - ) - - class HeadsOrm(BaseOrm[int]): - def __init__(self, connection: SAConnection): - super().__init__( - projects_vc_heads, - connection, - writeonce={"repo_id"}, - ) - - # ------------ - - async def _get_head_branch( - self, repo_id: int, conn: SAConnection - ) -> RowProxy | None: - if h := await self.HeadsOrm(conn).fetch("head_branch_id", rowid=repo_id): - branch = ( - await self.BranchesOrm(conn) - .set_filter(id=h.head_branch_id) - .fetch("id name head_commit_id") - ) - return branch - return None - - async def _get_HEAD_commit( - self, repo_id: int, conn: SAConnection - ) -> CommitProxy | None: - if branch := await self._get_head_branch(repo_id, conn): - commit = ( - await self.CommitsOrm(conn).set_filter(id=branch.head_commit_id).fetch() - ) - return commit - return None - - async def _fetch_workcopy_project_id( - self, repo_id: int, commit_id: int, conn: SAConnection - ) -> ProjectIDStr: - # commit has a workcopy associated? - found = ( - await self.TagsOrm(conn).set_filter(commit_id=commit_id).fetch_all("name") - ) - for tag in found: - if workcopy_project_id := parse_workcopy_project_tag_name(tag.name): - return ProjectIDStr(workcopy_project_id) - - repo = await self.ReposOrm(conn).set_filter(id=repo_id).fetch("project_uuid") - assert repo # nosec - return cast(ProjectIDStr, repo.project_uuid) - - async def _update_state( - self, repo_id: int, conn: SAConnection - ) -> tuple[RepoProxy, CommitProxy | None, ProjectProxy]: - head_commit: CommitProxy | None = await self._get_HEAD_commit(repo_id, conn) - - # current repo - repo_orm = self.ReposOrm(conn).set_filter(id=repo_id) - returning_cols = "id project_uuid project_checksum modified" - repo = await repo_orm.fetch(returning_cols) - assert repo # nosec - - # fetch working copy - workcopy_project_id = await self._fetch_workcopy_project_id( - repo_id, head_commit.id if head_commit else -1, conn - ) - workcopy_project = ( - await self.ProjectsOrm(conn) - .set_filter(uuid=workcopy_project_id) - .fetch("last_change_date workbench ui uuid") - ) - assert workcopy_project # nosec - - # uses checksum cached in repo table to avoid re-computing checksum - checksum: SHA1Str | None = repo.project_checksum - if not checksum or ( - checksum and repo.modified < workcopy_project.last_change_date - ): - checksum = compute_workbench_checksum(workcopy_project.workbench) - - repo = await repo_orm.update(returning_cols, project_checksum=checksum) - assert repo - return repo, head_commit, workcopy_project - - @staticmethod - async def _upsert_snapshot( - project_checksum: str, - project: RowProxy | SimpleNamespace, - conn: SAConnection, - ): - # has changes wrt previous commit - assert project_checksum # nosec - insert_stmt = pg_insert(projects_vc_snapshots).values( - checksum=project_checksum, - content={ - "workbench": json.loads(json_dumps(project.workbench)), - "ui": json.loads(json_dumps(project.ui)), - }, - ) - upsert_snapshot = insert_stmt.on_conflict_do_update( - constraint=projects_vc_snapshots.primary_key, - set_=dict(content=insert_stmt.excluded.content), - ) - await conn.execute(upsert_snapshot) - - # PUBLIC - - async def list_repos( - self, - offset: NonNegativeInt = 0, - limit: PositiveInt | None = None, - ) -> tuple[list[RowProxy], NonNegativeInt]: - async with self.engine.acquire() as conn: - repo_orm = self.ReposOrm(conn) - - rows: list[RowProxy] - rows, total_count = await repo_orm.fetch_page( - "project_uuid", offset=offset, limit=limit - ) - - return rows, total_count - - async def get_repo_id(self, project_uuid: UUID) -> int | None: - async with self.engine.acquire() as conn: - repo_orm = self.ReposOrm(conn).set_filter(project_uuid=str(project_uuid)) - repo = await repo_orm.fetch("id") - return int(repo.id) if repo else None - - async def init_repo(self, project_uuid: UUID) -> int: - async with self.engine.acquire() as conn: - async with conn.begin(): - # create repo - repo_orm = self.ReposOrm(conn) - repo_id = await repo_orm.insert(project_uuid=str(project_uuid)) - assert repo_id is not None # nosec - assert isinstance(repo_id, int) # nosec - - repo = await repo_orm.fetch(rowid=repo_id) - assert repo # nosec - - # create main branch - branches_orm = self.BranchesOrm(conn) - branch_id = await branches_orm.insert(repo_id=repo.id) - assert branch_id is not None - assert isinstance(branch_id, int) # nosec - - main_branch: RowProxy | None = await branches_orm.fetch(rowid=branch_id) - assert main_branch # nosec - assert main_branch.name == "main" # nosec - - # assign head branch - heads_orm = self.HeadsOrm(conn) - await heads_orm.insert(repo_id=repo.id, head_branch_id=branch_id) - - return repo_id - - async def commit( - self, repo_id: int, tag: str | None = None, message: str | None = None - ) -> int: - """add changes, commits and tags (if tag is not None) - - Message is added to tag if set otherwise to commit - """ - if tag in ["HEAD", HEAD]: - raise InvalidParameterError(name="tag", reason="is a reserved word") - - async with self.engine.acquire() as conn: - # get head branch - branch = await self._get_head_branch(repo_id, conn) - if not branch: - raise NotImplementedError("Detached heads still not implemented") - - _logger.info("On branch %s", branch.name) - - # get head commit - repo, head_commit, workcopy_project = await self._update_state( - repo_id, conn - ) - - if head_commit is None: - previous_checksum = None - commit_id = None - else: - previous_checksum = head_commit.snapshot_checksum - commit_id = head_commit.id - - async with conn.begin(): - # take a snapshot if changes - if repo.project_checksum != previous_checksum: - await self._upsert_snapshot( - repo.project_checksum, workcopy_project, conn - ) - - # commit new snapshot in history - commit_id = await self.CommitsOrm(conn).insert( - repo_id=repo_id, - parent_commit_id=commit_id, - message=message, - snapshot_checksum=repo.project_checksum, - ) - assert commit_id # nosec - - # updates head/branch to this commit - await self.BranchesOrm(conn).set_filter(id=branch.id).update( - head_commit_id=commit_id - ) - - # tag it (again) - if tag: - insert_stmt = pg_insert(projects_vc_tags).values( - repo_id=repo_id, - commit_id=commit_id, - name=tag, - message=message, - hidden=False, - ) - upsert_tag = insert_stmt.on_conflict_do_update( - constraint="repo_tag_uniqueness", - set_=dict(name=insert_stmt.excluded.name), - ) - await conn.execute(upsert_tag) - else: - _logger.info("Nothing to commit, working tree clean") - - assert isinstance(commit_id, int) # nosec - return commit_id - - async def get_commit_log(self, commit_id: int) -> CommitLog: - async with self.engine.acquire() as conn: - commit = await self.CommitsOrm(conn).fetch(rowid=commit_id) - if commit: - assert isinstance(commit, RowProxy) # nosec - - tags: list[TagProxy] = ( - await self.TagsOrm(conn) - .set_filter(commit_id=commit.id, hidden=False) - .fetch_all("name message") - ) - return commit, tags - raise NotFoundError(name="commit", value=commit_id) - - async def log( - self, - repo_id: int, - offset: NonNegativeInt = 0, - limit: PositiveInt | None = None, - ) -> tuple[list[CommitLog], NonNegativeInt]: - async with self.engine.acquire() as conn: - commits_orm = self.CommitsOrm(conn).set_filter(repo_id=repo_id) - tags_orm = self.TagsOrm(conn) - - commits: list[CommitProxy] - commits, total_count = await commits_orm.fetch_page( - offset=offset, - limit=limit, - sort_by=sa.desc(commits_orm.columns["created"]), - ) - - logs = [] - for commit in commits: - tags: list[TagProxy] - tags = await tags_orm.set_filter(commit_id=commit.id).fetch_all() - logs.append((commit, tags)) - - return logs, total_count - - async def update_annotations( - self, - repo_id: int, - commit_id: CommitID, - message: str | None = None, - tag_name: str | None = None, - ): - async with self.engine.acquire() as conn: - async with conn.begin(): - if message: - await self.CommitsOrm(conn).set_filter(id=commit_id).update( - message=message - ) - - if tag_name: - tag = ( - await self.TagsOrm(conn) - .set_filter(repo_id=repo_id, commit_id=commit_id, hidden=False) - .fetch("id") - ) - - if tag: - await self.TagsOrm(conn).set_filter(rowid=tag.id).update( - name=tag_name - ) - - async def as_repo_and_commit_ids( - self, project_uuid: UUID, ref_id: RefID - ) -> tuple[int, CommitID]: - """Translates (project-uuid, ref-id) to (repo-id, commit-id) - - :return: tuple with repo and commit identifiers - """ - async with self.engine.acquire() as conn: - repo = ( - await self.ReposOrm(conn) - .set_filter(project_uuid=str(project_uuid)) - .fetch("id") - ) - commit_id = None - if repo: - if ref_id == HEAD: - commit = await self._get_HEAD_commit(repo.id, conn) - if commit: - commit_id = commit.id - elif isinstance(ref_id, CommitID): - commit_id = ref_id - else: - assert isinstance(ref_id, str) # nosec - # head branch or tag - raise NotImplementedError( - f"WIP: Tag or head branches as ref_id={ref_id}" - ) - - if not commit_id or not repo: - raise NotFoundError( - name="project {project_uuid} reference", value=ref_id - ) - - return repo.id, commit_id - - async def checkout(self, repo_id: int, commit_id: int) -> int: - """checks out working copy of project_uuid to commit ref_id - - :raises RuntimeError: if local copy has changes (i.e. dirty) - :return: commit id - :rtype: int - """ - async with self.engine.acquire() as conn: - repo, head_commit, workcopy_project = await self._update_state( - repo_id, conn - ) - - if head_commit is None: - raise NoCommitError( - details="Cannot checkout without commit changes first" - ) - - # check if working copy has changes, if so, fail - if repo.project_checksum != head_commit.snapshot_checksum: - raise CleanRequiredError( - details="Your local changes would be overwritten by checkout. " - "Cannot checkout without commit changes first." - ) - - # already in head commit - if head_commit.id == commit_id: - return commit_id - - async with conn.begin(): - commit = ( - await self.CommitsOrm(conn) - .set_filter(id=commit_id) - .fetch("snapshot_checksum") - ) - assert commit # nosec - - # restores project snapshot ONLY if main workcopy project - if workcopy_project.uuid == repo.project_uuid: - snapshot = ( - await self.SnapshotsOrm(conn) - .set_filter(commit.snapshot_checksum) - .fetch("content") - ) - assert snapshot # nosec - - await self.ProjectsOrm(conn).set_filter( - uuid=repo.project_uuid - ).update(**snapshot.content) - - # create detached branch that points to (repo_id, commit_id) - # upsert "detached" branch - insert_stmt = ( - pg_insert(projects_vc_branches) - .values( - repo_id=repo_id, - head_commit_id=commit_id, - name=f"{commit_id}-DETACHED", - ) - .returning(projects_vc_branches.c.id) - ) - upsert_tag = insert_stmt.on_conflict_do_update( - constraint="repo_branch_uniqueness", - set_=dict(head_commit_id=insert_stmt.excluded.head_commit_id), - ) - branch_id = await conn.scalar(upsert_tag) - - # updates head - await self.HeadsOrm(conn).set_filter(repo_id=repo_id).update( - head_branch_id=branch_id - ) - - return commit_id - - async def get_snapshot_content( - self, repo_id: int, commit_id: int - ) -> dict[str, Any]: - async with self.engine.acquire() as conn: - if ( - commit := await self.CommitsOrm(conn) - .set_filter(repo_id=repo_id, id=commit_id) - .fetch("snapshot_checksum") - ): - if ( - snapshot := await self.SnapshotsOrm(conn) - .set_filter(checksum=commit.snapshot_checksum) - .fetch("content") - ): - content: dict[str, Any] = snapshot.content - return content - - raise NotFoundError(name="snapshot for commit", value=(repo_id, commit_id)) - - async def get_workbench_view(self, repo_id: int, commit_id: int) -> dict[str, Any]: - async with self.engine.acquire() as conn: - if ( - commit := await self.CommitsOrm(conn) - .set_filter(repo_id=repo_id, id=commit_id) - .fetch("snapshot_checksum") - ): - repo = ( - await self.ReposOrm(conn) - .set_filter(id=repo_id) - .fetch("project_uuid") - ) - assert repo # nosec - - # if snapshot differs from workcopy, then show working copy - workcopy_project_id = await self._fetch_workcopy_project_id( - repo_id, commit_id, conn - ) - - # NOTE: For the moment, all wcopies except for the repo's main workcopy - # (i.e. repo.project_uuid) are READ-ONLY - if workcopy_project_id != repo.project_uuid: - if project := ( - await self.ProjectsOrm(conn) - .set_filter(uuid=workcopy_project_id) - .fetch("workbench ui") - ): - return dict(project.items()) - else: - if ( - snapshot := await self.SnapshotsOrm(conn) - .set_filter(checksum=commit.snapshot_checksum) - .fetch("content") - ): - assert isinstance(snapshot.content, dict) # nosec - return snapshot.content - - raise NotFoundError(name="snapshot for commit", value=(repo_id, commit_id)) diff --git a/services/web/server/src/simcore_service_webserver/version_control/errors.py b/services/web/server/src/simcore_service_webserver/version_control/errors.py deleted file mode 100644 index e0b3dd3ba63..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/errors.py +++ /dev/null @@ -1,29 +0,0 @@ -from ..errors import WebServerBaseError - - -class VersionControlValueError(WebServerBaseError, ValueError): - pass - - -class VersionControlRuntimeError(WebServerBaseError, RuntimeError): - pass - - -class NotFoundError(VersionControlValueError): - msg_template = "Could not find {name} '{value}'" - - -class InvalidParameterError(VersionControlValueError): - msg_template = "Invalid {name}: {reason}" - - -class NoCommitError(VersionControlRuntimeError): - msg_template = "No commit found: {details}" - - -class CleanRequiredError(VersionControlRuntimeError): - msg_template = "Working copy w/o changes (clean) is required: {details}" - - -class UserUndefinedError(VersionControlRuntimeError): - msg_template = "User required but undefined" diff --git a/services/web/server/src/simcore_service_webserver/version_control/models.py b/services/web/server/src/simcore_service_webserver/version_control/models.py deleted file mode 100644 index 505758d53d2..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/models.py +++ /dev/null @@ -1,110 +0,0 @@ -from datetime import datetime -from typing import Annotated, Any, TypeAlias, Union - -from aiopg.sa.result import RowProxy -from models_library.basic_types import SHA1Str -from models_library.projects import ProjectID -from models_library.projects_nodes import Node -from pydantic import ( - BaseModel, - ConfigDict, - Field, - PositiveInt, - StrictBool, - StrictFloat, - StrictInt, -) -from pydantic.networks import HttpUrl - -BuiltinTypes: TypeAlias = Union[StrictBool, StrictInt, StrictFloat, str] - -# alias for readability -# SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances - -BranchProxy: TypeAlias = RowProxy -CommitProxy: TypeAlias = RowProxy -RepoProxy: TypeAlias = RowProxy -TagProxy: TypeAlias = RowProxy -CommitLog: TypeAlias = tuple[CommitProxy, list[TagProxy]] - - -HEAD = f"{__file__}/ref/HEAD" - -CommitID: TypeAlias = int -BranchID: TypeAlias = int -RefID: TypeAlias = Annotated[CommitID | str, Field(union_mode="left_to_right")] - -CheckpointID: TypeAlias = PositiveInt - - -class Checkpoint(BaseModel): - id: CheckpointID - checksum: SHA1Str - created_at: datetime - tags: tuple[str, ...] - message: str | None = None - parents_ids: tuple[PositiveInt, ...] | None = Field(default=None) - - @classmethod - def from_commit_log(cls, commit: RowProxy, tags: list[RowProxy]) -> "Checkpoint": - return cls( - id=commit.id, - checksum=commit.snapshot_checksum, - tags=tuple(tag.name for tag in tags), - message=commit.message, - parents_ids=(commit.parent_commit_id,) if commit.parent_commit_id else None, - created_at=commit.created, - ) - - -class WorkbenchView(BaseModel): - """A view (i.e. read-only and visual) of the project's workbench""" - - model_config = ConfigDict(from_attributes=True) - - # NOTE: Tmp replacing UUIDS by str due to a problem serializing to json UUID keys - # in the response https://github.com/samuelcolvin/pydantic/issues/2096#issuecomment-814860206 - workbench: dict[str, Node] - ui: dict[str, Any] = {} - - -# API models --------------- - - -class RepoApiModel(BaseModel): - project_uuid: ProjectID - url: HttpUrl - - -class CheckpointApiModel(Checkpoint): - url: HttpUrl - - -class CheckpointNew(BaseModel): - tag: str - message: str | None = None - # new_branch: Optional[str] = None - - -class CheckpointAnnotations(BaseModel): - tag: str | None = None - message: str | None = None - - -class WorkbenchViewApiModel(WorkbenchView): - url: HttpUrl - checkpoint_url: HttpUrl - - -__all__: tuple[str, ...] = ( - "BranchID", - "BranchProxy", - "CheckpointID", - "CommitID", - "CommitLog", - "CommitProxy", - "HEAD", - "RefID", - "RepoProxy", - "TagProxy", -) diff --git a/services/web/server/src/simcore_service_webserver/version_control/plugin.py b/services/web/server/src/simcore_service_webserver/version_control/plugin.py deleted file mode 100644 index a1e31611f43..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/plugin.py +++ /dev/null @@ -1,29 +0,0 @@ -""" An add-on on projects module - - Adds version control to projects - -""" -import logging - -from aiohttp import web -from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup - -from .._constants import APP_SETTINGS_KEY -from . import _handlers - -_logger = logging.getLogger(__name__) - - -@app_module_setup( - __name__, - ModuleCategory.ADDON, - settings_name="WEBSERVER_VERSION_CONTROL", - depends=[ - "simcore_service_webserver.projects", - ], - logger=_logger, -) -def setup_version_control(app: web.Application): - assert app[APP_SETTINGS_KEY].WEBSERVER_VERSION_CONTROL # nosec - - app.add_routes(_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py b/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py deleted file mode 100644 index cc3559c118b..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py +++ /dev/null @@ -1,83 +0,0 @@ -""" - -- How to detect that a particular feature/characteristic in an entity has changed over time? - -Feature/charateristics of an entity at a given moment can be "snapshot" and given a hash value -- If the same feature at another moment results in a different hash value, it means that this feature -has changed - - -""" - -from typing import Any -from uuid import UUID, uuid3 - -from models_library.basic_types import SHA1Str -from models_library.projects import ProjectID, ProjectIDStr -from models_library.projects_nodes import Node - -from ..projects.models import ProjectProxy -from ..utils import compute_sha1_on_small_dataset - - -def compute_workbench_checksum(workbench: dict[str, Any]) -> SHA1Str: - # - # NOTE that UI is NOT accounted in the checksum - # - normalized = { - str(k): (Node(**v) if not isinstance(v, Node) else v) - for k, v in workbench.items() - } - - checksum = compute_sha1_on_small_dataset( - { - k: node.model_dump( - exclude_unset=True, - exclude_defaults=True, - exclude_none=True, - include={ - "key", - "version", - "inputs", - "input_nodes", - "outputs", - "output_nodes", - }, - ) - for k, node in normalized.items() - } - ) - return checksum - - -def _eval_checksum(repo, project: ProjectProxy) -> SHA1Str: - # cached checksum of project workcopy - checksum: SHA1Str | None = repo.project_checksum - is_invalid = not checksum or (checksum and repo.modified < project.last_change_date) - if is_invalid: - # invalid -> recompute - checksum = compute_workbench_checksum(project.workbench) - assert checksum # nosec - return checksum - - -def eval_workcopy_project_id( - repo_project_uuid: ProjectID | ProjectIDStr, snapshot_checksum: SHA1Str -) -> ProjectID: - """ - A working copy is a real project associated to a snapshot so it can be operated - as a project resource (e.g. run, save, etc). - - The uuid of the workcopy is a composition of the repo-project uuid and the snapshot-checksum - i.e. all identical snapshots (e.g. different iterations commits) map to the same project workcopy - can avoid re-run - - If a snapshot is identical but associated to two different repos, then it will still be - treated as a separate project to avoid colision between e.g. two users having coincidentaly the same - worbench blueprint. Nonetheless, this could be refined in the future since we could use this - knowledge to reuse results. - """ - if isinstance(repo_project_uuid, str): - repo_project_uuid = UUID(repo_project_uuid) - - return uuid3(repo_project_uuid, snapshot_checksum) diff --git a/services/web/server/src/simcore_service_webserver/version_control/vc_tags.py b/services/web/server/src/simcore_service_webserver/version_control/vc_tags.py deleted file mode 100644 index 5b9d86df791..00000000000 --- a/services/web/server/src/simcore_service_webserver/version_control/vc_tags.py +++ /dev/null @@ -1,15 +0,0 @@ -import re - -from models_library.basic_regex import UUID_RE_BASE -from models_library.projects import ProjectID - - -def compose_workcopy_project_tag_name(workcopy_project_id: ProjectID) -> str: - return f"project:{workcopy_project_id}" - - -def parse_workcopy_project_tag_name(name: str) -> ProjectID | None: - if m := re.match(rf"^project:(?P