Skip to content

Commit fae22eb

Browse files
bisgaard-itisbisgaard-itis
committed
Merge branch 'master' into fix-celery-abort-test
2 parents 8e2be0d + 1aecd75 commit fae22eb

File tree

13 files changed

+177
-55
lines changed

13 files changed

+177
-55
lines changed

‎services/static-webserver/client/source/class/osparc/dashboard/CardBase.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -526,12 +526,13 @@ qx.Class.define("osparc.dashboard.CardBase", {
526526
});
527527

528528
if (resourceData["resourceType"] === "study" || resourceData["resourceType"] === "template") {
529-
osparc.store.Services.getStudyServices(this.getResourceData()["uuid"])
529+
osparc.store.Services.getStudyServices(resourceData.uuid)
530530
.then(resp => {
531531
const services = resp["services"];
532532
resourceData["services"] = services;
533533
this.setServices(services);
534-
});
534+
})
535+
.catch(err => console.error(err));
535536

536537
osparc.study.Utils.guessIcon(resourceData)
537538
.then(iconSource => this.setIcon(iconSource));
@@ -664,14 +665,21 @@ qx.Class.define("osparc.dashboard.CardBase", {
664665
}
665666

666667
// Block card
667-
const unaccessibleServices = osparc.study.Utils.getCantExecuteServices(services);
668-
if (unaccessibleServices.length) {
668+
const cantReadServices = osparc.study.Utils.getCantExecuteServices(services);
669+
let inaccessibleServices = [];
670+
if (this.isResourceType("study") || this.isResourceType("template")) {
671+
inaccessibleServices = osparc.store.Services.getInaccessibleServices(this.getResourceData()["workbench"]);
672+
}
673+
if (cantReadServices.length || inaccessibleServices.length) {
669674
this.setBlocked("UNKNOWN_SERVICES");
670675
const image = "@FontAwesome5Solid/ban/";
671-
let toolTipText = this.tr("Unaccessible service(s):");
672-
unaccessibleServices.forEach(unSrv => {
676+
let toolTipText = this.tr("Inaccessible service(s):");
677+
cantReadServices.forEach(unSrv => {
673678
toolTipText += "<br>" + unSrv.key + ":" + osparc.service.Utils.extractVersionDisplay(unSrv.release);
674679
});
680+
inaccessibleServices.forEach(unSrv => {
681+
toolTipText += "<br>" + unSrv.key + ":" + unSrv.version;
682+
});
675683
this.__showBlockedCard(image, toolTipText);
676684
}
677685

‎services/static-webserver/client/source/class/osparc/dashboard/ResourceDetails.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,10 @@ qx.Class.define("osparc.dashboard.ResourceDetails", {
4949
case "study":
5050
case "template": {
5151
osparc.store.Services.getStudyServicesMetadata(latestResourceData)
52-
.then(() => {
52+
.finally(() => {
5353
this.__resourceModel = new osparc.data.model.Study(latestResourceData);
5454
this.__resourceModel["resourceType"] = resourceData["resourceType"];
55+
this.__resourceData["services"] = resourceData["services"];
5556
this.__addPages();
5657
})
5758
break;

‎services/static-webserver/client/source/class/osparc/data/model/Workbench.js‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -272,12 +272,12 @@ qx.Class.define("osparc.data.model.Workbench", {
272272
},
273273

274274
__createNode: function(study, metadata, uuid) {
275-
osparc.utils.Utils.localCache.serviceToFavs(metadata.key);
276275
const node = new osparc.data.model.Node(study, metadata, uuid);
277276
node.addListener("keyChanged", () => this.fireEvent("reloadModel"), this);
278277
node.addListener("changeInputNodes", () => this.fireDataEvent("pipelineChanged"), this);
279278
node.addListener("reloadModel", () => this.fireEvent("reloadModel"), this);
280279
node.addListener("updateStudyDocument", () => this.fireEvent("updateStudyDocument"), this);
280+
osparc.utils.Utils.localCache.serviceToFavs(metadata.key);
281281
return node;
282282
},
283283

@@ -685,15 +685,20 @@ qx.Class.define("osparc.data.model.Workbench", {
685685

686686
__deserializeNodes: function(workbenchData, workbenchUIData = {}) {
687687
const nodeIds = Object.keys(workbenchData);
688-
689-
const metadataPromises = [];
688+
const serviceMetadataPromises = [];
690689
nodeIds.forEach(nodeId => {
691690
const nodeData = workbenchData[nodeId];
692-
metadataPromises.push(osparc.store.Services.getService(nodeData.key, nodeData.version));
691+
serviceMetadataPromises.push(osparc.store.Services.getService(nodeData.key, nodeData.version));
693692
});
694-
695-
return Promise.all(metadataPromises)
696-
.then(values => {
693+
return Promise.allSettled(serviceMetadataPromises)
694+
.then(results => {
695+
const missing = results.filter(result => result.status === "rejected" || result.value === null)
696+
if (missing.length) {
697+
const errorMsg = qx.locale.Manager.tr("Service metadata missing");
698+
osparc.FlashMessenger.logError(errorMsg);
699+
return;
700+
}
701+
const values = results.map(result => result.value);
697702
// Create first all the nodes
698703
for (let i=0; i<nodeIds.length; i++) {
699704
const metadata = values[i];

‎services/static-webserver/client/source/class/osparc/desktop/MainPageHandler.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,8 @@ qx.Class.define("osparc.desktop.MainPageHandler", {
102102
throw new Error(msg);
103103
}
104104

105-
// check if it's corrupt
106-
if (osparc.study.Utils.isCorrupt(studyData)) {
105+
// check if there is any linked node missing
106+
if (osparc.study.Utils.isAnyLinkedNodeMissing(studyData)) {
107107
const msg = `${qx.locale.Manager.tr("We encountered an issue with the")} ${studyAlias} <br>${qx.locale.Manager.tr("Please contact support.")}`;
108108
throw new Error(msg);
109109
}
@@ -113,10 +113,10 @@ qx.Class.define("osparc.desktop.MainPageHandler", {
113113

114114
osparc.store.Services.getStudyServicesMetadata(studyData)
115115
.finally(() => {
116-
const inaccessibleServices = osparc.store.Services.getInaccessibleServices(studyData["workbench"])
116+
const inaccessibleServices = osparc.store.Services.getInaccessibleServices(studyData["workbench"]);
117117
if (inaccessibleServices.length) {
118118
const msg = osparc.store.Services.getInaccessibleServicesMsg(inaccessibleServices, studyData["workbench"]);
119-
osparc.FlashMessenger.getInstance().logError(msg);
119+
osparc.FlashMessenger.logError(msg);
120120
this.showDashboard();
121121
return;
122122
}

‎services/static-webserver/client/source/class/osparc/store/Services.js‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ qx.Class.define("osparc.store.Services", {
4040
this.__addExtraTypeInfos(servicesObj);
4141

4242
Object.values(servicesObj).forEach(serviceKey => {
43-
Object.values(serviceKey).forEach(service => this.__addToCache(service));
43+
Object.values(serviceKey).forEach(service => this.__addServiceToCache(service));
4444
});
4545

4646
resolve(servicesObj);
@@ -110,7 +110,10 @@ qx.Class.define("osparc.store.Services", {
110110
if (
111111
useCache &&
112112
this.__isInCache(key, version) &&
113-
"history" in this.__servicesCached[key][version]
113+
(
114+
this.__servicesCached[key][version] === null ||
115+
"history" in this.__servicesCached[key][version]
116+
)
114117
) {
115118
resolve(this.__servicesCached[key][version]);
116119
return;
@@ -127,17 +130,16 @@ qx.Class.define("osparc.store.Services", {
127130
this.__addHit(service);
128131
this.__addTSRInfo(service);
129132
this.__addExtraTypeInfo(service);
130-
this.__addToCache(service)
133+
this.__addServiceToCache(service);
134+
delete this.__servicesPromisesCached[key][version];
131135
resolve(service);
132136
})
133137
.catch(err => {
134138
// store it in cache to avoid asking again
135-
this.__servicesCached[key][version] = null;
139+
this.__addToCache(key, version, null);
140+
delete this.__servicesPromisesCached[key][version];
136141
console.error(err);
137142
reject();
138-
})
139-
.finally(() => {
140-
delete this.__servicesPromisesCached[key][version];
141143
});
142144
});
143145
},
@@ -282,23 +284,27 @@ qx.Class.define("osparc.store.Services", {
282284
wbServices.forEach(srv => {
283285
promises.push(this.getService(srv["key"], srv["version"]));
284286
});
285-
return Promise.all(promises);
287+
return Promise.allSettled(promises);
286288
},
287289

288290
getInaccessibleServices: function(workbench) {
289291
const allServices = this.__servicesCached;
290-
const unaccessibleServices = [];
292+
const inaccessibleServices = [];
291293
const wbServices = osparc.study.Utils.extractUniqueServices(workbench);
292294
wbServices.forEach(srv => {
293-
if (srv.key in allServices && srv.version in allServices[srv.key]) {
295+
if (
296+
srv.key in allServices &&
297+
srv.version in allServices[srv.key] &&
298+
allServices[srv.key][srv.version] // check metadata is not null
299+
) {
294300
return;
295301
}
296-
const idx = unaccessibleServices.findIndex(unSrv => unSrv.key === srv.key && unSrv.version === srv.version);
302+
const idx = inaccessibleServices.findIndex(unSrv => unSrv.key === srv.key && unSrv.version === srv.version);
297303
if (idx === -1) {
298-
unaccessibleServices.push(srv);
304+
inaccessibleServices.push(srv);
299305
}
300306
});
301-
return unaccessibleServices;
307+
return inaccessibleServices;
302308
},
303309

304310
getInaccessibleServicesMsg: function(inaccessibleServices, workbench) {
@@ -340,13 +346,17 @@ qx.Class.define("osparc.store.Services", {
340346
return this.getLatest("simcore/services/frontend/iterator-consumer/probe/"+type);
341347
},
342348

343-
__addToCache: function(service) {
349+
__addServiceToCache: function(service) {
344350
const key = service.key;
345351
const version = service.version;
352+
this.__addToCache(key, version, service);
353+
},
354+
355+
__addToCache: function(key, version, value) {
346356
if (!(key in this.__servicesCached)) {
347357
this.__servicesCached[key] = {};
348358
}
349-
this.__servicesCached[key][version] = service;
359+
this.__servicesCached[key][version] = value;
350360
},
351361

352362
__isInCache: function(key, version) {

‎services/static-webserver/client/source/class/osparc/store/Store.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,14 @@ qx.Class.define("osparc.store.Store", {
226226
check: "Array",
227227
init: []
228228
},
229+
organizationMembers: {
230+
check: "Array",
231+
init: null,
232+
},
233+
notifications: {
234+
check: "Array",
235+
init: null,
236+
},
229237
},
230238

231239
events: {

‎services/static-webserver/client/source/class/osparc/study/Utils.js‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,13 @@ qx.Class.define("osparc.study.Utils", {
2323
type: "static",
2424

2525
statics: {
26-
__isAnyLinkedNodeMissing: function(studyData) {
26+
isAnyLinkedNodeMissing: function(studyData) {
2727
const existingNodeIds = Object.keys(studyData["workbench"]);
2828
const linkedNodeIds = osparc.data.model.Workbench.getLinkedNodeIds(studyData["workbench"]);
2929
const allExist = linkedNodeIds.every(linkedNodeId => existingNodeIds.includes(linkedNodeId));
3030
return !allExist;
3131
},
3232

33-
isCorrupt: function(studyData) {
34-
return this.__isAnyLinkedNodeMissing(studyData);
35-
},
36-
3733
extractUniqueServices: function(workbench) {
3834
const services = new Set([]);
3935
Object.values(workbench).forEach(srv => {
@@ -274,8 +270,9 @@ qx.Class.define("osparc.study.Utils", {
274270

275271
__getBlockedState: function(studyData) {
276272
if (studyData["services"]) {
277-
const unaccessibleServices = osparc.study.Utils.getCantExecuteServices(studyData["services"])
278-
if (unaccessibleServices.length) {
273+
const cantReadServices = osparc.study.Utils.getCantExecuteServices(studyData["services"]);
274+
const inaccessibleServices = osparc.store.Services.getInaccessibleServices(studyData["workbench"]);
275+
if (cantReadServices.length || inaccessibleServices.length) {
279276
return "UNKNOWN_SERVICES";
280277
}
281278
}
@@ -358,7 +355,7 @@ qx.Class.define("osparc.study.Utils", {
358355
const wbService = wbServices[0];
359356
osparc.store.Services.getService(wbService.key, wbService.version)
360357
.then(serviceMetadata => {
361-
if (serviceMetadata["icon"]) {
358+
if (serviceMetadata && serviceMetadata["icon"]) {
362359
resolve(serviceMetadata["icon"]);
363360
}
364361
resolve(defaultIcon);

‎services/storage/src/simcore_service_storage/exceptions/errors.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
from common_library.errors_classes import OsparcErrorMixin
22

33

4-
class StorageRuntimeError(OsparcErrorMixin, RuntimeError):
5-
...
4+
class StorageRuntimeError(OsparcErrorMixin, RuntimeError): ...
65

76

87
class ConfigurationError(StorageRuntimeError):
@@ -45,3 +44,7 @@ class InvalidFileIdentifierError(AccessLayerError):
4544

4645
class DatCoreCredentialsMissingError(StorageRuntimeError):
4746
msg_template: str = "DatCore credentials are incomplete. TIP: Check your settings"
47+
48+
49+
class SelectionNotAllowedError(StorageRuntimeError):
50+
msg_template: str = "Selection='{selection}' must be from the same folder"

‎services/storage/src/simcore_service_storage/simcore_s3_dsm.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
LinkAlreadyExistsError,
6262
ProjectAccessRightError,
6363
ProjectNotFoundError,
64+
SelectionNotAllowedError,
6465
)
6566
from .models import (
6667
DatasetMetaData,
@@ -81,9 +82,11 @@
8182
from .modules.s3 import get_s3_client
8283
from .utils.s3_utils import S3TransferDataCB
8384
from .utils.simcore_s3_dsm_utils import (
85+
UserSelectionStr,
8486
compute_file_id_prefix,
8587
create_and_upload_export,
8688
create_random_export_name,
89+
ensure_user_selection_from_same_base_directory,
8790
expand_directory,
8891
get_accessible_project_ids,
8992
get_directory_file_id,
@@ -1249,7 +1252,11 @@ async def create_s3_export(
12491252
*,
12501253
progress_bar: ProgressBarData,
12511254
) -> StorageFileID:
1252-
source_object_keys: set[StorageFileID] = set()
1255+
source_object_keys: set[tuple[UserSelectionStr, StorageFileID]] = set()
1256+
1257+
# ensure all selected items have the same parent
1258+
if not ensure_user_selection_from_same_base_directory(object_keys):
1259+
raise SelectionNotAllowedError(selection=object_keys)
12531260

12541261
# check access rights
12551262
for object_key in object_keys:
@@ -1279,7 +1286,7 @@ async def create_s3_export(
12791286
self.simcore_bucket_name, object_key
12801287
):
12811288
for entry in meta_data_files:
1282-
source_object_keys.add(entry.object_key)
1289+
source_object_keys.add((object_key, entry.object_key))
12831290

12841291
_logger.debug(
12851292
"User selection '%s' includes '%s' files",

‎services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
from contextlib import suppress
22
from pathlib import Path
3+
from typing import TypeAlias
34
from uuid import uuid4
45

56
import orjson
67
from aws_library.s3 import S3MetaData, SimcoreS3API
78
from aws_library.s3._constants import STREAM_READER_CHUNK_SIZE
9+
from aws_library.s3._models import S3ObjectKey
810
from models_library.api_schemas_storage.storage_schemas import S3BucketName
911
from models_library.projects import ProjectID
1012
from models_library.projects_nodes_io import (
@@ -143,20 +145,40 @@ def create_random_export_name(user_id: UserID) -> StorageFileID:
143145
)
144146

145147

148+
def ensure_user_selection_from_same_base_directory(
149+
object_keys: list[S3ObjectKey],
150+
) -> bool:
151+
parents = [Path(x).parent for x in object_keys]
152+
return len(set(parents)) <= 1
153+
154+
155+
UserSelectionStr: TypeAlias = str
156+
157+
158+
def _base_path_parent(base_path: UserSelectionStr, s3_object: S3ObjectKey) -> str:
159+
base_path_parent_path = Path(base_path).parent
160+
s3_object_path = Path(s3_object)
161+
if base_path_parent_path == s3_object_path:
162+
return s3_object_path.name
163+
164+
result = s3_object_path.relative_to(base_path_parent_path)
165+
return f"{result}"
166+
167+
146168
async def create_and_upload_export(
147169
s3_client: SimcoreS3API,
148170
bucket: S3BucketName,
149171
*,
150-
source_object_keys: set[StorageFileID],
172+
source_object_keys: set[tuple[UserSelectionStr, StorageFileID]],
151173
destination_object_keys: StorageFileID,
152174
progress_bar: ProgressBarData,
153175
) -> None:
154176
archive_entries: ArchiveEntries = [
155177
(
156-
s3_object,
178+
_base_path_parent(selection, s3_object),
157179
await s3_client.get_bytes_streamer_from_object(bucket, s3_object),
158180
)
159-
for s3_object in source_object_keys
181+
for (selection, s3_object) in source_object_keys
160182
]
161183

162184
async with progress_bar:

0 commit comments

Comments
 (0)