List files would return everything if projects were empyt (#934)

Author: mguidon

services/storage/src/simcore_service_storage/data/docker-dev-config.yaml

@@ -3,7 +3,7 @@ main:
   host: 0.0.0.0
   log_level: INFO
   port: 8080
-  testing: True
+  testing: False
   max_workers: 8
   test_datcore:
      token_key: ${BF_API_KEY}

services/storage/src/simcore_service_storage/data/docker-prod-config.yaml

@@ -3,7 +3,7 @@ main:
   host: 0.0.0.0
   log_level: INFO
   port: 8080
-  testing: True
+  testing: False
   test_datcore:
      token_key: ${BF_API_KEY}
      token_secret: ${BF_API_SECRET}

services/storage/src/simcore_service_storage/data/host-dev-config.yaml

@@ -4,7 +4,7 @@ main:
   host: 127.0.0.1
   log_level: INFO
   port: 8080
-  testing: true
+  testing: False
   test_datcore:
      token_key: ${BF_API_KEY}
      token_secret: ${BF_API_SECRET}

services/storage/src/simcore_service_storage/dsm.py

@@ -43,7 +43,6 @@
 
 async def _setup_dsm(app: web.Application):
     cfg = app[APP_CONFIG_KEY]
-    main_cfg = cfg["main"]
 
     main_cfg = cfg["main"]
 
@@ -57,7 +56,8 @@ async def _setup_dsm(app: web.Application):
     s3_cfg = get_config_s3(app)
     bucket_name = s3_cfg["bucket_name"]
 
-    dsm = DataStorageManager(s3_client, engine, loop, pool, bucket_name)
+    testing = main_cfg["testing"]
+    dsm = DataStorageManager(s3_client, engine, loop, pool, bucket_name, not testing)
 
     app[APP_DSM_KEY] = dsm
 
@@ -112,6 +112,8 @@ class DataStorageManager:
     loop: object
     pool: ThreadPoolExecutor
     simcore_bucket_name: str
+    has_project_db: bool
+
     datcore_tokens: Dict[str, DatCoreApiToken]=attr.Factory(dict)
     # TODO: perhaps can be used a cache? add a lifetime?
 
@@ -185,25 +187,29 @@ async def list_files(self, user_id: str, location: str, uuid_filter: str ="", re
                     d = FileMetaData(**result_dict)
                     data.append(d)
 
-            uuid_name_dict = {}
-            # now parse the project to search for node/project names
-            try:
-                async with self.engine.acquire() as conn:
-                    joint_table = user_to_projects.join(projects)
-                    query = sa.select([projects]).select_from(joint_table)\
-                        .where(user_to_projects.c.user_id == user_id)
-
-                    async for row in conn.execute(query):
-                        proj_data = {key:value for key,value in row.items()}
-
-                        uuid_name_dict[proj_data["uuid"]] = proj_data["name"]
-                        wb = proj_data['workbench']
-                        for node in wb.keys():
-                            uuid_name_dict[node] = wb[node]['label']
-            except DBAPIError as _err:
-                logger.exception("Error querying database for project names")
+            if self.has_project_db:
+                uuid_name_dict = {}
+                # now parse the project to search for node/project names
+                try:
+                    async with self.engine.acquire() as conn:
+                        joint_table = user_to_projects.join(projects)
+                        query = sa.select([projects]).select_from(joint_table)\
+                            .where(user_to_projects.c.user_id == user_id)
+
+                        async for row in conn.execute(query):
+                            proj_data = {key:value for key,value in row.items()}
+
+                            uuid_name_dict[proj_data["uuid"]] = proj_data["name"]
+                            wb = proj_data['workbench']
+                            for node in wb.keys():
+                                uuid_name_dict[node] = wb[node]['label']
+                except DBAPIError as _err:
+                    logger.exception("Error querying database for project names")
+
+                if not uuid_name_dict:
+                    # there seems to be no project whatsoever for user_id
+                    return []
 
-            if uuid_name_dict:
                 # only keep files from non-deleted project --> This needs to be fixed
                 clean_data = []
                 for d in data:
@@ -228,16 +234,14 @@ async def list_files(self, user_id: str, location: str, uuid_filter: str ="", re
                             clean_data.append(d)
 
                 data = clean_data
-                for d in data:
-                    logger.info(d)
 
                 # same as above, make sure file is physically present on s3
                 clean_data = []
                 # MaG: This is inefficient: Do this automatically when file is modified
                 _loop = asyncio.get_event_loop()
                 session = aiobotocore.get_session(loop=_loop)
                 async with session.create_client('s3', endpoint_url="http://"+self.s3_client.endpoint, aws_access_key_id=self.s3_client.access_key,
-                     aws_secret_access_key=self.s3_client.secret_key) as client:
+                        aws_secret_access_key=self.s3_client.secret_key) as client:
                     responses = await asyncio.gather(*[client.list_objects_v2(Bucket=d.bucket_name, Prefix=_d) for _d in [__d.object_name for __d in data]])
                     for d, resp in zip(data, responses):
                         if 'Contents' in resp:

services/storage/tests/conftest.py

@@ -308,7 +308,7 @@ async def datcore_testbucket(loop, mock_files_factory):
 def dsm_fixture(s3_client, postgres_engine, loop):
     pool = ThreadPoolExecutor(3)
     dsm_fixture = DataStorageManager(
-        s3_client, postgres_engine, loop, pool, BUCKET_NAME)
+        s3_client, postgres_engine, loop, pool, BUCKET_NAME, False)
 
     api_token = os.environ.get("BF_API_KEY", "none")
     api_secret = os.environ.get("BF_API_SECRET", "none")

services/storage/tests/test_dsm.py

@@ -355,6 +355,7 @@ def test_fmd_build():
 async def test_dsm_complete_db(dsm_fixture, dsm_mockup_complete_db):
     dsm = dsm_fixture
     _id = "21"
+    dsm.has_project_db = True
     data = await dsm.list_files(user_id=_id, location=SIMCORE_S3_STR)
 
     assert len(data) == 2

services/storage/tests/test_rest.py

@@ -38,6 +38,7 @@ def client(loop, aiohttp_unused_port, aiohttp_client, postgres_service, minio_se
         'port': aiohttp_unused_port(),
         'host': 'localhost',
         "max_workers" : 4,
+        "testing" : True,
         "test_datcore" : { 'api_token' : api_token, 'api_secret' : api_secret}
     }
     rest_cfg = {