Merge pull request #960 from jverswijver/optimize_s3_test

Remove MC dependency.

Author: guzman-raphael

LNX-docker-compose.yml

@@ -77,8 +77,6 @@ services:
       - -c
       - |
         set -e
-        wget -P /home/dja/.local/bin/ https://dl.min.io/client/mc/release/linux-amd64/mc
-        chmod +x /home/dja/.local/bin/mc
         pip install --user -r test_requirements.txt
         pip install -e .
         pip freeze | grep datajoint

README.md

@@ -134,7 +134,7 @@ GID=1000
 * Launch local terminal
 * `export` environment variables in shell
 * Add entry in `/etc/hosts` for `127.0.0.1 fakeservices.datajoint.io`
-* Install [minio client(mc)](https://min.io/download#/linux) so that you can use it in terminal
+
 
 
 

local-docker-compose.yml

@@ -82,8 +82,6 @@ services:
       - -c
       - |
         set -e
-        wget -P /home/dja/.local/bin/ https://dl.min.io/client/mc/release/linux-amd64/mc
-        chmod +x /home/dja/.local/bin/mc
         pip install --user nose nose-cov coveralls flake8 ptvsd
         pip install -e .
         pip freeze | grep datajoint

tests/test_s3.py

@@ -1,12 +1,10 @@
 from . import S3_CONN_INFO
 from minio import Minio
-import json
 import urllib3
 import certifi
 from nose.tools import assert_true, raises
 from .schema_external import schema, SimpleRemote
 from datajoint.errors import DataJointError
-import os
 from datajoint.hash import uuid_from_buffer
 from datajoint.blob import pack
 
@@ -62,60 +60,6 @@ def test_connection_secure():
     def test_remove_object_exception():
         # https://github.com/datajoint/datajoint-python/issues/952
 
-        # Initialize minioClient with an endpoint and access/secret keys.
-        minio_client = Minio(
-            'minio:9000',
-            access_key='jeffjeff',
-            secret_key='jeffjeff',
-            secure=False)
-
-        # Create new user
-        os.system('mc admin user add myminio jeffjeff jeffjeff')
-        # json for test policy for permissionless user
-        testpolicy = {
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Action": [
-                        "s3:GetBucketLocation",
-                        "s3:ListBucket",
-                        "s3:ListBucketMultipartUploads",
-                        "s3:ListAllMyBuckets"
-                    ],
-                    "Effect": "Allow",
-                    "Resource": [
-                        "arn:aws:s3:::datajoint.test",
-                        "arn:aws:s3:::datajoint.migrate"
-                    ],
-                    "Sid": ""
-                },
-                {
-                    "Action": [
-                        "s3:GetObject",
-                        "s3:ListMultipartUploadParts"
-                    ],
-                    "Effect": "Allow",
-                    "Resource": [
-                        "arn:aws:s3:::datajoint.test/*",
-                        "arn:aws:s3:::datajoint.migrate/*"
-                    ],
-                    "Sid": ""
-                }
-            ]
-        }
-
-        # Write test json to tmp directory so we can use it to create a new user policy
-        with open('/tmp/policy.json', 'w') as f:
-            f.write(json.dumps(testpolicy))
-
-        # Add alias myminio
-        os.system(
-            'mc alias set myminio/ http://fakeservices.datajoint.io datajoint datajoint')
-
-        # Add the policy and apply it to the user
-        os.system('mc admin policy add myminio test /tmp/policy.json')
-        os.system('mc admin policy set myminio test user=jeffjeff')
-
         # Insert some test data and remove it so that the external table is populated
         test = [1, [1, 2, 3]]
         SimpleRemote.insert1(test)
@@ -124,20 +68,20 @@ def test_remove_object_exception():
         # Save the old external table minio client
         old_client = schema.external['share'].s3.client
 
-        # Apply our new minio client to the external table that has permissions restrictions
-        schema.external['share'].s3.client = minio_client
+        # Apply our new minio client which has a user that does not exist
+        schema.external['share'].s3.client = Minio(
+            'minio:9000',
+            access_key='jeffjeff',
+            secret_key='jeffjeff',
+            secure=False)
 
         # This method returns a list of errors
         error_list = schema.external['share'].delete(delete_external_files=True,
                                                      errors_as_string=False)
 
         # Teardown
-        os.system('mc admin policy remove myminio test')
-        os.system('mc admin user remove myminio jeffjeff')
         schema.external['share'].s3.client = old_client
         schema.external['share'].delete(delete_external_files=True)
-        os.remove("/tmp/policy.json")
-        os.system('mc alias remove myminio/')
 
         # Raise the error we want if the error matches the expected uuid
         if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))):