Merge pull request #673 from guzman-raphael/migrate_method

Add DJ011 blob migration utility function, Test improvements, and minor updates

Author: dimitri-yatsenko

LNX-docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - DJ_TEST_HOST=db
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
-      - S3_ENDPOINT=minio:9000
+      - S3_ENDPOINT=fakeminio.datajoint.io:9000
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
       - S3_BUCKET=datajoint-test
@@ -70,5 +70,26 @@ services:
       timeout: 5s
       retries: 60
       interval: 1s
+  fakeminio.datajoint.io:
+    <<: *net
+    image: nginx:alpine
+    environment:
+      - URL=datajoint.io
+      - SUBDOMAINS=fakeminio
+      - MINIO_SERVER=http://minio:9000
+    entrypoint: /entrypoint.sh
+    healthcheck:
+      test: wget --quiet --tries=1 --spider https://fakeminio.datajoint.io:443/minio/health/live || exit 1
+      timeout: 5s
+      retries: 300
+      interval: 1s
+    # ports:
+    #   - "9000:9000"
+    #   - "443:443"
+    volumes:
+      - ./tests/nginx/base.conf:/base.conf
+      - ./tests/nginx/entrypoint.sh:/entrypoint.sh
+      - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
+      - ./tests/nginx/privkey.pem:/certs/privkey.pem
 networks:
   main:

datajoint/__init__.py

@@ -41,5 +41,6 @@
 from .attribute_adapter import AttributeAdapter
 from . import errors
 from .errors import DataJointError
+from .migrate import migrate_dj011_external_blob_storage_to_dj012
 
 ERD = Di = Diagram   # Aliases for Diagram

datajoint/external.py

@@ -316,7 +316,7 @@ def delete(self, *, delete_external_files=None, limit=None, display_progress=Tru
             raise DataJointError("The delete_external_files argument must be set to either True or False in delete()")
 
         if not delete_external_files:
-            self.unused.delete_quick()
+            self.unused().delete_quick()
         else:
             items = self.unused().fetch_external_paths(limit=limit)
             if display_progress:

datajoint/fetch.py

@@ -50,7 +50,7 @@ def _get(connection, attr, data, squeeze, download_path):
     adapt = attr.adapter.get if attr.adapter else lambda x: x
 
     if attr.is_filepath:
-        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])
+        return str(adapt(extern.download_filepath(uuid.UUID(bytes=data))[0]))
 
     if attr.is_attachment:
         # Steps:
@@ -65,22 +65,22 @@ def _get(connection, attr, data, squeeze, download_path):
         if local_filepath.is_file():
             attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)
             if attachment_checksum == hash.uuid_from_file(local_filepath, init_string=attachment_name + '\0'):
-                return adapt(local_filepath)  # checksum passed, no need to download again
+                return str(adapt(local_filepath))  # checksum passed, no need to download again
             # generate the next available alias filename
             for n in itertools.count():
                 f = local_filepath.parent / (local_filepath.stem + '_%04x' % n + local_filepath.suffix)
                 if not f.is_file():
                     local_filepath = f
                     break
                 if attachment_checksum == hash.uuid_from_file(f, init_string=attachment_name + '\0'):
-                    return adapt(f)  # checksum passed, no need to download again
+                    return str(adapt(f))  # checksum passed, no need to download again
         # Save attachment
         if attr.is_external:
             extern.download_attachment(_uuid, attachment_name, local_filepath)
         else:
             # write from buffer
             safe_write(local_filepath, data.split(b"\0", 1)[1])
-        return adapt(local_filepath)  # download file from remote store
+        return str(adapt(local_filepath))  # download file from remote store
 
     return adapt(uuid.UUID(bytes=data) if attr.uuid else (
             blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze) 

datajoint/migrate.py

@@ -0,0 +1,167 @@
+import datajoint as dj
+from pathlib import Path
+import re
+from .utils import user_choice
+
+
+def migrate_dj011_external_blob_storage_to_dj012(migration_schema, store):
+    """
+    Utility function to migrate external blob data from 0.11 to 0.12.
+    :param migration_schema: string of target schema to be migrated
+    :param store: string of target dj.config['store'] to be migrated
+    """
+    if not isinstance(migration_schema, str):
+        raise ValueError(
+                'Expected type {} for migration_schema, not {}.'.format(
+                str, type(migration_schema)))
+
+    do_migration = False
+    do_migration = user_choice(
+            """
+Warning: Ensure the following are completed before proceeding.
+- Appropriate backups have been taken,
+- Any existing DJ 0.11.X connections are suspended, and
+- External config has been updated to new dj.config['stores'] structure.
+Proceed?
+            """, default='no') == 'yes'
+    if do_migration:
+        _migrate_dj011_blob(dj.schema(migration_schema), store)
+        print('Migration completed for schema: {}, store: {}.'.format(
+                migration_schema, store))
+        return
+    print('No migration performed.')
+
+
+def _migrate_dj011_blob(schema, default_store):
+    query = schema.connection.query
+
+    LEGACY_HASH_SIZE = 43
+
+    legacy_external = dj.FreeTable(
+        schema.connection,
+        '`{db}`.`~external`'.format(db=schema.database))
+
+    # get referencing tables
+    refs = query("""
+    SELECT concat('`', table_schema, '`.`', table_name, '`')
+            as referencing_table, column_name, constraint_name
+    FROM information_schema.key_column_usage
+    WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
+    """.format(
+        tab=legacy_external.table_name,
+        db=legacy_external.database), as_dict=True).fetchall()
+
+    for ref in refs:
+        # get comment
+        column = query(
+            'SHOW FULL COLUMNS FROM {referencing_table}'
+            'WHERE Field="{column_name}"'.format(
+                **ref), as_dict=True).fetchone()
+
+        store, comment = re.match(
+            r':external(-(?P<store>.+))?:(?P<comment>.*)',
+            column['Comment']).group('store', 'comment')
+
+        # get all the hashes from the reference
+        hashes = {x[0] for x in query(
+            'SELECT `{column_name}` FROM {referencing_table}'.format(
+                **ref))}
+
+        # sanity check make sure that store suffixes match
+        if store is None:
+            assert all(len(_) == LEGACY_HASH_SIZE for _ in hashes)
+        else:
+            assert all(_[LEGACY_HASH_SIZE:] == store for _ in hashes)
+
+        # create new-style external table
+        ext = schema.external[store or default_store]
+
+        # add the new-style reference field
+        temp_suffix = 'tempsub'
+
+        try:
+            query("""ALTER TABLE {referencing_table}
+                ADD COLUMN `{column_name}_{temp_suffix}` {type} DEFAULT NULL
+            COMMENT ":blob@{store}:{comment}"
+            """.format(
+                type=dj.declare.UUID_DATA_TYPE,
+                temp_suffix=temp_suffix,
+                store=(store or default_store), comment=comment, **ref))
+        except:
+            print('Column already added')
+            pass
+
+        # Copy references into the new external table
+        # No Windows! Backslashes will cause problems
+
+        contents_hash_function = {
+            'file': lambda ext, relative_path: dj.hash.uuid_from_file(
+                str(Path(ext.spec['location'], relative_path))),
+            's3': lambda ext, relative_path: dj.hash.uuid_from_buffer(
+                ext.s3.get(relative_path))
+        }
+
+        for _hash, size in zip(*legacy_external.fetch('hash', 'size')):
+            if _hash in hashes:
+                relative_path = str(Path(schema.database, _hash).as_posix())
+                uuid = dj.hash.uuid_from_buffer(init_string=relative_path)
+                external_path = ext._make_external_filepath(relative_path)
+                if ext.spec['protocol'] == 's3':
+                    contents_hash = dj.hash.uuid_from_buffer(ext._download_buffer(external_path))
+                else:
+                    contents_hash = dj.hash.uuid_from_file(external_path)
+                ext.insert1(dict(
+                    filepath=relative_path,
+                    size=size,
+                    contents_hash=contents_hash,
+                    hash=uuid
+                ), skip_duplicates=True)
+
+                query(
+                    'UPDATE {referencing_table} '
+                    'SET `{column_name}_{temp_suffix}`=%s '
+                    'WHERE `{column_name}` = "{_hash}"'
+                    .format(
+                        _hash=_hash,
+                        temp_suffix=temp_suffix, **ref), uuid.bytes)
+
+        # check that all have been copied
+        check = query(
+            'SELECT * FROM {referencing_table} '
+            'WHERE `{column_name}` IS NOT NULL'
+            '  AND `{column_name}_{temp_suffix}` IS NULL'
+            .format(temp_suffix=temp_suffix, **ref)).fetchall()
+
+        assert len(check) == 0, 'Some hashes havent been migrated'
+
+        # drop old foreign key, rename, and create new foreign key
+        query("""
+            ALTER TABLE {referencing_table}
+            DROP FOREIGN KEY `{constraint_name}`,
+            DROP COLUMN `{column_name}`,
+            CHANGE COLUMN `{column_name}_{temp_suffix}` `{column_name}`
+                {type} DEFAULT NULL
+                COMMENT ":blob@{store}:{comment}",
+            ADD FOREIGN KEY (`{column_name}`) REFERENCES {ext_table_name}
+                (`hash`)
+            """.format(
+                temp_suffix=temp_suffix,
+                ext_table_name=ext.full_table_name,
+                type=dj.declare.UUID_DATA_TYPE,
+                store=(store or default_store), comment=comment, **ref))
+
+    # Drop the old external table but make sure it's no longer referenced
+    # get referencing tables
+    refs = query("""
+    SELECT concat('`', table_schema, '`.`', table_name, '`') as
+        referencing_table, column_name, constraint_name
+    FROM information_schema.key_column_usage
+    WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
+    """.format(
+        tab=legacy_external.table_name,
+        db=legacy_external.database), as_dict=True).fetchall()
+
+    assert not refs, 'Some references still exist'
+
+    # drop old external table
+    legacy_external.drop_quick()

requirements.txt

@@ -4,7 +4,7 @@ pyparsing
 ipython
 pandas
 tqdm
-networkx
+networkx<2.4
 pydot
 minio
 matplotlib

tests/nginx/base.conf

@@ -0,0 +1,47 @@
+server {
+  listen 9000;
+  server_name {{SUBDOMAINS}}.{{URL}};
+  client_max_body_size 0;
+  proxy_buffering off;
+  ignore_invalid_headers off;
+  
+  location / {
+    access_log off;
+    proxy_http_version 1.1;
+    proxy_set_header Host $http_host;
+    proxy_pass {{MINIO_SERVER}};
+  }
+}
+
+server {
+  listen 443 ssl;
+  server_name {{SUBDOMAINS}}.{{URL}};
+  client_max_body_size 0;
+  proxy_buffering off;
+  ignore_invalid_headers off;
+
+  ssl_certificate /certs/fullchain.pem;
+  ssl_certificate_key /certs/privkey.pem;
+
+  # session settings
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:SSL:50m;
+  ssl_session_tickets off;
+
+  # protocols
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_prefer_server_ciphers on;
+  ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+
+  # OCSP Stapling
+  ssl_stapling on;
+  ssl_stapling_verify on;
+  resolver 127.0.0.11 valid=30s; # Docker DNS Server
+
+  location / {
+    access_log off;
+    proxy_http_version 1.1;
+    proxy_set_header Host $http_host;
+    proxy_pass {{MINIO_SERVER}};
+  }
+}

tests/nginx/entrypoint.sh

@@ -0,0 +1,5 @@
+#! /bin/sh
+sed "s|{{SUBDOMAINS}}|${SUBDOMAINS}|g" /base.conf | tee /etc/nginx/conf.d/base.conf
+sed -i "s|{{URL}}|${URL}|g" /etc/nginx/conf.d/base.conf
+sed -i "s|{{MINIO_SERVER}}|${MINIO_SERVER}|g" /etc/nginx/conf.d/base.conf
+nginx -g "daemon off;"

tests/nginx/fullchain.pem

@@ -0,0 +1,64 @@
+-----BEGIN CERTIFICATE-----
+MIIGZDCCBUygAwIBAgISA10k5JmyN2nyzqvMRyO2LntJMA0GCSqGSIb3DQEBCwUA
+MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
+ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xOTEwMjEyMTAwMjFaFw0y
+MDAxMTkyMTAwMjFaMCExHzAdBgNVBAMTFmZha2VtaW5pby5kYXRham9pbnQuaW8w
+ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDYwRR1YoZ1pJgZO+oU6zla
+47UiTnMO2KwYaS3VAPa1ks9nmQBInH7nA3i3wIzCiX+CeKCsiaKsjA5xlQmtmeM4
+FrL5U1ZBUyhroowRSluVyni9g3uJO/NG29BiWU+NBcwtsApbVUXRp4v9BQ2KgRZS
+KhK74XLXu1/6NRl3sjzye6MkfTo1rkzmm+pnFvBDkPEdI8/R7mBTQFXTSXzrqo+5
+ZNBY3sYWpGVrLOi+LRvFJR6kNs1z1cxOYGXQRMBFNMyu4xZAYDaowR+HVQ0OsQYw
+90PeuakMyB5qeIPe1zelfqP+/HO6L9MTZdLKBNm5YkJYrVm2YD5BcVCDJeUkAact
+DKW/AX2FL983D0WepM4mPm1MuRqpYVSEne3eeA4/Gm8TVpmqQyuW10HJbCsgZR9g
+X/gokz54uguHu7SZHvLuadoWzxMADFSvbOoM52rFgCsKeZecNDi9H54yAHlhjIj7
+Fs9zVRkELil5U2Fnolw8gOyfV/2ghqor8Y4950fcuy9DldcKeCmpjjGoemff/REL
+p4tgib+XAX/3bVmfgW4aTW1RoQ+duThfPovzumPXxffXNrRlstX7IaR/Asz0bhSJ
+C91vmemedgyExcUSuyqX2qzJrgdx5TCBpP+J47b5oHdjS9uWeg5BX7JpofiR/klP
+5OADP/F2a68aWgox7Z+CRQIDAQABo4ICazCCAmcwDgYDVR0PAQH/BAQDAgWgMB0G
+A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud
+DgQWBBSZJjFCP4IjST+FfsC54OKupBDWYTAfBgNVHSMEGDAWgBSoSmpjBH3duubR
+ObemRWXv86jsoTBvBggrBgEFBQcBAQRjMGEwLgYIKwYBBQUHMAGGImh0dHA6Ly9v
+Y3NwLmludC14My5sZXRzZW5jcnlwdC5vcmcwLwYIKwYBBQUHMAKGI2h0dHA6Ly9j
+ZXJ0LmludC14My5sZXRzZW5jcnlwdC5vcmcvMCEGA1UdEQQaMBiCFmZha2VtaW5p
+by5kYXRham9pbnQuaW8wTAYDVR0gBEUwQzAIBgZngQwBAgEwNwYLKwYBBAGC3xMB
+AQEwKDAmBggrBgEFBQcCARYaaHR0cDovL2Nwcy5sZXRzZW5jcnlwdC5vcmcwggEE
+BgorBgEEAdZ5AgQCBIH1BIHyAPAAdwCyHgXMi6LNiiBOh2b5K7mKJSBna9r6cOey
+SVMt74uQXgAAAW3wVdvZAAAEAwBIMEYCIQDeOafRS+nnooUWcFQlH82sK2lTrR3N
+uJqKJLIeoJdJVwIhAI+tVJJ103wbH6bC/ZwuRDlB/Omya0QwwO4m4Af4u/SEAHUA
+8JWkWfIA0YJAEC0vk4iOrUv+HUfjmeHQNKawqKqOsnMAAAFt8FXd3gAABAMARjBE
+AiAD8IITk6e1Ms01r2SUBUwaIwAA5z6NqYK8YBudhHRU6gIgBAzTx3OLwKo7aOjY
+8rf03Mcttz72VDI1dIDPt9vXxEcwDQYJKoZIhvcNAQELBQADggEBAFeAxIE70OgD
+1hx34hdJzfSOPUm3bjReUdfif6LTNYhEK1KjEKDwNK7r978t3wcOuxuJAwBdClzE
+dE/7EfuZilXWjVJ2La4J0DdQcrjt+O4bvFghNTWsOoYl5X0LzgKZLbl/9hvK8cE3
+/d3Pjf0zHflT0pJYjLP89ntwKJdFsAjFQc1+kX85SehYIj9c7t/W5/9MDhtebtvj
+Os1inUb4l15jbGTO3po8tPmmHLAvfTM6d/KIGueLHAn63EzCg1tmnQUjhhM1Zyzl
+Djdshrw0nr6BFOJvw/h/DYo6MqtLuTlrVjfdULjkqH5wq2wh7gqnGcQbqcI8Eixd
+SQbaP7xJreA=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+-----END CERTIFICATE-----