AB-357: Evaluate both MB database access methods to decide which one is more efficient

README.md

@@ -69,6 +69,83 @@ In order to load a psql session, use the following command:
 
     ./develop.sh psql
 
+### Setup the MusicBrainz Server
+
+MusicBrainz database containing all the MusicBrainz metadata is needed for
+setting up your application. The ``mbdump.tar.bz2`` is the core MusicBrainz
+archive which includes the tables for artist, release_group etc.
+The ``mbdump-derived.tar.bz2`` archive contains annotations, user tags and search indexes.
+These archives include all the data required for setting up an instance of
+AcousticBrainz.
+
+You can import the database dumps by downloading and importing the data in
+a single command:
+
+    docker-compose -f docker/docker-compose.dev.yml run musicbrainz_db
+
+**Note**
+
+One can also manually download the dumps and then import it:-
+
+  1. For this, you have to download the dumps ``mbdump.tar.bz2`` and ``mbdump-derived.tar.bz2``
+     from http://ftp.musicbrainz.org/pub/musicbrainz/data/fullexport/.
+
+       **Warning**
+
+       Make sure to get the latest dumps
+
+  2. Then the environment variable ``DUMPS_DIR`` must be set to the path of the
+     folders containing the dumps. This can be done by:
+
+         export DUMPS_DIR="Path of the folder containing the dumps"
+
+     You can check that the variable ``DUMPS_DIR`` has been succesfully assigned or not by:
+
+         echo $DUMPS_DIR
+
+      This must display the path of your folder containing the database dumps. The folder must contain at least the file    ``mbdump.tar.bz2``.
+
+  3. Then import the database dumps by this command:
+
+         docker-compose -f docker/docker-compose.dev.yml run -v $DUMPS_DIR:/home/musicbrainz/dumps \
+         -v $PWD/data/mbdata:/var/lib/postgresql/data/pgdata musicbrainz_db
+
+**Note**
+
+  You can also use the smaller sample dumps available at http://ftp.musicbrainz.org/pub/musicbrainz/data/sample/
+  to set up the MusicBrainz database. However, note that these dumps are .tar.xz
+  dumps while AcousticBrainz currently only supports import of .tar.bz2 dumps.
+  So, a decompression of the sample dumps and recompression into .tar.bz2 dumps
+  will be needed. This can be done using the following command:
+
+     xzcat mbdump-sample.tar.xz | bzip2 > mbdump.tar.bz2
+
+**Warning**
+
+Keep in mind that this process is very time consuming, so make sure that you don't delete the ``data/mbdata`` directory accidently. Also make sure that you have about 25GB of free space to keep the MusicBrainz data.
+
+Initialization of AcousticBrainz database is also required:
+
+    ./develop.sh run --rm  webserver python2 manage.py init_db
+
+Then you can start all the services:
+
+    ./develop.sh up --build
+
+## Initialize the MusicBrainz database:
+
+    ./develop.sh run --rm  webserver python2 manage.py init_mb_db
+
+## Import the MusicBrainz database in AcousticBrainz database:
+
+    ./develop.sh run --rm  webserver python2 manage.py import_musicbrainz_db
+
+### Manually
+
+Full installation instructions are available in [INSTALL.md](https://github.com/metabrainz/acousticbrainz-server/blob/master/INSTALL.md) file. After installing, continue the following steps.
+
+## Configuration and development
+
 ### Building static files
 
 We use webpack as our JavaScript/CSS build system.

admin/sql/create_musicbrainz_foreign_keys.sql

@@ -0,0 +1,175 @@
+BEGIN;
+
+ALTER TABLE musicbrainz.artist
+  ADD CONSTRAINT artist_fk_type
+  FOREIGN KEY (type)
+  REFERENCES musicbrainz.artist_type(id);
+
+ALTER TABLE musicbrainz.artist
+  ADD CONSTRAINT artist_fk_area
+  FOREIGN KEY (area)
+  REFERENCES musicbrainz.area(id);
+
+ALTER TABLE musicbrainz.artist
+  ADD CONSTRAINT artist_fk_gender
+  FOREIGN KEY (gender)
+  REFERENCES musicbrainz.gender(id);
+
+ALTER TABLE musicbrainz.artist
+  ADD CONSTRAINT artist_fk_begin_area
+  FOREIGN KEY (begin_area)
+  REFERENCES musicbrainz.area(id);
+
+ALTER TABLE musicbrainz.artist
+  ADD CONSTRAINT artist_fk_end_area
+  FOREIGN KEY (end_area)
+  REFERENCES musicbrainz.area(id);
+
+ALTER TABLE musicbrainz.artist_credit_name
+  ADD CONSTRAINT artist_credit_name_fk_artist_credit
+  FOREIGN KEY (artist_credit)
+  REFERENCES musicbrainz.artist_credit(id)
+  ON DELETE CASCADE;
+
+ALTER TABLE musicbrainz.artist_credit_name
+  ADD CONSTRAINT artist_credit_name_fk_artist
+  FOREIGN KEY (artist)
+  REFERENCES musicbrainz.artist(id)
+  ON DELETE CASCADE;
+
+ALTER TABLE musicbrainz.artist_gid_redirect
+  ADD CONSTRAINT artist_gid_redirect_fk_new_id
+  FOREIGN KEY (new_id)
+  REFERENCES musicbrainz.artist(id);
+
+ALTER TABLE musicbrainz.area
+  ADD CONSTRAINT area_fk_type
+  FOREIGN KEY (type)
+  REFERENCES musicbrainz.area_type(id);
+
+ALTER TABLE musicbrainz.area_type
+  ADD CONSTRAINT area_type_fk_parent
+  FOREIGN KEY (parent)
+REFERENCES musicbrainz.area_type(id);
+
+ALTER TABLE musicbrainz.recording
+  ADD CONSTRAINT recording_fk_artist_credit
+  FOREIGN KEY (artist_credit)
+  REFERENCES musicbrainz.artist_credit(id);
+
+ALTER TABLE musicbrainz.recording_gid_redirect
+  ADD CONSTRAINT recording_gid_redirect_fk_new_id
+  FOREIGN KEY (new_id)
+  REFERENCES musicbrainz.recording(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_artist_credit
+  FOREIGN KEY (artist_credit)
+  REFERENCES musicbrainz.artist_credit(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_release_group
+  FOREIGN KEY (release_group)
+  REFERENCES musicbrainz.release_group(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_status
+  FOREIGN KEY (status)
+  REFERENCES musicbrainz.release_status(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_packaging
+  FOREIGN KEY (packaging)
+  REFERENCES musicbrainz.release_packaging(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_language
+  FOREIGN KEY (language)
+  REFERENCES musicbrainz.language(id);
+
+ALTER TABLE musicbrainz.release
+  ADD CONSTRAINT release_fk_script
+  FOREIGN KEY (script)
+  REFERENCES musicbrainz.script(id);
+
+ALTER TABLE musicbrainz.release_gid_redirect
+  ADD CONSTRAINT release_gid_redirect_fk_new_id
+  FOREIGN KEY (new_id)
+  REFERENCES musicbrainz.release(id);
+
+ALTER TABLE musicbrainz.track
+  ADD CONSTRAINT track_fk_recording
+  FOREIGN KEY (recording)
+  REFERENCES musicbrainz.recording(id);
+
+ALTER TABLE musicbrainz.track
+  ADD CONSTRAINT track_fk_medium
+  FOREIGN KEY (medium)
+  REFERENCES musicbrainz.medium(id);
+
+ALTER TABLE musicbrainz.track
+  ADD CONSTRAINT track_fk_artist_credit
+  FOREIGN KEY (artist_credit)
+  REFERENCES musicbrainz.artist_credit(id);
+
+ALTER TABLE musicbrainz.track_gid_redirect
+  ADD CONSTRAINT track_gid_redirect_fk_new_id
+  FOREIGN KEY (new_id)
+  REFERENCES musicbrainz.track(id);
+
+ALTER TABLE musicbrainz.release_group
+  ADD CONSTRAINT release_group_fk_artist_credit
+  FOREIGN KEY (artist_credit)
+  REFERENCES musicbrainz.artist_credit(id);
+
+ALTER TABLE musicbrainz.release_group
+  ADD CONSTRAINT release_group_fk_type
+  FOREIGN KEY (type)
+  REFERENCES musicbrainz.release_group_primary_type(id);
+
+ALTER TABLE musicbrainz.release_group_primary_type
+  ADD CONSTRAINT release_group_primary_type_fk_parent
+  FOREIGN KEY (parent)
+  REFERENCES musicbrainz.release_group_primary_type;
+
+ALTER TABLE musicbrainz.release_group_gid_redirect
+  ADD CONSTRAINT release_group_gid_redirect_fk_new_id
+  FOREIGN KEY (new_id)
+  REFERENCES musicbrainz.release_group(id);
+
+ALTER TABLE musicbrainz.medium
+  ADD CONSTRAINT medium_fk_release
+  FOREIGN KEY (release)
+  REFERENCES musicbrainz.release(id);
+
+ALTER TABLE musicbrainz.medium
+  ADD CONSTRAINT medium_fk_format
+  FOREIGN KEY (format)
+  REFERENCES musicbrainz.medium_format(id);
+
+ALTER TABLE musicbrainz.medium_format
+  ADD CONSTRAINT medium_format_fk_parent
+  FOREIGN KEY (parent)
+REFERENCES musicbrainz.medium_format(id);
+
+ALTER TABLE musicbrainz.release_status
+  ADD CONSTRAINT release_status_fk_parent
+  FOREIGN KEY (parent)
+  REFERENCES musicbrainz.release_status(id);
+
+ALTER TABLE musicbrainz.release_packaging
+  ADD CONSTRAINT release_packaging_fk_parent
+  FOREIGN KEY (parent)
+  REFERENCES musicbrainz.release_packaging(id);
+
+ALTER TABLE musicbrainz.gender
+  ADD CONSTRAINT gender_fk_parent
+  FOREIGN KEY (parent)
+  REFERENCES musicbrainz.gender(id);
+
+ALTER TABLE musicbrainz.artist_type
+  ADD CONSTRAINT artist_type_fk_parent
+  FOREIGN KEY (parent)
+  REFERENCES musicbrainz.artist_type(id);
+
+COMMIT;

admin/sql/create_musicbrainz_indexes.sql

@@ -0,0 +1,67 @@
+BEGIN;
+
+CREATE UNIQUE INDEX artist_idx_gid ON musicbrainz.artist (gid);
+CREATE INDEX artist_idx_name ON musicbrainz.artist (name);
+CREATE INDEX artist_idx_sort_name ON musicbrainz.artist (sort_name);
+CREATE INDEX artist_idx_area ON musicbrainz.artist (area);
+CREATE INDEX artist_idx_begin_area ON musicbrainz.artist (begin_area);
+CREATE INDEX artist_idx_end_area ON musicbrainz.artist (end_area);
+
+CREATE UNIQUE INDEX artist_idx_null_comment ON musicbrainz.artist (name) WHERE comment IS NULL;
+CREATE UNIQUE INDEX artist_idx_uniq_name_comment ON musicbrainz.artist (name, comment) WHERE comment IS NOT NULL;
+
+CREATE UNIQUE INDEX area_type_idx_gid ON musicbrainz.area_type (gid);
+
+CREATE UNIQUE INDEX area_idx_gid ON musicbrainz.area (gid);
+CREATE INDEX area_idx_name ON musicbrainz.area (name);
+
+CREATE INDEX artist_credit_name_idx_artist ON musicbrainz.artist_credit_name (artist);
+
+CREATE UNIQUE INDEX recording_idx_gid ON musicbrainz.recording (gid);
+CREATE INDEX recording_idx_name ON musicbrainz.recording (name);
+CREATE INDEX recording_idx_artist_credit ON musicbrainz.recording (artist_credit);
+
+CREATE UNIQUE INDEX release_idx_gid ON musicbrainz.release (gid);
+CREATE INDEX release_idx_name ON musicbrainz.release (name);
+CREATE INDEX release_idx_release_group ON musicbrainz.release (release_group);
+CREATE INDEX release_idx_artist_credit ON musicbrainz.release (artist_credit);
+
+CREATE UNIQUE INDEX track_idx_gid ON musicbrainz.track (gid);
+CREATE INDEX track_idx_recording ON musicbrainz.track (recording);
+CREATE INDEX track_idx_name ON musicbrainz.track (name);
+CREATE INDEX track_idx_artist_credit ON musicbrainz.track (artist_credit);
+
+CREATE INDEX artist_gid_redirect_idx_new_id ON musicbrainz.artist_gid_redirect (new_id);
+
+CREATE INDEX recording_gid_redirect_idx_new_id ON musicbrainz.recording_gid_redirect (new_id);
+
+CREATE INDEX release_gid_redirect_idx_new_id ON musicbrainz.release_gid_redirect (new_id);
+
+CREATE INDEX release_group_gid_redirect_idx_new_id ON musicbrainz.release_group_gid_redirect (new_id);
+
+CREATE INDEX track_gid_redirect_idx_new_id ON musicbrainz.track_gid_redirect (new_id);
+
+CREATE UNIQUE INDEX release_group_idx_gid ON musicbrainz.release_group (gid);
+CREATE INDEX release_group_idx_name ON musicbrainz.release_group (name);
+CREATE INDEX release_group_idx_artist_credit ON musicbrainz.release_group (artist_credit);
+
+CREATE INDEX medium_idx_track_count ON musicbrainz.medium (track_count);
+
+CREATE UNIQUE INDEX medium_format_idx_gid ON musicbrainz.medium_format (gid);
+
+CREATE UNIQUE INDEX release_status_idx_gid ON musicbrainz.release_status (gid);
+
+CREATE UNIQUE INDEX language_idx_iso_code_2b ON musicbrainz.language (iso_code_2b);
+CREATE UNIQUE INDEX language_idx_iso_code_2t ON musicbrainz.language (iso_code_2t);
+CREATE UNIQUE INDEX language_idx_iso_code_1 ON musicbrainz.language (iso_code_1);
+CREATE UNIQUE INDEX language_idx_iso_code_3 ON musicbrainz.language (iso_code_3);
+
+CREATE UNIQUE INDEX release_packaging_idx_gid ON musicbrainz.release_packaging (gid);
+
+CREATE UNIQUE INDEX script_idx_iso_code ON musicbrainz.script (iso_code);
+
+CREATE UNIQUE INDEX gender_idx_gid ON musicbrainz.gender (gid);
+
+CREATE UNIQUE INDEX artist_type_idx_gid ON musicbrainz.artist_type (gid);
+
+COMMIT;

admin/sql/create_musicbrainz_primary_keys.sql

@@ -0,0 +1,27 @@
+BEGIN;
+
+ALTER TABLE musicbrainz.artist ADD CONSTRAINT artist_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.artist_credit ADD CONSTRAINT artist_credit_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.artist_credit_name ADD CONSTRAINT artist_credit_name_pkey PRIMARY KEY (artist_credit, position);
+ALTER TABLE musicbrainz.artist_gid_redirect ADD CONSTRAINT artist_gid_redirect_pkey PRIMARY KEY (gid);
+ALTER TABLE musicbrainz.area ADD CONSTRAINT area_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.area_type ADD CONSTRAINT area_type_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.recording ADD CONSTRAINT recording_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.recording_gid_redirect ADD CONSTRAINT recording_gid_redirect_pkey PRIMARY KEY (gid);
+ALTER TABLE musicbrainz.release ADD CONSTRAINT release_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.release_gid_redirect ADD CONSTRAINT release_gid_redirect_pkey PRIMARY KEY (gid);
+ALTER TABLE musicbrainz.track ADD CONSTRAINT track_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.track_gid_redirect ADD CONSTRAINT track_gid_redirect_pkey PRIMARY KEY (gid);
+ALTER TABLE musicbrainz.release_group ADD CONSTRAINT release_group_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.release_group_gid_redirect ADD CONSTRAINT release_group_gid_redirect_pkey PRIMARY KEY (gid);
+ALTER TABLE musicbrainz.medium ADD CONSTRAINT medium_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.medium_format ADD CONSTRAINT medium_format_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.release_status ADD CONSTRAINT release_status_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.release_group_primary_type ADD CONSTRAINT release_group_primary_type_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.language ADD CONSTRAINT language_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.release_packaging ADD CONSTRAINT release_packaging_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.script ADD CONSTRAINT script_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.gender ADD CONSTRAINT gender_pkey PRIMARY KEY (id);
+ALTER TABLE musicbrainz.artist_type ADD CONSTRAINT artist_type_pkey PRIMARY KEY (id);
+
+COMMIT;

admin/sql/create_musicbrainz_schema.sql

@@ -0,0 +1,7 @@
+-- Create the musicbrainz schema.
+
+BEGIN;
+
+CREATE SCHEMA IF NOT EXISTS musicbrainz;
+
+COMMIT;