Prepare for 4.2.0 release (#239)

* Add script to perform LVM snapshot-based backup

* Add grants files for new ispyb expeye roles

* Add entry for new version 4.2.0 in HISTORY.rst

* Bump version to 4.2.0

* Refresh schema files to v4.2.0

Author: KarlLevik

HISTORY.rst

@@ -5,6 +5,24 @@ History
 Unreleased / master
 -------------------
 
+4.2.0 (2024-08-14)
+-------------------
+
+Roles and permissions:  grants files for new ispyb expeye roles
+
+New or modified tables, columns, lookup values:
+
+- New table ``ProcessedTomogram``
+- New rows for 'VMXm-Cartridge' and 'VMXm-Cartridge' in the ``ContainerType`` table
+- New FK column ``ownerId`` (references ``Person.personId``) in the ``BLSampleGroup`` table
+- New 'Metal ID' enum item in ``DiffractionPlan.experimentKind``
+- New 'Metal ID' enum item in ``DataCollectionGroup.experimentType``
+- New row for 'Metal ID' (for MX) in ``ExperimentType`` table
+
+Stored procedures: ``upsert_dc_group_v3`` now populates ``DataCollectionGroup.experimentTypeId`` based on ``p_experimenttype``
+
+New script to perform LVM snapshot-based backup
+
 4.1.0 (2024-03-26)
 -------------------
 

bin/mdb_lvm_backup.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/bash
+
+# Fail on unset variables
+set -u
+
+#
+# Assumptions - adapt as needed to your environment:
+#
+# - /mnt/snapshot/ exists
+# - The user running the script has relevant password-less sudo permissions
+# - The volume group is called VolGroup00
+# - The source logical volume name is called varlibmysql
+# - The snapshot logical volume name is called mariadb-backup
+
+# Arguments:
+#
+# 1: Path to the final destination directory
+#
+
+export DEST_DIR="${1:?arg not defined}"
+export LVM_SUPPRESS_FD_WARNINGS=1
+
+# Make sure the destination directory exists and has a unique subdir
+export DEST_DIR_SUB="${DEST_DIR:?not defined}"/$(date +%Y-%m-%d_%H%M%S-%A)
+mkdir "${DEST_DIR_SUB:?not defined}"/
+
+# The following lines are run from inside the mariadb client. The reason is
+# that the MariaDB "BACKUP STAGE" commands need to all run inside the same
+# connection, and the BACKUP STAGE END command needs to run after the lvcreate
+# command.
+
+# shellcheck disable=SC2016
+mariadb -e '
+-- Ensure that all non-transactional tables are properly flushed to disk before
+-- the snapshot is done. Using BACKUP STAGE commands is also more efficient than
+-- using the FLUSH TABLES WITH READ LOCK command as this set of commands
+-- will not block or be blocked by write operations to transactional tables.
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+
+-- When using the LVM backup to set up a replication slave, we need to
+-- know the current replication position, so write this to a text file
+-- in case we need it later:
+system echo $(mariadb -Nse "SELECT @@global.gtid_binlog_pos") > ${DEST_DIR_SUB}/gtid_binlog_pos.txt
+
+-- Create the actual snapshot
+system sudo /usr/sbin/lvcreate -l100%FREE -s -n mariadb-backup /dev/VolGroup00/varlibmysql
+
+-- Complete the backup stage
+BACKUP STAGE END;'
+
+# Copy the snapshot to other server/disk
+sudo mount /dev/VolGroup00/mariadb-backup /mnt/snapshot -o nouuid -t xfs
+sudo cp -pr /mnt/snapshot "${DEST_DIR_SUB:?not defined}"/.
+
+# Dispose of the snapshot
+sudo umount /mnt/snapshot
+sudo /usr/sbin/lvremove --yes /dev/VolGroup00/mariadb-backup

grants/ispyb_expeye.sql

@@ -0,0 +1,18 @@
+-- Create the Exp-eye Core API role.
+CREATE ROLE IF NOT EXISTS ispyb_expeye_core_role;
+
+-- You must also create a database user and grant this role to them, e.g.
+-- CREATE USER ispyb_expeye_core@'%' IDENTIFIED BY 'the_password';
+-- GRANT ispyb_expeye_core_role to ispyb_expeye_core@'%';
+-- SET DEFAULT ROLE ispyb_expeye_core_role FOR ispyb_expeye_core@'%';
+
+GRANT ispyb_propagation TO ispyb_expeye_core_role;
+
+CREATE ROLE ispyb_expeye_acquisition_role;
+
+GRANT SELECT, INSERT, UPDATE ON RobotAction TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON GridInfo TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON DataCollection TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON DataCollectionGroup TO ispyb_expeye_acquisition_role;
+GRANT SELECT, INSERT, UPDATE ON Position TO ispyb_expeye_acquisition_role;
+GRANT ispyb_expeye_acquisition_role TO ispyb_expeye_core_role;

grants/ispyb_expeye_samples_grants.sql

@@ -0,0 +1,52 @@
+-- Create the Exp-eye Samples API role.
+CREATE ROLE IF NOT EXISTS ispyb_expeye_samples_role;
+
+-- You must also create a database user and grant this role to them, e.g.
+-- CREATE USER ispyb_expeye_samples@'%' IDENTIFIED BY 'the_password';
+-- GRANT ispyb_expeye_samples_role to ispyb_expeye_samples@'%';
+-- SET DEFAULT ROLE ispyb_expeye_samples_role FOR ispyb_expeye_core@'%';
+
+GRANT SELECT ON Proposal TO ispyb_expeye_samples_role;
+GRANT SELECT ON BLSession TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Shipping TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ShippingHasSession TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Project_has_Shipping TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON LabContact TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Dewar TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarLocation TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarLocationList TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarRegistry TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarRegistry_has_Proposal TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON DewarReport TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE ON DewarTransportHistory TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON Container TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerInspection TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerQueue TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerQueueSample TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerRegistry TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerRegistry_has_Proposal TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerReport TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ContainerType TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE ON ContainerHistory TO ispyb_expeye_samples_role;
+
+GRANT SELECT ON ProcessingPipeline TO ispyb_expeye_samples_role;
+GRANT SELECT ON ProcessingPipelineCategory TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSample TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSampleGroup TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSampleGroup_has_BLSample TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON BLSubSample TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Project_has_BLSample TO ispyb_expeye_samples_role;
+
+GRANT SELECT ON Protein TO ispyb_expeye_samples_role;
+GRANT SELECT ON Crystal TO ispyb_expeye_samples_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON DiffractionPlan TO ispyb_expeye_samples_role;
+
+GRANT SELECT ON ComponentType TO ispyb_expeye_samples_role;
+GRANT SELECT ON ComponentSubType TO ispyb_expeye_samples_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON Component_has_SubType TO ispyb_expeye_samples_role;

schemas/ispyb/lookups.sql

@@ -10,7 +10,7 @@
 /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
 
 /*!40000 ALTER TABLE `AdminVar` DISABLE KEYS */;
-INSERT INTO `AdminVar` (`varId`, `name`, `value`) VALUES (4,'schemaVersion','4.1.0');
+INSERT INTO `AdminVar` (`varId`, `name`, `value`) VALUES (4,'schemaVersion','4.2.0');
 /*!40000 ALTER TABLE `AdminVar` ENABLE KEYS */;
 
 /*!40000 ALTER TABLE `SchemaStatus` DISABLE KEYS */;
@@ -256,7 +256,13 @@ INSERT INTO `SchemaStatus` (`schemaStatusId`, `scriptName`, `schemaStatus`, `rec
 (285,'2024_03_20_Shipping_Dewar_externalIds.sql','DONE','2024-03-26 16:40:34'),
 (286,'2024_03_26_AdminVar_bump_version.sql','DONE','2024-03-26 16:40:34'),
 (287,'2024_03_26_AutoProc_index_refined_unit_cell.sql','DONE','2024-03-26 16:40:34'),
-(288,'2023_11_16_AdminVar_bump_version.sql','DONE','2024-03-26 16:47:25');
+(288,'2023_11_16_AdminVar_bump_version.sql','DONE','2024-03-26 16:47:25'),
+(289,'2024_05_14_DataCollectionGroup_experimentType.sql','DONE','2024-08-14 11:23:41'),
+(290,'2024_05_30_DiffractionPlan_experimentKind.sql','DONE','2024-08-14 11:23:41'),
+(291,'2024_07_08_BLSampleGroup_ownerId.sql','DONE','2024-08-14 11:23:41'),
+(292,'2024_07_16_ContainerType_insert_new.sql','DONE','2024-08-14 11:23:41'),
+(293,'2024_08_08_ProcessedTomogram.sql','DONE','2024-08-14 11:23:41'),
+(294,'2024_08_14_AdminVar_bump_version.sql','DONE','2024-08-14 11:23:41');
 /*!40000 ALTER TABLE `SchemaStatus` ENABLE KEYS */;
 
 /*!40000 ALTER TABLE `ComponentType` DISABLE KEYS */;
@@ -325,7 +331,9 @@ INSERT INTO `ContainerType` (`containerTypeId`, `name`, `proposalType`, `active`
 (32,'Box','xpdf',1,25,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),
 (33,'Puck-22','xpdf',1,22,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),
 (34,'I22_Grid_81','saxs',1,81,9,1,1,1,1,0,0,-1),
-(35,'I22_Capillary_Rack_25','saxs',1,25,25,1,1,1,1,0,0,-1);
+(35,'I22_Capillary_Rack_25','saxs',1,25,25,1,1,1,1,0,0,-1),
+(36,'VMXm-Cartridge','mx',1,5,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),
+(37,'VMXm-GridBox','mx',1,4,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
 /*!40000 ALTER TABLE `ContainerType` ENABLE KEYS */;
 
 /*!40000 ALTER TABLE `ExperimentType` DISABLE KEYS */;
@@ -370,7 +378,8 @@ INSERT INTO `ExperimentType` (`experimentTypeId`, `name`, `proposalType`, `activ
 (39,'Serial Jet','mx',1),
 (40,'Still','mx',1),
 (41,'SSX-Chip','mx',1),
-(42,'SSX-Jet','mx',1);
+(42,'SSX-Jet','mx',1),
+(43,'Metal ID','mx',1);
 /*!40000 ALTER TABLE `ExperimentType` ENABLE KEYS */;
 
 /*!40000 ALTER TABLE `InspectionType` DISABLE KEYS */;

schemas/ispyb/routines.sql

@@ -130,7 +130,7 @@ DELIMITER ;
 /*!50003 SET character_set_results = utf8mb3 */ ;
 /*!50003 SET collation_connection  = utf8mb3_general_ci */ ;
 DELIMITER ;;
-CREATE FUNCTION `retrieve_proposal_title`(p_proposal_code varchar(5), p_proposal_number int) RETURNS varchar(255) CHARSET latin1 COLLATE latin1_swedish_ci
+CREATE FUNCTION `retrieve_proposal_title`(p_proposal_code varchar(5), p_proposal_number int) RETURNS varchar(255) CHARSET latin1
     READS SQL DATA
 BEGIN
 	DECLARE ret_title varchar(255);
@@ -155,7 +155,7 @@ DELIMITER ;
 /*!50003 SET character_set_results = utf8mb3 */ ;
 /*!50003 SET collation_connection  = utf8mb3_general_ci */ ;
 DELIMITER ;;
-CREATE FUNCTION `retrieve_proposal_title_v2`(p_proposalCode varchar(5), p_proposalNumber int) RETURNS varchar(255) CHARSET latin1 COLLATE latin1_swedish_ci
+CREATE FUNCTION `retrieve_proposal_title_v2`(p_proposalCode varchar(5), p_proposalNumber int) RETURNS varchar(255) CHARSET latin1
     READS SQL DATA
     COMMENT 'Retrieve the title for a given proposal code and number.'
 BEGIN
@@ -206,7 +206,7 @@ DELIMITER ;
 /*!50003 SET character_set_results = utf8mb3 */ ;
 /*!50003 SET collation_connection  = utf8mb3_general_ci */ ;
 DELIMITER ;;
-CREATE FUNCTION `root_replace`(p_str varchar(255), p_oldroot varchar(255), p_newroot varchar(255)) RETURNS varchar(255) CHARSET latin1 COLLATE latin1_swedish_ci
+CREATE FUNCTION `root_replace`(p_str varchar(255), p_oldroot varchar(255), p_newroot varchar(255)) RETURNS varchar(255) CHARSET latin1
     COMMENT 'Returns a varchar where the old root p_oldroot (the leftmost part) of p_str has been replaced with a new root p_newroot'
 BEGIN
  DECLARE path_len smallint unsigned DEFAULT LENGTH(p_oldroot);
@@ -9448,7 +9448,7 @@ CREATE PROCEDURE `upsert_dc_group_v3`(
      p_sessionNumber int(10),
      p_sampleId int(10) unsigned,
      p_sampleBarcode varchar(45),
-     p_experimenttype varchar(45),
+     p_experimenttype varchar(45), 
      p_starttime datetime,
      p_endtime datetime,
      p_crystalClass varchar(20),
@@ -9459,53 +9459,57 @@ CREATE PROCEDURE `upsert_dc_group_v3`(
      p_actualContainerSlotInSC integer(10),
      p_comments varchar(1024),
      p_xtalSnapshotFullPath	varchar(255),
-		 p_scanParameters longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin
+     p_scanParameters longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin
      )
     MODIFIES SQL DATA
     COMMENT 'Inserts or updates info about data collection group (p_id).\nMandatory columns:\nFor insert: Either p_sessionId or a valid session described by (p_proposalCode, p_proposalNumber, p_sessionNumber)\nFor update: p_id\nNote: In order to associate the data collection group with a sample, one of the following sets of parameters are required:\n* p_sampleId\n* p_proposalCode, p_proposalNumber, p_sessionNumber + p_sampleBarcode\n* p_actualContainerBarcode + p_actualSampleSlotInContainer\nReturns: Record ID in p_id.'
 BEGIN
-	DECLARE row_proposal_id int(10) unsigned DEFAULT NULL;
-	DECLARE row_sample_id int(10) unsigned DEFAULT NULL;
+  DECLARE row_proposal_id int(10) unsigned DEFAULT NULL;
+  DECLARE row_sample_id int(10) unsigned DEFAULT NULL;
+  DECLARE row_experiment_id int(10) unsigned DEFAULT NULL;
 
-	IF p_sessionId IS NULL AND p_proposalCode IS NOT NULL AND p_proposalNumber IS NOT NULL AND p_sessionNumber IS NOT NULL THEN
+  IF p_sessionId IS NULL AND p_proposalCode IS NOT NULL AND p_proposalNumber IS NOT NULL AND p_sessionNumber IS NOT NULL THEN
       SELECT max(bs.sessionid), p.proposalId INTO p_sessionId, row_proposal_id
       FROM Proposal p INNER JOIN BLSession bs ON p.proposalid = bs.proposalid
       WHERE p.proposalCode = p_proposalCode AND p.proposalNumber = p_proposalNumber AND bs.visit_number = p_sessionNumber;
 	END IF;
 
 	IF p_id IS NOT NULL OR p_sessionId IS NOT NULL THEN
-
+	  
       IF p_sessionId IS NOT NULL AND p_sampleId IS NULL AND p_sampleBarcode IS NOT NULL THEN
-	    IF row_proposal_id IS NULL THEN
-          SELECT proposalId INTO row_proposal_id
-          FROM BLSession
-          WHERE sessionId = p_sessionId;
+			    IF row_proposal_id IS NULL THEN
+		          SELECT proposalId INTO row_proposal_id
+		          FROM BLSession
+		          WHERE sessionId = p_sessionId;
+			    END IF;
+	        SELECT max(bls.blSampleId) INTO p_sampleId
+	        FROM BLSample bls
+					  INNER JOIN Container c on c.containerId = bls.containerId
+			          INNER JOIN Dewar d on d.dewarId = c.dewarId
+			          INNER JOIN Shipping s on s.shippingId = d.shippingId
+			    WHERE bls.code = p_sampleBarcode AND s.proposalId = row_proposal_id;
 	    END IF;
-        SELECT max(bls.blSampleId) INTO p_sampleId
-        FROM BLSample bls
-		  INNER JOIN Container c on c.containerId = bls.containerId
-          INNER JOIN Dewar d on d.dewarId = c.dewarId
-          INNER JOIN Shipping s on s.shippingId = d.shippingId
-        WHERE bls.code = p_sampleBarcode AND s.proposalId = row_proposal_id;
-	  END IF;
 
-	  IF p_sampleId IS NULL AND (p_actualContainerBarcode IS NOT NULL) AND (p_actualSampleSlotInContainer IS NOT NULL) THEN
-	    SELECT max(bls.blSampleId) INTO p_sampleId
-        FROM BLSample bls
-          INNER JOIN Container c on c.containerId = bls.containerId
-	    WHERE c.barcode = p_actualContainerBarcode AND bls.location = p_actualSampleSlotInContainer;
-      END IF;
+		  IF p_sampleId IS NULL AND (p_actualContainerBarcode IS NOT NULL) AND (p_actualSampleSlotInContainer IS NOT NULL) THEN
+		    SELECT max(bls.blSampleId) INTO p_sampleId
+	        FROM BLSample bls
+	          INNER JOIN Container c on c.containerId = bls.containerId
+		    WHERE c.barcode = p_actualContainerBarcode AND bls.location = p_actualSampleSlotInContainer;
+	    END IF;
 
-      INSERT INTO DataCollectionGroup (datacollectionGroupId, sessionId, blsampleId, experimenttype, starttime, endtime,
+			SELECT experimentTypeId INTO row_experiment_id FROM ExperimentType WHERE name = p_experimenttype;
+
+      INSERT INTO DataCollectionGroup (datacollectionGroupId, sessionId, blsampleId, experimenttype, experimentTypeId, starttime, endtime,
         crystalClass, detectorMode, actualSampleBarcode, actualSampleSlotInContainer, actualContainerBarcode, actualContainerSlotInSC,
         comments, xtalSnapshotFullPath, scanParameters)
-        VALUES (p_id, p_sessionId, p_sampleId, p_experimenttype, p_starttime, p_endtime, p_crystalClass, p_detectorMode,
+        VALUES (p_id, p_sessionId, p_sampleId, p_experimenttype, row_experiment_id, p_starttime, p_endtime, p_crystalClass, p_detectorMode,
         p_actualSampleBarcode, p_actualSampleSlotInContainer, p_actualContainerBarcode, p_actualContainerSlotInSC,
         p_comments, p_xtalSnapshotFullPath, p_scanParameters)
 	    ON DUPLICATE KEY UPDATE
 		  sessionId = IFNULL(p_sessionId, sessionId),
           blsampleId = IFNULL(p_sampleId, blsampleId),
           experimenttype = IFNULL(p_experimenttype, experimenttype),
+					experimentTypeId = row_experiment_id,
           starttime = IFNULL(p_starttime, starttime),
           endtime = IFNULL(p_endtime, endtime),
           crystalClass = IFNULL(p_crystalClass, crystalClass),
@@ -9518,13 +9522,13 @@ BEGIN
           xtalSnapshotFullPath = IFNULL(p_xtalSnapshotFullPath, xtalSnapshotFullPath),
 					scanParameters = IFNULL (p_scanParameters, scanParameters);
 
-	    IF p_id IS NULL THEN
-		    SET p_id = LAST_INSERT_ID();
+      IF p_id IS NULL THEN
+          SET p_id = LAST_INSERT_ID();
       END IF;
-    ELSE
-      SIGNAL SQLSTATE '45000' SET MYSQL_ERRNO=1644, MESSAGE_TEXT='Mandatory argument(s) are NULL: p_id OR p_sessionId OR a valid session described by (p_proposalCode and p_proposalNumber and p_sessionNumber) must be non-NULL.';
-    END IF;
-  END ;;
+  ELSE
+    SIGNAL SQLSTATE '45000' SET MYSQL_ERRNO=1644, MESSAGE_TEXT='Mandatory argument(s) are NULL: p_id OR p_sessionId OR a valid session described by (p_proposalCode and p_proposalNumber and p_sessionNumber) must be non-NULL.';
+  END IF;
+END ;;
 DELIMITER ;
 /*!50003 SET sql_mode              = @saved_sql_mode */ ;
 /*!50003 SET character_set_client  = @saved_cs_client */ ;