Add AUTO RELEASE TEMP BLOBID transaction option

doc/sql.extensions/README.set_transaction.txt

@@ -27,6 +27,12 @@ commit fails. This option is mostly used by gfix.
 LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a
 transaction waits for a lock in a record before giving up and reporting an error.
 
+AUTO RELEASE TEMP BLOBID: makes the transaction release temporary ID of user BLOB
+just after its materialization. It's useful for massive insertions of records with
+user-defined BLOBs because it eliminates the memory overhead caused by creating and
+keeping temporary IDs until the transaction ends. This option is used during the
+database restore.
+
 
 Author:
    Claudio Valderrama C.

src/burp/BurpTasks.cpp

@@ -898,6 +898,7 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item)
 			ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3);
 			tpb.insertTag(isc_tpb_concurrency);
 			tpb.insertTag(isc_tpb_no_auto_undo);
+			tpb.insertTag(isc_tpb_auto_release_temp_blobid);
 
 			item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer());
 

src/burp/restore.epp

@@ -421,7 +421,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name)
 		// Block added to verbose index creation by Toni Martir
 		// Always try to activate deferred indices - it helps for some broken backups,
 		// and in normal cases doesn't take much time to look for such indices. AP-2008.
-		EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 
@@ -445,7 +445,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name)
 			general_on_error ();
 		END_ERROR;
 
-		EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 
@@ -481,7 +481,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name)
 		END_ERROR;
 	}
 
-	EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO;
+	EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 	if (gds_status->hasData())
 		EXEC SQL SET TRANSACTION;
 
@@ -745,7 +745,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name)
 			END_ERROR;
 		END_ERROR;
 
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}
@@ -3166,7 +3166,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation)
 		} // end of while
 	END_ERROR;
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 	if (gds_status->hasData())
 		EXEC SQL SET TRANSACTION;
 }
@@ -7777,7 +7777,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 			general_on_error ();
 		END_ERROR;
 
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}
@@ -8004,7 +8004,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 						general_on_error ();
 					END_ERROR;
 				END_ERROR;
-				EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+				EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 				if (gds_status->hasData())
 					EXEC SQL SET TRANSACTION;
 			}
@@ -8041,7 +8041,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 		general_on_error ();
 	END_ERROR;
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 	if (gds_status->hasData())
 		EXEC SQL SET TRANSACTION;
 
@@ -9085,7 +9085,7 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}
@@ -9441,7 +9441,7 @@ bool get_trigger(BurpGlobals* tdgbl)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}
@@ -9537,7 +9537,7 @@ bool get_trigger_message(BurpGlobals* tdgbl)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}
@@ -10391,7 +10391,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 
 	create_database(tdgbl, provider, database_name);
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 	if (gds_status->hasData())
 		EXEC SQL SET TRANSACTION;
 
@@ -10751,7 +10751,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 				ON_ERROR
 					general_on_error ();
 				END_ERROR;
-				EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+				EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 				if (gds_status->hasData())
 					EXEC SQL SET TRANSACTION;
 				flag = false;
@@ -10875,7 +10875,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 		ON_ERROR
 			general_on_error ();
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
 		if (gds_status->hasData())
 			EXEC SQL SET TRANSACTION;
 	}

src/common/ParserTokens.h

@@ -97,6 +97,7 @@ PARSER_TOKEN(TOK_BINARY, "BINARY", false)
 PARSER_TOKEN(TOK_BIND, "BIND", true)
 PARSER_TOKEN(TOK_BIT_LENGTH, "BIT_LENGTH", false)
 PARSER_TOKEN(TOK_BLOB, "BLOB", false)
+PARSER_TOKEN(TOK_BLOBID, "BLOBID", true)
 PARSER_TOKEN(TOK_BLOB_APPEND, "BLOB_APPEND", true)
 PARSER_TOKEN(TOK_BLOCK, "BLOCK", true)
 PARSER_TOKEN(TOK_BODY, "BODY", true)
@@ -492,6 +493,7 @@ PARSER_TOKEN(TOK_TAN, "TAN", true)
 PARSER_TOKEN(TOK_TANH, "TANH", true)
 PARSER_TOKEN(TOK_TARGET, "TARGET", true)
 PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true)
+PARSER_TOKEN(TOK_TEMP, "TEMP", true)
 PARSER_TOKEN(TOK_THEN, "THEN", false)
 PARSER_TOKEN(TOK_TIES, "TIES", true)
 PARSER_TOKEN(TOK_TIME, "TIME", false)

src/dsql/StmtNodes.cpp

@@ -9720,6 +9720,9 @@ SetTransactionNode* SetTransactionNode::dsqlPass(DsqlCompilerScratch* dsqlScratc
 	if (autoCommit.isAssigned())
 		dsqlScratch->appendUChar(isc_tpb_autocommit);
 
+	if (autoReleaseTempBlobID.isAssigned())
+		dsqlScratch->appendUChar(isc_tpb_auto_release_temp_blobid);
+
 	if (lockTimeout.has_value())
 	{
 		dsqlScratch->appendUChar(isc_tpb_lock_timeout);

src/dsql/StmtNodes.h

@@ -1607,6 +1607,7 @@ class SetTransactionNode : public TransactionNode
 		NODE_PRINT(printer, ignoreLimbo);
 		NODE_PRINT(printer, restartRequests);
 		NODE_PRINT(printer, autoCommit);
+		NODE_PRINT(printer, autoReleaseTempBlobID);
 		NODE_PRINT(printer, lockTimeout);
 		//// FIXME-PRINT: NODE_PRINT(printer, reserveList);
 		NODE_PRINT(printer, tpb);
@@ -1633,6 +1634,7 @@ class SetTransactionNode : public TransactionNode
 	Firebird::TriState ignoreLimbo;
 	Firebird::TriState restartRequests;
 	Firebird::TriState autoCommit;
+	Firebird::TriState autoReleaseTempBlobID;
 };
 
 

src/dsql/parse.y

@@ -163,6 +163,7 @@ using namespace Firebird;
 %token <metaNamePtr> BEGIN
 %token <metaNamePtr> BETWEEN
 %token <metaNamePtr> BLOB
+%token <metaNamePtr> BLOBID
 %token <metaNamePtr> BY
 %token <metaNamePtr> CAST
 %token <metaNamePtr> CHARACTER
@@ -308,6 +309,7 @@ using namespace Firebird;
 %token <metaNamePtr> SUSPEND
 %token <metaNamePtr> SUM
 %token <metaNamePtr> TABLE
+%token <metaNamePtr> TEMP
 %token <metaNamePtr> THEN
 %token <metaNamePtr> TO
 %token <metaNamePtr> TRANSACTION
@@ -5938,6 +5940,8 @@ tran_option($setTransactionNode)
 		{ setClause($setTransactionNode->restartRequests, "RESTART REQUESTS", true); }
 	| AUTO COMMIT
 		{ setClause($setTransactionNode->autoCommit, "AUTO COMMIT", true); }
+	| AUTO RELEASE TEMP BLOBID
+		{ setClause($setTransactionNode->autoReleaseTempBlobID, "AUTO RELEASE TEMP BLOBID", true); }
 	// timeout
 	| LOCK TIMEOUT nonneg_short_integer
 		{ setClause($setTransactionNode->lockTimeout, "LOCK TIMEOUT", (USHORT) $3); }
@@ -9749,10 +9753,12 @@ non_reserved_word
 	// added in FB 4.0.2
 	| BLOB_APPEND
 	// added in FB 5.0
+	| BLOBID
 	| LOCKED
 	| OPTIMIZE
 	| QUARTER
 	| TARGET
+	| TEMP
 	| TIMEZONE_NAME
 	| UNICODE_CHAR
 	| UNICODE_VAL

src/gpre/cmp.cpp

@@ -408,6 +408,8 @@ void CMP_t_start( gpre_tra* trans)
 		*text++ = isc_tpb_autocommit;
 	if (trans->tra_flags & TRA_no_auto_undo)
 		*text++ = isc_tpb_no_auto_undo;
+	if (trans->tra_flags & TRA_auto_release_temp_blobid)
+		*text++ = isc_tpb_auto_release_temp_blobid;
 	*text = 0;
 	const USHORT tpb_len = text - tpb_buffer;
 

src/gpre/gpre.h

@@ -1414,7 +1414,8 @@ enum tra_flags_vals {
 	TRA_read_committed	= 32,
 	TRA_autocommit		= 64,
 	TRA_rec_version		= 128,
-	TRA_no_auto_undo	= 256
+	TRA_no_auto_undo	= 256,
+	TRA_auto_release_temp_blobid = 512
 };
 
 const int MAX_TRA_OPTIONS	= 8;

src/gpre/hsh.h

@@ -306,6 +306,7 @@
 	{"PROCEDURE", KW_PROCEDURE},
 	{"PROTECTED", KW_PROTECTED},
 	{"PUBLIC", KW_PUBLIC},
+	{"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID},
 	{"PUT_SEGMENT", KW_PUT_SEGMENT},
 	{"PUT_SLICE", KW_PUT_SLICE},
 	{"QUADWORD", KW_QUAD},

src/gpre/sql.cpp

@@ -4424,6 +4424,12 @@ static act* act_set_transaction()
 			continue;
 		}
 
+		if (MSC_match(KW_AUTO_RELEASE_TEMP_BLOBID))
+		{
+			trans->tra_flags |= TRA_auto_release_temp_blobid;
+			continue;
+		}
+
 		break;
 	}
 

src/gpre/words.h

@@ -270,6 +270,7 @@ enum kwwords_t {
 	KW_PRIVILEGES,
 	KW_PROTECTED,
 	KW_PUBLIC,
+	KW_AUTO_RELEASE_TEMP_BLOBID,
 	KW_QUAD,
 	KW_RAW_PARTITIONS, // ???
 	KW_READ,

src/include/firebird/impl/consts_pub.h

@@ -268,6 +268,7 @@
 #define isc_tpb_lock_timeout              21
 #define isc_tpb_read_consistency          22
 #define isc_tpb_at_snapshot_number        23
+#define isc_tpb_auto_release_temp_blobid  24
 
 
 /************************/

src/include/gen/Firebird.pas

@@ -4111,6 +4111,7 @@ 	IProfilerStatsImpl = class(IProfilerStats)
 	isc_tpb_lock_timeout = byte(21);
 	isc_tpb_read_consistency = byte(22);
 	isc_tpb_at_snapshot_number = byte(23);
+	isc_tpb_auto_release_temp_blobid = byte(24);
 	isc_bpb_version1 = byte(1);
 	isc_bpb_source_type = byte(1);
 	isc_bpb_target_type = byte(2);

src/jrd/Monitoring.cpp

@@ -1138,6 +1138,9 @@ void Monitoring::putTransaction(SnapshotData::DumpRecord& record, const jrd_tra*
 	// statistics
 	const int stat_id = fb_utils::genUniqueId();
 	record.storeGlobalId(f_mon_tra_stat_id, getGlobalId(stat_id));
+	// auto release temp blobid flag
+	temp = (transaction->tra_flags & TRA_auto_release_temp_blobid) ? 1 : 0;
+	record.storeInteger(f_mon_tra_auto_release_temp_blobid, temp);
 
 	record.write();
 

src/jrd/blb.cpp

@@ -326,6 +326,9 @@ blb* blb::create2(thread_db* tdbb,
 
 	blb* blob = allocate_blob(tdbb, transaction);
 
+	if (userBlob)
+		blob->blb_flags |= BLB_user;
+
 	if (type & isc_bpb_type_stream)
 		blob->blb_flags |= BLB_stream;
 
@@ -1296,7 +1299,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc,
 			array->arr_request = own_request;
 	}
 
-	blob->destroy(!materialized_blob);
+	const bool purgeBlob = !materialized_blob ||
+		((transaction->tra_flags & TRA_auto_release_temp_blobid) && (blob->blb_flags & BLB_user));
+	blob->destroy(purgeBlob);
 }
 
 

src/jrd/blb.h

@@ -181,6 +181,7 @@ const int BLB_seek			= 32;		// Seek is pending
 const int BLB_large_scan	= 64;		// Blob is larger than page buffer cache
 const int BLB_close_on_read = 128;		// Temporary blob is not closed until read
 const int BLB_bulk			= 256;		// Blob created by bulk insert operation
+const int BLB_user			= 512;		// User-defined blob
 
 /* Blob levels are:
 

src/jrd/names.h

@@ -287,6 +287,7 @@ NAME("MON$ATTACHMENT_ID", nam_mon_att_id)
 NAME("MON$ATTACHMENT_NAME", nam_mon_att_name)
 NAME("MON$AUTH_METHOD", nam_mon_auth_method)
 NAME("MON$AUTO_COMMIT", nam_mon_auto_commit)
+NAME("MON$AUTO_RELEASE_TEMP_BLOBID", nam_mon_auto_release_temp_blobid)
 NAME("MON$AUTO_UNDO", nam_mon_auto_undo)
 NAME("MON$BACKUP_STATE", nam_mon_backup_state)
 NAME("MON$BACKVERSION_READS", nam_mon_bkversion_reads)

src/jrd/relations.h

@@ -550,6 +550,7 @@ RELATION(nam_mon_transactions, rel_mon_transactions, ODS_11_1, rel_virtual)
 	FIELD(f_mon_tra_auto_commit, nam_mon_auto_commit, fld_flag_nullable, 0, ODS_11_1)
 	FIELD(f_mon_tra_auto_undo, nam_mon_auto_undo, fld_flag_nullable, 0, ODS_11_1)
 	FIELD(f_mon_tra_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_11_1)
+	FIELD(f_mon_tra_auto_release_temp_blobid, nam_mon_auto_release_temp_blobid, fld_flag_nullable, 0, ODS_14_0)
 END_RELATION
 
 // Relation 36 (MON$STATEMENTS)

src/jrd/tra.cpp

@@ -3162,6 +3162,10 @@ static void transaction_options(thread_db* tdbb,
 			transaction->tra_flags |= TRA_no_auto_undo;
 			break;
 
+		case isc_tpb_auto_release_temp_blobid:
+			transaction->tra_flags |= TRA_auto_release_temp_blobid;
+			break;
+
 		case isc_tpb_lock_write:
 			// Cannot set a R/W table reservation if the whole txn is R/O.
 			if (read_only.asBool())

src/jrd/tra.h

@@ -435,10 +435,11 @@ const ULONG TRA_read_consistency	= 0x40000L; 	// ensure read consistency in this
 const ULONG TRA_ex_restart			= 0x80000L; 	// Exception was raised to restart request
 const ULONG TRA_replicating			= 0x100000L;	// transaction is allowed to be replicated
 const ULONG TRA_no_blob_check		= 0x200000L;	// disable blob access checking
+const ULONG TRA_auto_release_temp_blobid	= 0x400000L;	// remove temp ids of materialized user blobs from tra_blobs
 
 // flags derived from TPB, see also transaction_options() at tra.cpp
 const ULONG TRA_OPTIONS_MASK = (TRA_degree3 | TRA_readonly | TRA_ignore_limbo | TRA_read_committed |
-	TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests);
+	TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests | TRA_auto_release_temp_blobid);
 
 const int TRA_MASK				= 3;
 //const int TRA_BITS_PER_TRANS	= 2;