[PGPRO-12159] Added the output of tsv lexemes positions.

If you create an index with the operator class rum_tsvector_ops,
the positions of the lexemes will be saved as additional information.
The positions are stored in compressed form in bytea.

There is a problem that is related to the fact that in the posting tree,
additional information for the senior keys is stored in a different way,
which is why it has not yet been possible to output it. For all other
cases, the output of additional information works correctly.

Tags: rum

Author: Arseny Kositsyn

src/rum.h

@@ -21,6 +21,7 @@
 #include "storage/bufmgr.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
+#include "tsearch/ts_type.h"
 
 #include "rumsort.h"
 
@@ -836,6 +837,8 @@ extern RumItem *rumGetBAEntry(BuildAccumulator *accum,
 #define RUM_ADDINFO_JOIN			10
 #define RUMNProcs					10
 
+#define LOWERMASK 0x1F
+
 extern PGDLLEXPORT Datum rum_extract_tsvector(PG_FUNCTION_ARGS);
 extern PGDLLEXPORT Datum rum_extract_tsquery(PG_FUNCTION_ARGS);
 extern PGDLLEXPORT Datum rum_tsvector_config(PG_FUNCTION_ARGS);
@@ -847,6 +850,9 @@ extern PGDLLEXPORT Datum rum_ts_distance_td(PG_FUNCTION_ARGS);
 
 extern PGDLLEXPORT Datum tsquery_to_distance_query(PG_FUNCTION_ARGS);
 
+extern char* decompress_pos(char *ptr, WordEntryPos *pos);
+extern unsigned int count_pos(char *ptr, int len);
+
 /* rum_arr_utils.c */
 typedef enum SimilarityType
 {

src/rum_debug_funcs.c

@@ -27,6 +27,7 @@
 #include "access/relation.h"
 #include "utils/varlena.h"
 #include "rum.h"
+#include "tsearch/ts_type.h"
 
 PG_FUNCTION_INFO_V1(rum_metapage_info);
 PG_FUNCTION_INFO_V1(rum_page_opaque_info);
@@ -115,6 +116,8 @@ static Datum category_get_datum_text(RumNullCategory category);
 static Oid find_add_info_oid(RumState *rum_state_ptr);
 static OffsetNumber find_add_info_atrr_num(RumState *rum_state_ptr);
 
+static Datum get_positions_to_text_datum(Datum add_info);
+
 /*
  * The rum_metapage_info() function is used to retrieve 
  * information stored on the meta page of the rum index.
@@ -386,12 +389,6 @@ rum_leaf_data_page_items(PG_FUNCTION_ARGS)
 		/* Allocating memory for a long-lived structure */
 		inter_call_data = palloc(sizeof(rum_page_items_state));
 
-		/* Initializing the RumState structure */
-		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
-		initRumState(inter_call_data->rum_state_ptr, rel);
-
-		relation_close(rel, AccessShareLock);
-
 		/* Getting a copy of the page from the raw page */
 		page = get_page_from_raw(raw_page);
 
@@ -422,6 +419,12 @@ rum_leaf_data_page_items(PG_FUNCTION_ARGS)
 					 errdetail("Flags %04X, expected %04X",
 							   opaq->flags, (RUM_DATA | RUM_LEAF))));
 
+		/* Initializing the RumState structure */
+		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
+		initRumState(inter_call_data->rum_state_ptr, rel);
+
+		relation_close(rel, AccessShareLock);
+
 		/* Build a tuple descriptor for our result type */ 
 		if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
 			elog(ERROR, "return type must be a row type");
@@ -494,9 +497,24 @@ rum_leaf_data_page_items(PG_FUNCTION_ARGS)
 			values[2] = BoolGetDatum(high_key_ptr->addInfoIsNull);
 
 			/* Returning add info */
-			if(!high_key_ptr->addInfoIsNull && inter_call_data->add_info_oid != 0)
+			if(!(high_key_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+				&& inter_call_data->add_info_oid != BYTEAOID)
+			{
 				values[3] = get_datum_text_by_oid(high_key_ptr->addInfo, 
 								  inter_call_data->add_info_oid);
+			}
+
+			/* 
+			 * In this case, we are dealing with the positions 
+			 * of tokens and they need to be decoded. 
+			 */
+			else if (!(high_key_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+					&& inter_call_data->add_info_oid == BYTEAOID) 
+			{
+				/* values[3] = get_positions_to_text_datum(high_key_ptr->addInfo); */
+				values[3] = CStringGetTextDatum("high key positions in posting tree is not supported");
+			}
+
 			else nulls[3] = true;
 
 			/* Forming the returned tuple */
@@ -536,8 +554,23 @@ rum_leaf_data_page_items(PG_FUNCTION_ARGS)
 		values[2] = BoolGetDatum(rum_item_ptr->addInfoIsNull);
 
 		/* Returning add info */
-		if(!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0)
-			values[3] = get_datum_text_by_oid(rum_item_ptr->addInfo, inter_call_data->add_info_oid);
+		if(!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+			&& inter_call_data->add_info_oid != BYTEAOID)
+		{
+			values[3] = get_datum_text_by_oid(rum_item_ptr->addInfo, 
+									 inter_call_data->add_info_oid);
+		}
+
+		/* 
+		 * In this case, we are dealing with the positions 
+		 * of tokens and they need to be decoded. 
+		 */
+		else if (!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+				&& inter_call_data->add_info_oid == BYTEAOID) 
+		{
+			values[3] = get_positions_to_text_datum(rum_item_ptr->addInfo); 
+		}
+
 		else nulls[3] = true;
 
 		/* Forming the returned tuple */
@@ -619,12 +652,6 @@ rum_internal_data_page_items(PG_FUNCTION_ARGS)
 		/* Allocating memory for a long-lived structure */
 		inter_call_data = palloc(sizeof(rum_page_items_state));
 
-		/* Initializing the RumState structure */
-		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
-		initRumState(inter_call_data->rum_state_ptr, rel);
-
-		relation_close(rel, AccessShareLock);
-
 		/* Getting a copy of the page from the raw page */
 		page = get_page_from_raw(raw_page);
 
@@ -655,6 +682,12 @@ rum_internal_data_page_items(PG_FUNCTION_ARGS)
 					 errdetail("Flags %04X, expected %04X",
 							   opaq->flags, (RUM_DATA & ~RUM_LEAF))));
 
+		/* Initializing the RumState structure */
+		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
+		initRumState(inter_call_data->rum_state_ptr, rel);
+
+		relation_close(rel, AccessShareLock);
+
 		/* Build a tuple descriptor for our result type */ 
 		if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
 			elog(ERROR, "return type must be a row type");
@@ -721,9 +754,24 @@ rum_internal_data_page_items(PG_FUNCTION_ARGS)
 			values[3] = BoolGetDatum(high_key_ptr->addInfoIsNull);
 
 			/* Returning add info */
-			if(!high_key_ptr->addInfoIsNull && inter_call_data->add_info_oid != 0)
+			if(!(high_key_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0
+				&& inter_call_data->add_info_oid != BYTEAOID)
+			{
 				values[4] = get_datum_text_by_oid(high_key_ptr->addInfo, 
 								  inter_call_data->add_info_oid);
+			}
+
+			/* 
+			 * In this case, we are dealing with the positions 
+			 * of tokens and they need to be decoded. 
+			 */
+			else if (!(high_key_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+					&& inter_call_data->add_info_oid == BYTEAOID) 
+			{
+				/* values[4] = get_positions_to_text_datum(high_key_ptr->addInfo); */
+				values[4] = CStringGetTextDatum("high key positions in posting tree is not supported");
+			}
+
 			else nulls[4] = true;
 
 			/* Forming the returned tuple */
@@ -745,9 +793,24 @@ rum_internal_data_page_items(PG_FUNCTION_ARGS)
 		values[3] = BoolGetDatum(posting_item_ptr->item.addInfoIsNull);
 
 		/* Returning add info */
-		if(!posting_item_ptr->item.addInfoIsNull && inter_call_data->add_info_oid != 0)
+		if(!posting_item_ptr->item.addInfoIsNull && inter_call_data->add_info_oid != 0
+			&& inter_call_data->add_info_oid != BYTEAOID)
+		{
 			values[4] = get_datum_text_by_oid(posting_item_ptr->item.addInfo, 
 							  inter_call_data->add_info_oid);
+		}
+
+		/* 
+		 * In this case, we are dealing with the positions 
+		 * of tokens and they need to be decoded. 
+		 */
+		else if (!posting_item_ptr->item.addInfoIsNull && inter_call_data->add_info_oid != 0 
+				&& inter_call_data->add_info_oid == BYTEAOID) 
+		{
+			/* values[4] = get_positions_to_text_datum(posting_item_ptr->item.addInfo); */
+			values[4] = CStringGetTextDatum("high key positions in posting tree is not supported");
+		}
+
 		else nulls[4] = true;
 
 		/* Forming the returned tuple */
@@ -833,12 +896,6 @@ rum_leaf_entry_page_items(PG_FUNCTION_ARGS)
 		/* Allocating memory for a long-lived structure */
 		inter_call_data = palloc(sizeof(rum_page_items_state));
 
-		/* Initializing the RumState structure */
-		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
-		initRumState(inter_call_data->rum_state_ptr, rel);
-
-		relation_close(rel, AccessShareLock);
-
 		/* Getting a copy of the page from the raw page */
 		page = get_page_from_raw(raw_page);
 
@@ -869,6 +926,12 @@ rum_leaf_entry_page_items(PG_FUNCTION_ARGS)
 					 errdetail("Flags %04X, expected %04X",
 							   opaq->flags, RUM_LEAF)));
 
+		/* Initializing the RumState structure */
+		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
+		initRumState(inter_call_data->rum_state_ptr, rel);
+
+		relation_close(rel, AccessShareLock);
+
 		/* Build a tuple descriptor for our result type */ 
 		if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
 			elog(ERROR, "return type must be a row type");
@@ -1008,10 +1071,23 @@ rum_leaf_entry_page_items(PG_FUNCTION_ARGS)
 		values[3] = ItemPointerGetDatum(&(rum_item_ptr->iptr)); 
 		values[4] = BoolGetDatum(rum_item_ptr->addInfoIsNull);
 
-
 		/* Returning add info */
-		if(!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0)
+		if (!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 && 
+			inter_call_data->add_info_oid != BYTEAOID)
+		{
 			values[5] = get_datum_text_by_oid(rum_item_ptr->addInfo, inter_call_data->add_info_oid);
+		}
+
+		/* 
+		 * In this case, we are dealing with the positions 
+		 * of tokens and they need to be decoded. 
+		 */
+		else if (!(rum_item_ptr->addInfoIsNull) && inter_call_data->add_info_oid != 0 
+				&& inter_call_data->add_info_oid == BYTEAOID) 
+		{
+			values[5] = get_positions_to_text_datum(rum_item_ptr->addInfo); 
+		}
+		
 		else nulls[5] = true;
 
 		/* The current IndexTuple does not contain a posting tree */
@@ -1101,12 +1177,6 @@ rum_internal_entry_page_items(PG_FUNCTION_ARGS)
 		/* Allocating memory for a long-lived structure */
 		inter_call_data = palloc(sizeof(rum_page_items_state));
 
-		/* Initializing the RumState structure */
-		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
-		initRumState(inter_call_data->rum_state_ptr, rel);
-
-		relation_close(rel, AccessShareLock);
-
 		/* Getting a copy of the page from the raw page */
 		page = get_page_from_raw(raw_page);
 
@@ -1137,6 +1207,12 @@ rum_internal_entry_page_items(PG_FUNCTION_ARGS)
 					 errdetail("Flags %04X, expected %04X",
 							   opaq->flags, 0)));
 
+		/* Initializing the RumState structure */
+		inter_call_data->rum_state_ptr = palloc(sizeof(RumState));
+		initRumState(inter_call_data->rum_state_ptr, rel);
+
+		relation_close(rel, AccessShareLock);
+
 		/* Build a tuple descriptor for our result type */ 
 		if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
 			elog(ERROR, "return type must be a row type");
@@ -1355,7 +1431,7 @@ get_page_from_raw(bytea *raw_page)
  * TODO: All types accepted by rum must be checked, but 
  * perhaps some types are missing or some are superfluous.
  */
-static Datum  
+static Datum
 get_datum_text_by_oid(Datum info, Oid info_oid)
 {
 	char *str_info = NULL;
@@ -1602,3 +1678,69 @@ find_add_info_atrr_num(RumState *rum_state_ptr)
 	/* Need to add 1 because the attributes are numbered from 1 */
 	return add_info_attr_num + 1;
 }
+
+#define POS_STR_BUF_LENGHT 1024
+#define POS_MAX_VAL_LENGHT 6
+
+/*
+ * A function for extracting the positions of tokens from additional 
+ * information. Returns a string in which the positions of the tokens 
+ * are recorded. The memory that the string occupies must be cleared later.
+ */
+static Datum
+get_positions_to_text_datum(Datum add_info)
+{
+	bytea	   		*positions;
+	char	   		*ptrt;
+	WordEntryPos 	position = 0;
+	int32			npos;
+
+	Datum 			res;
+	char 			*positions_str;
+	char 			*positions_str_cur_ptr;
+	int 			cur_max_str_lenght;
+
+	positions = DatumGetByteaP(add_info);
+	ptrt = (char *) VARDATA_ANY(positions);
+	npos = count_pos(VARDATA_ANY(positions),
+					 VARSIZE_ANY_EXHDR(positions));
+
+	/* Initialize the string */
+	positions_str = (char*) palloc(POS_STR_BUF_LENGHT * sizeof(char));
+	positions_str[0] = '\0';
+	cur_max_str_lenght = POS_STR_BUF_LENGHT;
+	positions_str_cur_ptr = positions_str;
+
+	/* Extract the positions of the tokens and put them in the string */
+	for (int i = 0; i < npos; i++)
+	{
+		/* At each iteration decode the position */
+		ptrt = decompress_pos(ptrt, &position);
+
+		/* Write this position in the string */
+		sprintf(positions_str_cur_ptr, "%d,", position);
+
+		/* Moving the pointer forward */
+		positions_str_cur_ptr += strlen(positions_str_cur_ptr);
+
+		/* 
+		 * Check that there is not too little left to the 
+		 * end of the line and, if necessary, overspend 
+		 * the memory. 
+		 */
+		if (cur_max_str_lenght - (positions_str_cur_ptr - positions_str) <= POS_MAX_VAL_LENGHT)
+		{
+			cur_max_str_lenght +=  POS_STR_BUF_LENGHT;
+			positions_str = (char*) repalloc(positions_str, cur_max_str_lenght * sizeof(char));
+			positions_str_cur_ptr = positions_str + strlen(positions_str);
+		}
+	}
+
+	/* Delete the last comma if there has been at least one iteration of the loop */
+	if (npos > 0)
+		positions_str[strlen(positions_str) - 1] = '\0';
+
+	res = CStringGetTextDatum(positions_str);
+	pfree(positions_str);
+	return res;
+}

src/rum_ts_utils.c

@@ -16,7 +16,6 @@
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "miscadmin.h"
-#include "tsearch/ts_type.h"
 #include "tsearch/ts_utils.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
@@ -80,8 +79,6 @@ PG_FUNCTION_INFO_V1(rum_ts_join_pos);
 
 PG_FUNCTION_INFO_V1(tsquery_to_distance_query);
 
-static unsigned int	count_pos(char *ptr, int len);
-static char *decompress_pos(char *ptr, WordEntryPos *pos);
 static Datum build_tsvector_entry(TSVector vector, WordEntry *we);
 static Datum build_tsvector_hash_entry(TSVector vector, WordEntry *we);
 static Datum build_tsquery_entry(TSQuery query, QueryOperand *operand);
@@ -964,7 +961,6 @@ rum_tsquery_timestamp_consistent(PG_FUNCTION_ARGS)
 }
 
 #define SIXTHBIT 0x20
-#define LOWERMASK 0x1F
 
 static unsigned int
 compress_pos(char *target, WordEntryPos *pos, int npos)
@@ -999,7 +995,7 @@ compress_pos(char *target, WordEntryPos *pos, int npos)
 	return ptr - target;
 }
 
-static char *
+extern char *
 decompress_pos(char *ptr, WordEntryPos *pos)
 {
 	int			i;
@@ -1027,7 +1023,7 @@ decompress_pos(char *ptr, WordEntryPos *pos)
 	}
 }
 
-static unsigned int
+extern unsigned int
 count_pos(char *ptr, int len)
 {
 	int			count = 0,