Skip to content

Commit 19393e8

Browse files
chands10chands10
committed
Port protobuf changes
Signed-off-by: Salil Chandra <[email protected]>
1 parent fce1d14 commit 19393e8

File tree

1 file changed

+68
-8
lines changed

1 file changed

+68
-8
lines changed

‎cdb2api/cdb2api.c‎

Lines changed: 68 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,13 @@ static int cdb2_alarm_unread_socket_data = 0;
160160
static int cdb2_alarm_unread_socket_data_set_from_env = 0;
161161
static int cdb2_max_discard_records = 1;
162162
static int cdb2_max_discard_records_set_from_env = 0;
163+
/* flattens column values in protobuf structure */
164+
static int cdb2_flat_col_vals = 0;
165+
static int cdb2_flat_col_vals_set_from_env = 0;
166+
/* estimates how much memory protobuf will need, and pre-allocates that much */
167+
static int CDB2_PROTOBUF_HEURISTIC_INIT_SIZE = 1024;
168+
static int cdb2_protobuf_heuristic = 0;
169+
static int cdb2_protobuf_heuristic_set_from_env = 0;
163170

164171
static int CDB2_REQUEST_FP = 0;
165172

@@ -177,15 +184,14 @@ static void *cdb2_protobuf_alloc(void *allocator_data, size_t size)
177184
struct cdb2_hndl *hndl = allocator_data;
178185
void *p = NULL;
179186
if (hndl->protobuf_data && (size <= hndl->protobuf_size - hndl->protobuf_offset)) {
180-
p = (char*)hndl->protobuf_data + hndl->protobuf_offset;
181-
if (size % 8) {
182-
hndl->protobuf_offset += size + (8 - size % 8);
183-
} else {
184-
hndl->protobuf_offset += size;
185-
}
187+
p = hndl->protobuf_data + hndl->protobuf_offset;
188+
hndl->protobuf_offset += ((size + 7) & ~7);
186189
if (hndl->protobuf_offset > hndl->protobuf_size)
187190
hndl->protobuf_offset = hndl->protobuf_size;
191+
LOG_CALL("%s: got %zu bytes from pool max %d current %d\n", __func__, size, hndl->protobuf_size,
192+
hndl->protobuf_offset);
188193
} else {
194+
LOG_CALL("%s: malloc(%zu) max %d current %d\n", __func__, size, hndl->protobuf_size, hndl->protobuf_offset);
189195
p = malloc(size);
190196
}
191197
return p;
@@ -198,6 +204,7 @@ void cdb2_protobuf_free(void *allocator_data, void *ptr)
198204
char *end = start + hndl->protobuf_size;
199205
char *p = ptr;
200206
if (hndl->protobuf_data == NULL || p < start || p >= end) {
207+
LOG_CALL("%s: calling system free\n", __func__);
201208
free(p);
202209
}
203210
}
@@ -1639,6 +1646,9 @@ static void read_comdb2db_environment_cfg(cdb2_hndl_tp *hndl, const char *comdb2
16391646
process_env_var_int("COMDB2_CONFIG_COMDB2DB_TIMEOUT", &COMDB2DB_TIMEOUT, &cdb2_comdb2db_timeout_set_from_env);
16401647
process_env_var_int("COMDB2_CONFIG_SOCKET_TIMEOUT", &CDB2_SOCKET_TIMEOUT, &cdb2_socket_timeout_set_from_env);
16411648
process_env_var_int("COMDB2_CONFIG_PROTOBUF_SIZE", &CDB2_PROTOBUF_SIZE, &cdb2_protobuf_size_set_from_env);
1649+
process_env_var_int("COMDB2_FEATURE_PROTOBUF_HEURISTIC", &cdb2_protobuf_heuristic,
1650+
&cdb2_protobuf_heuristic_set_from_env);
1651+
process_env_var_int("COMDB2_FEATURE_FLAT_COL_VALS", &cdb2_flat_col_vals, &cdb2_flat_col_vals_set_from_env);
16421652
process_env_var_int("COMDB2_FEATURE_USE_BMSD", &cdb2_use_bmsd, &cdb2_use_bmsd_set_from_env);
16431653
process_env_var_int("COMDB2_FEATURE_COMDB2DB_FALLBACK", &cdb2_comdb2db_fallback,
16441654
&cdb2_comdb2db_fallback_set_from_env);
@@ -1851,6 +1861,12 @@ static void read_comdb2db_cfg(cdb2_hndl_tp *hndl, SBUF2 *s, const char *comdb2db
18511861
tok = strtok_r(NULL, " =:,", &last);
18521862
if (tok && is_valid_int(tok))
18531863
cdb2_max_discard_records = atoi(tok);
1864+
} else if (!cdb2_flat_col_vals_set_from_env && strcasecmp("flat_col_vals", tok) == 0) {
1865+
if ((tok = strtok_r(NULL, " =:,", &last)) != NULL)
1866+
cdb2_flat_col_vals = value_on_off(tok, &err);
1867+
} else if (!cdb2_protobuf_heuristic_set_from_env && strcasecmp("protobuf_heuristic", tok) == 0) {
1868+
if ((tok = strtok_r(NULL, " =:,", &last)) != NULL)
1869+
cdb2_protobuf_heuristic = value_on_off(tok, &err);
18541870
}
18551871
} else if (strcasecmp("comdb2_config", tok) == 0) {
18561872
tok = strtok_r(NULL, " =:,", &last);
@@ -4582,7 +4598,8 @@ static int cdb2_send_query(cdb2_hndl_tp *hndl, cdb2_hndl_tp *event_hndl,
45824598
features[n_features++] = CDB2_CLIENT_FEATURES__REQUEST_FP;
45834599
/* Request server to send back row data flat, instead of storing it in
45844600
a nested data structure. This helps reduce server's memory footprint. */
4585-
features[n_features++] = CDB2_CLIENT_FEATURES__FLAT_COL_VALS;
4601+
if (cdb2_flat_col_vals)
4602+
features[n_features++] = CDB2_CLIENT_FEATURES__FLAT_COL_VALS;
45864603

45874604
features[n_features++] = CDB2_CLIENT_FEATURES__ALLOW_MASTER_DBINFO;
45884605
if ((hndl->flags & (CDB2_DIRECT_CPU | CDB2_MASTER)) ||
@@ -5981,6 +5998,39 @@ static void attach_to_handle(cdb2_hndl_tp *child, cdb2_hndl_tp *parent)
59815998
child->context_msgs.has_changed = child->context_msgs.count > 0;
59825999
}
59836000

6001+
static void pb_alloc_heuristic(cdb2_hndl_tp *hndl)
6002+
{
6003+
if (!cdb2_protobuf_heuristic)
6004+
return;
6005+
if (hndl->first_record_read)
6006+
return;
6007+
if (hndl->firstresponse == NULL || hndl->firstresponse->n_value <= 0)
6008+
return;
6009+
6010+
/*
6011+
* Protobuf-c stores scanned repeated fields in its memory slabs. Each memory slab is
6012+
* twice the size of the previous slab. There's a 32-byte overhead to scan each repeated field.
6013+
* So we calculate how many slabs protobuf-c will likely need, multiply it by 32,
6014+
* and allocate twice as much to also accommodate real unpacked data.
6015+
*/
6016+
int nrepeated = hndl->firstresponse->n_value;
6017+
if (cdb2_flat_col_vals) {
6018+
/* both value and isnull are repeated */
6019+
nrepeated <<= 1;
6020+
}
6021+
6022+
int nextpow2 = 1;
6023+
while (nextpow2 < nrepeated)
6024+
nextpow2 <<= 1;
6025+
int newsize = 32 * nextpow2 * 2;
6026+
LOG_CALL("%s: ncolumns %ld, current alloc size %d, new est alloc size %d\n", __func__, hndl->firstresponse->n_value,
6027+
hndl->protobuf_size, newsize);
6028+
if (newsize > hndl->protobuf_size) {
6029+
hndl->protobuf_data = realloc(hndl->protobuf_data, newsize);
6030+
hndl->protobuf_size = newsize;
6031+
}
6032+
}
6033+
59846034
static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int ntypes, int *types, int line,
59856035
int *set_stmt)
59866036
{
@@ -6620,6 +6670,8 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
66206670
cleanup_query_list(hndl, &commit_query_list, __LINE__);
66216671
PRINT_AND_RETURN(return_value);
66226672
}
6673+
6674+
pb_alloc_heuristic(hndl);
66236675
int rc = cdb2_next_record_int(hndl, 1);
66246676
if (rc == CDB2_OK_DONE || rc == CDB2_OK) {
66256677
return_value = cdb2_convert_error_code(hndl->firstresponse->error_code);
@@ -6911,6 +6963,8 @@ int cdb2_column_size(cdb2_hndl_tp *hndl, int col)
69116963
{
69126964
if (hndl->fdb_hndl)
69136965
hndl = hndl->fdb_hndl;
6966+
if (hndl->lastresponse == NULL)
6967+
return -1;
69146968
CDB2SQLRESPONSE *lastresponse = hndl->lastresponse;
69156969
/* sanity check. just in case. */
69166970
if (lastresponse == NULL)
@@ -6929,6 +6983,9 @@ void *cdb2_column_value(cdb2_hndl_tp *hndl, int col)
69296983
{
69306984
if (hndl->fdb_hndl)
69316985
hndl = hndl->fdb_hndl;
6986+
if (hndl->lastresponse == NULL)
6987+
return NULL;
6988+
69326989
CDB2SQLRESPONSE *lastresponse = hndl->lastresponse;
69336990
/* sanity check. just in case. */
69346991
if (lastresponse == NULL)
@@ -8923,7 +8980,10 @@ int cdb2_open(cdb2_hndl_tp **handle, const char *dbname, const char *type,
89238980
PROCESS_EVENT_CTRL_AFTER(hndl, e, rc, callbackrc);
89248981
}
89258982
}
8926-
if (!hndl->protobuf_size)
8983+
8984+
if (cdb2_protobuf_heuristic)
8985+
hndl->protobuf_size = CDB2_PROTOBUF_HEURISTIC_INIT_SIZE;
8986+
else if (!hndl->protobuf_size)
89278987
hndl->protobuf_size = CDB2_PROTOBUF_SIZE;
89288988

89298989
if (hndl->protobuf_size > 0) {

0 commit comments

Comments
 (0)