Skip to content

Commit

Permalink
Add partial datacopy support through csc2
Browse files Browse the repository at this point in the history
Signed-off-by: Salil Chandra <[email protected]>
  • Loading branch information
chands10 committed May 18, 2022
1 parent 8850ee1 commit 32ec6ed
Showing 84 changed files with 1,751 additions and 233 deletions.
8 changes: 4 additions & 4 deletions bdb/bdb_api.h
Original file line number Diff line number Diff line change
@@ -480,15 +480,15 @@ void bdb_free_cloned_handle_with_other_data_files(bdb_state_type *bdb_state);
bdb_state_type *
bdb_open_more(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int *bdberr);

/* same, but using a transaction */
bdb_state_type *
bdb_open_more_tran(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle,
tran_type *tran, uint32_t flags, int *bdberr);
@@ -527,15 +527,15 @@ bdb_state_type *bdb_create_more_lite(const char name[], const char dir[],
bdb_state_type *
bdb_create(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int temp,
int *bdberr);

bdb_state_type *
bdb_create_tran(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int temp,
int *bdberr, tran_type *);
1 change: 1 addition & 0 deletions bdb/bdb_int.h
Original file line number Diff line number Diff line change
@@ -830,6 +830,7 @@ struct bdb_state_tag {
short numix; /* number of indexes */
short ixlen[MAXINDEX]; /* size of each index */
signed char ixdta[MAXINDEX]; /* does this index contain the dta? */
int ixdtalen[MAXINDEX]; /* dta len in bytes (0 if index does not contain the dta or is full datacopy) */
signed char ixcollattr[MAXINDEX]; /* does this index contain the column
attributes? */
signed char ixnulls[MAXINDEX]; /*does this index contain any columns that
29 changes: 22 additions & 7 deletions bdb/bdb_verify.c
Original file line number Diff line number Diff line change
@@ -876,34 +876,49 @@ static int bdb_verify_key(verify_common_t *par, int ix, unsigned int lid)
/* if dtacopy, does data payload in the key match the data
* payload in the dta file? */
int expected_size;
int datacopy_size = bdb_state->ixdtalen[ix] > 0 ? bdb_state->ixdtalen[ix] : bdb_state->lrl;
uint8_t *expected_data;
uint8_t datacopy_buffer[bdb_state->lrl];
uint8_t datacopy_buffer[datacopy_size];
if (bdb_state->datacopy_odh) {
int odhlen;
unpack_index_odh(bdb_state, &dbt_data, &genid_right,
datacopy_buffer, sizeof(datacopy_buffer),
&odhlen, &ver);
expected_size = odhlen;
par->vtag_callback(par->db_table, datacopy_buffer,
&expected_size, ver);
if (bdb_state->ixdtalen[ix] == 0) { // full datacopy
par->vtag_callback(par->db_table, datacopy_buffer,
&expected_size, ver);
}
expected_data = datacopy_buffer;
} else {
expected_size = dbt_data.size - sizeof(genid);
expected_data = (uint8_t *)dbt_data.data + sizeof(genid);
memcpy(&genid_right, (uint8_t *)dbt_data.data, sizeof(genid));
}

if (expected_size != bdb_state->lrl) {
if (expected_size != datacopy_size) {
par->verify_status = 1;
locprint(par,
"!%016llx ix %d dtacpy payload wrong size expected %d "
"got %d",
genid_flipped, ix, bdb_state->lrl, expected_size);
genid_flipped, ix, datacopy_size, expected_size);
goto next_key;
}

if (memcmp(expected_data, dbt_dta_check_data.data,
bdb_state->lrl)) {
char tail[datacopy_size];
void *compared_data = dbt_dta_check_data.data;
if (bdb_state->ixdtalen[ix] > 0) { // partial datacopy
rc = par->partial_datacopy_callback(par->db_table, ix, dbt_dta_check_data.data, tail);
if (rc) {
par->verify_status = 1;
locprint(par, "!%016llx ix %d could not convert dta", genid_flipped);
goto next_key;
}
compared_data = tail;
}

if (memcmp(expected_data, compared_data,
datacopy_size)) {
par->verify_status = 1;
locprint(par, "!%016llx ix %d dtacpy data mismatch",
genid_flipped, ix);
1 change: 1 addition & 0 deletions bdb/bdb_verify.h
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@ typedef struct {
bdb_state_type *bdb_state;
struct dbtable *db_table;
const char *tablename;
int (*partial_datacopy_callback)(const struct dbtable *tbl, const int pd_ix, const void *inbuf, void *outbuf);
int (*formkey_callback)(const struct dbtable *tbl, void *dta, void *blob_parm,
int ix, void *keyout, int *keysz);
int (*get_blob_sizes_callback)(const struct dbtable *tbl, void *dta, int blobs[16],
8 changes: 5 additions & 3 deletions bdb/cursor.c
Original file line number Diff line number Diff line change
@@ -631,11 +631,12 @@ bdb_cursor_ifn_t *bdb_cursor_open(
cur->idx = ixnum;
cur->type = BDBC_IX;
if (bdb_state->ixdta[ixnum]) {
int datacopy_size = bdb_state->ixdtalen[ixnum] > 0 ? bdb_state->ixdtalen[ixnum] : bdb_state->lrl;
cur->datacopy =
malloc(bdb_state->lrl + 2 * sizeof(unsigned long long));
malloc(datacopy_size + 2 * sizeof(unsigned long long));
if (!cur->datacopy) {
logmsg(LOGMSG_ERROR, "%s: malloc %zu\n", __func__,
bdb_state->lrl + 2 * sizeof(unsigned long long));
datacopy_size + 2 * sizeof(unsigned long long));
*bdberr = BDBERR_MALLOC;
free(pcur_ifn);
return NULL;
@@ -6527,8 +6528,9 @@ static void *bdb_cursor_datacopy(bdb_cursor_ifn_t *cur)

if (bdb_state->ondisk_header && bdb_state->datacopy_odh &&
(c->type == BDBC_DT || !is_genid_synthetic(c->genid))) {
int datacopy_size = bdb_state->ixdtalen[c->idx] > 0 ? bdb_state->ixdtalen[c->idx] : bdb_state->lrl;
c->unpacked_datacopy = unpack_datacopy_odh(
cur, c->datacopy, bdb_state->lrl, from, size, &c->ver);
cur, c->datacopy, datacopy_size, from, size, &c->ver);
} else {
c->unpacked_datacopy = from;
c->ver = c->state->version;
3 changes: 2 additions & 1 deletion bdb/cursor_rowlocks.c
Original file line number Diff line number Diff line change
@@ -2668,7 +2668,8 @@ int bdb_berkdb_rowlocks_init(bdb_berkdb_t *berkdb, DB *db, int *bdberr)
r->keylen += sizeof(unsigned long long);
r->dtalen = sizeof(unsigned long long);
if (bdb_state->ixdta[cur->idx]) {
r->dtalen += bdb_state->lrl + (bdb_state->ondisk_header ? 7 : 0);
int datacopy_size = bdb_state->ixdtalen[cur->idx] > 0 ? bdb_state->ixdtalen[cur->idx] : bdb_state->lrl;
r->dtalen += datacopy_size + (bdb_state->ondisk_header ? 7 : 0);
}
r->keymem = malloc(r->keylen + 1);
r->key = r->keymem;
48 changes: 29 additions & 19 deletions bdb/file.c
Original file line number Diff line number Diff line change
@@ -4531,11 +4531,12 @@ static int open_dbs_int(bdb_state_type *bdb_state, int iammaster, int upgrade,
way to override this in the lrl file, we're gonna listen */
if (pagesize == 4096) {
/* for datacopy indexes, use a potentially larger pagesize */
if (bdb_state->ixdta[i])
if (bdb_state->ixdta[i]) {
int datacopy_size = bdb_state->ixdtalen[i] > 0 ? bdb_state->ixdtalen[i] : bdb_state->lrl;
pagesize =
calc_pagesize(bdb_state->attr->pagesizeix, bdb_state->lrl + bdb_state->ixlen[i]);
calc_pagesize(bdb_state->attr->pagesizeix, datacopy_size + bdb_state->ixlen[i]);
/*else if (bdb_state->ixcollattr[i]) ignore this for now */
else
} else
pagesize = calc_pagesize(bdb_state->attr->pagesizeix, bdb_state->ixlen[i]);
}

@@ -5445,7 +5446,7 @@ static void bdb_blobmem_init_once(void)
static bdb_state_type *bdb_open_int(
int envonly, const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_attr_type *bdb_attr, bdb_callback_type *bdb_callback,
void *usr_ptr, netinfo_type *netinfo, int upgrade, int create, int *bdberr,
@@ -5684,6 +5685,11 @@ static bdb_state_type *bdb_open_int(
else
bdb_state->ixdta[i] = 0;

if (ixdtalen)
bdb_state->ixdtalen[i] = ixdtalen[i];
else
bdb_state->ixdtalen[i] = 0;

if (ixcollattr)
bdb_state->ixcollattr[i] = ixcollattr[i];
else
@@ -6099,8 +6105,8 @@ bdb_state_type *bdb_open_env(const char name[], const char dir[],

return bdb_open_int(
1, /* envonly */
name, dir, 0, 0, NULL, NULL, NULL, NULL,
NULL, /* numix, ixlen, ixdups, ixrecnum, ixdta, ixcollattr */
name, dir, 0, 0, NULL, NULL, NULL, NULL, NULL,
NULL, /* numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen, ixcollattr */
NULL, /* ixnulls */
0, /* numdtafiles */
bdb_attr, /* bdb_attr */
@@ -6116,7 +6122,7 @@ bdb_state_type *bdb_open_env(const char name[], const char dir[],
bdb_state_type *
bdb_create_tran(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int temp,
int *bdberr, tran_type *trans)
@@ -6133,7 +6139,7 @@ bdb_create_tran(const char name[], const char dir[], int lrl, short numix,

ret =
bdb_open_int(0, /* envonly */
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
ixcollattr, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6148,7 +6154,7 @@ bdb_create_tran(const char name[], const char dir[], int lrl, short numix,
} else {
ret =
bdb_open_int(0, /* envonly */
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
ixcollattr, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6168,7 +6174,7 @@ bdb_create_tran(const char name[], const char dir[], int lrl, short numix,
bdb_state_type *
bdb_open_more_int(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle,
int *bdberr)
@@ -6178,7 +6184,7 @@ bdb_open_more_int(const char name[], const char dir[], int lrl, short numix,
*bdberr = BDBERR_NOERROR;

ret = bdb_open_int(0, /* envonly */
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
ixcollattr, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6194,13 +6200,13 @@ bdb_open_more_int(const char name[], const char dir[], int lrl, short numix,
bdb_state_type *
bdb_create(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int temp,
int *bdberr)
{
return bdb_create_tran(name, dir, lrl, numix, ixlen, ixdups, ixrecnum,
ixdta, ixcollattr, ixnulls, numdtafiles,
ixdta, ixdtalen, ixcollattr, ixnulls, numdtafiles,
parent_bdb_handle, temp, bdberr, NULL);
}

@@ -6209,7 +6215,7 @@ bdb_create(const char name[], const char dir[], int lrl, short numix,
bdb_state_type *
bdb_open_more(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle, int *bdberr)
{
@@ -6221,7 +6227,7 @@ bdb_open_more(const char name[], const char dir[], int lrl, short numix,
BDB_READLOCK("bdb_open_more");

ret = bdb_open_int(0, /* envonly */
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
ixcollattr, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6241,7 +6247,7 @@ bdb_open_more(const char name[], const char dir[], int lrl, short numix,
bdb_state_type *
bdb_open_more_tran(const char name[], const char dir[], int lrl, short numix,
const short ixlen[], const signed char ixdups[],
const signed char ixrecnum[], const signed char ixdta[],
const signed char ixrecnum[], const signed char ixdta[], const int ixdtalen[],
const signed char ixcollattr[], const signed char ixnulls[],
int numdtafiles, bdb_state_type *parent_bdb_handle,
tran_type *tran, uint32_t flags, int *bdberr)
@@ -6254,7 +6260,7 @@ bdb_open_more_tran(const char name[], const char dir[], int lrl, short numix,
BDB_READLOCK("bdb_open_more_tran");

ret = bdb_open_int(0, /* envonly */
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
ixcollattr, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6294,6 +6300,7 @@ bdb_state_type *bdb_open_more_lite(const char name[], const char dir[], int lrl,
signed char ixdups[1] = {0};
signed char ixrecnum[1] = {0};
signed char ixdta[1] = {0};
int ixdtalen[1] = {0};
signed char ixnulls[1] = {0};
short ixlen;

@@ -6307,7 +6314,7 @@ bdb_state_type *bdb_open_more_lite(const char name[], const char dir[], int lrl,
BDB_READLOCK("bdb_open_more_lite");

ret = bdb_open_int(0, /* envonly */
name, dir, lrl, numix, &ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, &ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
NULL, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
@@ -6342,6 +6349,7 @@ bdb_state_type *bdb_open_more_queue(const char name[], const char dir[],
NULL, /* ixdups */
NULL, /* ixrecnum */
NULL, /* ixdta */
NULL, /* ixdtalen */
NULL, /* ixcollattr */
NULL, /* ixnulls */
1, /* numdtafiles (berkdb queue file) */
@@ -6382,6 +6390,7 @@ bdb_state_type *bdb_create_queue_tran(tran_type *tran, const char name[],
NULL, /* ixdups */
NULL, /* ixrecnum */
NULL, /* ixdta */
NULL, /* ixdtalen */
NULL, /* ixcollattr */
NULL, /* ixnulls */
1, /* numdtafiles (berkdb queue file) */
@@ -6418,6 +6427,7 @@ bdb_state_type *bdb_create_more_lite(const char name[], const char dir[],
signed char ixdups[1] = {0};
signed char ixrecnum[1] = {0};
signed char ixdta[1] = {0};
int ixdtalen[1] = {0};
signed char ixnulls[1] = {0};
short ixlen;

@@ -6437,7 +6447,7 @@ bdb_state_type *bdb_create_more_lite(const char name[], const char dir[],
} else {
ret =
bdb_open_int(0, /* envonly */
name, dir, lrl, numix, &ixlen, ixdups, ixrecnum, ixdta,
name, dir, lrl, numix, &ixlen, ixdups, ixrecnum, ixdta, ixdtalen,
NULL, ixnulls, numdtafiles, NULL, /* bdb_attr */
NULL, /* bdb_callback */
NULL, /* usr_ptr */
7 changes: 7 additions & 0 deletions csc2/dynschemaload.h
Original file line number Diff line number Diff line change
@@ -4,6 +4,11 @@
#include <stdlib.h>
#include <stdio.h>

struct partial_datacopy {
char *field;
struct partial_datacopy *next;
};

enum fieldopttypes {
FLDOPT_DBSTORE = 0,
FLDOPT_DBLOAD = 1,
@@ -39,6 +44,7 @@ int dyns_is_idx_dup(int index);
int dyns_is_idx_recnum(int index);
int dyns_is_idx_primary(int index);
int dyns_is_idx_datacopy(int index);
int dyns_is_idx_partial_datacopy(int index);
int dyns_is_idx_uniqnulls(int index);
int dyns_get_idx_count(void);
int dyns_get_idx_size(int index);
@@ -57,6 +63,7 @@ int dyns_field_depth(int fidx, dpth_t *dpthinfo, int ndpthsinfo, int *ndpthout);
int dyns_field_type(int fidx);
int dyns_is_field_array(int fidx);
int dyns_get_field_arr_dims(int fidx, int *dims, int ndims, int *nodims);
int dyns_get_idx_partial_datacopy(int index, struct partial_datacopy **partial_datacopy);
int dyns_get_idx_tag(int index, char *tag, int tlen, char **where);

/* calls to work with multiple tables */
Loading

0 comments on commit 32ec6ed

Please sign in to comment.