123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680 |
- #include "store_mdb.h"
- /**
- * Number of DBs defined. See MAIN_TABLE and LOOKUP_TABLE defines below.
- */
- #define N_DB 13
- /**
- * Memory map size.
- */
- #if (defined DEBUG || defined TESTING)
- #define DEFAULT_MAPSIZE 1<<24 // 16Mb (limit for Valgrind)
- #elif !(defined __LP64__ || defined __LLP64__) || \
- defined _WIN32 && !defined _WIN64
- #define DEFAULT_MAPSIZE 1<<31 // 2Gb (limit for 32-bit systems)
- #else
- #define DEFAULT_MAPSIZE 1UL<<40 // 1Tb
- #endif
- #define ENV_DIR_MODE 0750
- #define ENV_FILE_MODE 0640
- /*
- * Data types.
- */
- typedef char DbLabel[8];
- typedef struct mdbstore_iter_t MDBIterator;
- /// Store state flags.
- typedef enum {
- LSSTORE_OPEN = 1<<0, ///< Env is open.
- } StoreFlags;
- /// Iterator state flags.
- typedef enum {
- ITER_OPEN_TXN = 1<<0, /**< A transaction is open.
- *
- * The iterator has begun a new
- * transaction on initialization
- * which needs to be closed. If
- * false, the iterator is using an
- * existing transaction which will
- * not be closed with
- * #mdbiter_free().
- */
- } IterFlags;
- typedef enum {
- OP_ADD,
- OP_REMOVE,
- } StoreOp;
- typedef struct mdbstore_t {
- MDB_env * env; ///< Environment handle.
- MDB_dbi dbi[N_DB]; ///< DB handles. Refer to DbIdx enum.
- StoreFlags flags; ///< Store state flags.
- } MDBStore;
- /** @brief Iterator operation.
- *
- * Function executed for each iteration of a #MDBIterator. It assumes that a
- * result triple has already been found and is ready to be composed and
- * yielded.
- *
- * Upon call, the rc value of the iterator structure is set to the MDB_* rc
- * value for the next result. It is up to the caller to evaluate this value
- * and decide whether to call the function again.
- */
- typedef void (*iter_op_fn_t)(MDBIterator *it);
- /// Triple iterator.
- typedef struct mdbstore_iter_t {
- MDBStore * store; ///< MDB store handle.
- IterFlags flags; ///< Iterator flags.
- MDB_txn * txn; ///< MDB transaction.
- MDB_cursor * cur; ///< MDB cursor.
- MDB_cursor * ctx_cur; ///< MDB c:spo index cursor.
- MDB_val key; ///< Internal data handler.
- MDB_val data; ///< Internal data handler.
- LSUP_TripleKey spok; ///< Triple to be populated with match.
- LSUP_Key * ck; /**< Context array.
- *
- * This shall be populated for each
- * matching triple if requested. */
- iter_op_fn_t iter_op_fn; ///< Function used to look up next match.
- const uint8_t * term_order; ///< Term order used in 1-2bound look-ups.
- LSUP_Key luk[3]; ///< 0÷3 lookup keys.
- LSUP_Key luc; ///< Ctx key to filter by. May be NULL_KEY.
- size_t i; ///< Internal counter for paged lookups.
- size_t ct; ///< Current count of records inserted or
- ///< results found.
- int rc; ///< MDB_* return code for the next result.
- } MDBIterator;
- /*
- * Static variables.
- */
- #define DUPSORT_MASK MDB_DUPSORT
- #define DUPFIXED_MASK MDB_DUPSORT | MDB_DUPFIXED
- /**
- * Main DBs. These are the master information containers.
- *
- * The number of entries must match the N_DB constant defined above.
- */
- #define MAIN_TABLE \
- /* #ID pfx #DB label #Flags */ \
- ENTRY( T_ST, "t:st", 0 ) /* Key to ser. term */ \
- ENTRY( SPO_C, "spo:c", DUPFIXED_MASK ) /* Triple to context */ \
- ENTRY( C_, "c:", 0 ) /* Track empty ctx */ \
- ENTRY( PFX_NS, "pfx:ns", 0 ) /* Prefix to NS */ \
- ENTRY( IDK_ID, "idk:id", 0 ) /* ID key to ID */ \
- /**
- * Lookup DBs. These are indices and may be destroyed and rebuilt.
- */
- #define LOOKUP_TABLE \
- /* #ID pfx #DB label #Flags */ \
- ENTRY( S_PO, "s:po", DUPFIXED_MASK ) /* 1-bound lookup */ \
- ENTRY( P_SO, "p:so", DUPFIXED_MASK ) /* 1-bound lookup */ \
- ENTRY( O_SP, "o:sp", DUPFIXED_MASK ) /* 1-bound lookup */ \
- ENTRY( PO_S, "po:s", DUPFIXED_MASK ) /* 2-bound lookup */ \
- ENTRY( SO_P, "so:p", DUPFIXED_MASK ) /* 2-bound lookup */ \
- ENTRY( SP_O, "sp:o", DUPFIXED_MASK ) /* 2-bound lookup */ \
- ENTRY( C_SPO, "c:spo", DUPFIXED_MASK ) /* Context lookup */ \
- ENTRY( NS_PFX, "ns:pfx", DUPSORT_MASK ) /* NS to prefix */ \
- /**
- * DB labels. They are prefixed with DB_
- */
- #define ENTRY(a, b, c) static const DbLabel DB_##a = b;
- MAIN_TABLE
- LOOKUP_TABLE
- #undef ENTRY
- /*
- * Numeric index of each DB. Prefixed with IDX_
- *
- * These index numbers are referred to in all the arrays defeined below. They
- * are independent from the LMDB dbi values which are considered opaque here.
- */
- typedef enum {
- #define ENTRY(a, b, c) IDX_##a,
- MAIN_TABLE
- LOOKUP_TABLE
- #undef ENTRY
- } DBIdx;
- /**
- * DB labels.
- */
- static const char *db_labels[N_DB] = {
- #define ENTRY(a, b, c) DB_##a,
- MAIN_TABLE
- LOOKUP_TABLE
- #undef ENTRY
- };
- /*
- * DB flags. These are aligned with the dbi_labels index.
- */
- static const unsigned int db_flags[N_DB] = {
- #define ENTRY(a, b, c) c,
- MAIN_TABLE
- LOOKUP_TABLE
- #undef ENTRY
- };
- /*
- * 1-bound and 2-bound lookup indices.
- *
- * N.B. Only the first 6 (1-bound and 2-bound term lookup) are used.
- * The others are added just because they belong logically to the lookup table.
- */
- static DBIdx lookup_indices[9] = {
- #define ENTRY(a, b, c) IDX_##a,
- LOOKUP_TABLE
- #undef ENTRY
- };
- static const uint8_t lookup_ordering_1bound[3][3] = {
- {0, 1, 2}, // s:po
- {1, 0, 2}, // p:so
- {2, 0, 1}, // o:sp
- };
- static const uint8_t lookup_ordering_2bound[3][3] = {
- {1, 2, 0}, // po:s
- {0, 2, 1}, // so:p
- {0, 1, 2}, // sp:o
- };
- /*
- * Static prototypes.
- */
- static int index_triple(
- MDBStore *store, StoreOp op, LSUP_TripleKey spok, LSUP_Key ck,
- MDB_txn *txn);
- static LSUP_rc mdbstore_add_term (void *h, const LSUP_Buffer *sterm, void *th);
- inline static LSUP_rc lookup_0bound (MDBIterator *it, size_t *ct);
- inline static LSUP_rc lookup_1bound (
- uint8_t idx0, MDBIterator *it, size_t *ct);
- inline static LSUP_rc lookup_2bound (
- uint8_t idx0, uint8_t idx1, MDBIterator *it, size_t *ct);
- inline static LSUP_rc lookup_3bound(MDBIterator *it, size_t *ct);
- /**
- * Store interface.
- */
- static LSUP_NSMap *
- mdbstore_nsm_get (void *h)
- {
- MDBStore *store = h;
- LSUP_NSMap *nsm = LSUP_nsmap_new();
- if (UNLIKELY (!nsm)) return NULL;
- MDB_txn *txn;
- mdb_txn_begin (store->env, NULL, MDB_RDONLY, &txn);
- MDB_cursor *cur;
- if (mdb_cursor_open (txn, store->dbi[IDX_PFX_NS], &cur) != MDB_SUCCESS) {
- mdb_txn_abort (txn);
- return NULL;
- }
- MDB_val ns_v, pfx_v;
- if (mdb_cursor_get (cur, &pfx_v, &ns_v, MDB_FIRST) != MDB_SUCCESS)
- goto finally;
- do {
- ns_pfx pfx;
- char *ns = malloc (ns_v.mv_size);
- strncpy (pfx, pfx_v.mv_data, pfx_v.mv_size);
- strncpy (ns, ns_v.mv_data, ns_v.mv_size);
- LSUP_nsmap_add (nsm, pfx, ns);
- free (ns);
- } while (mdb_cursor_get (
- cur, &pfx_v, &ns_v, MDB_NEXT_NODUP) == MDB_SUCCESS);
- finally:
- mdb_cursor_close (cur);
- mdb_txn_abort (txn);
- return nsm;
- }
- static LSUP_rc
- mdbstore_nsm_put (void *h, const LSUP_NSMap *nsm, void *th)
- {
- MDBStore *store = h;
- MDB_txn *txn;
- RCCK (mdb_txn_begin (store->env, (MDB_txn *) th, 0, &txn));
- LSUP_rc rc = LSUP_NOACTION;
- int db_rc;
- MDB_cursor *dcur = NULL, *icur = NULL;
- if (
- mdb_cursor_open (txn, store->dbi[IDX_PFX_NS], &dcur) != MDB_SUCCESS
- ||
- mdb_cursor_open (txn, store->dbi[IDX_NS_PFX], &icur) != MDB_SUCCESS
- ) {
- mdb_txn_abort (txn);
- return LSUP_DB_ERR;
- }
- MDB_val pfx_v, ns_v;
- const char ***nsm_data = LSUP_nsmap_dump (nsm);
- for (size_t i = 0; nsm_data[i] != NULL; i++) {
- // At least 1 action. If not OK, it will change during the iteration.
- if (i == 0) rc = LSUP_OK;
- // On previous error, just clean up the NSM data array.
- if (rc < 0) goto loop_end;
- pfx_v.mv_data = (void *) nsm_data[i][0];
- pfx_v.mv_size = strlen (nsm_data[i][0]) + 1;
- ns_v.mv_data = (void *) nsm_data[i][1];
- ns_v.mv_size = strlen (nsm_data[i][1]) + 1;
- // If either ns or pfx exist, skip.
- if (
- mdb_cursor_get (dcur, &pfx_v, &ns_v, MDB_SET) != MDB_NOTFOUND
- ||
- mdb_cursor_get (icur, &ns_v, &pfx_v, MDB_SET) != MDB_NOTFOUND
- ) {
- rc = LSUP_CONFLICT;
- goto loop_end;
- }
- db_rc = mdb_cursor_put (dcur, &pfx_v, &ns_v, 0);
- db_rc |= mdb_cursor_put (icur, &ns_v, &pfx_v, 0);
- if (db_rc != MDB_SUCCESS) {
- log_error ("DB error: %s", LSUP_strerror (db_rc));
- rc = LSUP_DB_ERR;
- }
- loop_end:
- free (nsm_data[i]);
- }
- free (nsm_data);
- if (UNLIKELY (rc != LSUP_OK)) mdb_txn_abort (txn);
- else if (UNLIKELY (mdb_txn_commit (txn) != MDB_SUCCESS)) {
- mdb_txn_abort (txn);
- rc = LSUP_TXN_ERR;
- }
- return rc;
- }
- static const char *
- mdbstore_path_from_id (const char *id)
- {
- // Set environment path.
- if (!id) id = getenv ("LSUP_MDB_STORE_URN");
- if (!id) {
- id = LSUP_MDB_STORE_URN;
- log_info (
- "`LSUP_MDB_STORE_URN' environment variable is not "
- "set. The default URN %s has been set as the store ID.", id
- );
- }
- if (strncmp ("file://", id, 7) != 0) {
- log_error ("MDB store ID must be in the `file://<abs_path>` format.");
- return NULL;
- }
- return id + 7;
- }
- /** @brief Create the MDB environment and databases on disk.
- *
- * This function takes care of creaating the environment path if not existing,
- * and checking that it's a writable directory. If the path is not specified
- * in the LSUP_MDB_STORE_URN environment variable, a default directory is used.
- */
- static LSUP_rc
- mdbstore_setup (const char *id, bool clear)
- {
- const char *path = mdbstore_path_from_id (id);
- if (!path) return LSUP_VALUE_ERR;
- // If the directory exists (unless clear == true), do nothing.
- if (clear) rm_r (path);
- LSUP_rc rc = mkdir_p (path, ENV_DIR_MODE);
- log_info ("Create dir rc: %d", rc);
- log_info ("LSUP Create dir rc: %d", rc);
- RCCK (rc);
- // Open a temporary environment and txn to create the DBs.
- MDB_env *env;
- RCCK (mdb_env_create (&env));
- RCCK (mdb_env_set_maxdbs (env, N_DB));
- RCCK (mdb_env_open (env, path, 0, ENV_FILE_MODE));
- log_debug ("Environment opened at %s.", path);
- MDB_txn *txn;
- RCCK (mdb_txn_begin (env, NULL, 0, &txn));
- for (int i = 0; i < N_DB; i++) {
- log_trace ("Creating DB %s", db_labels[i]);
- MDB_dbi dbi;
- RCCK (
- mdb_dbi_open (txn, db_labels[i], db_flags[i] | MDB_CREATE, &dbi)
- );
- }
- mdb_txn_commit (txn);
- mdb_env_close (env);
- return LSUP_OK;
- }
- /** @brief Open an MDB store.
- *
- * The store must have been set up with #mdbstore_setup.
- *
- * Some environment variables affect various store parameters:
- *
- * - LSUP_MDB_MAPSIZE Long int specifying the size of the memory map. Usually
- * it is not necessary to modify this, unless one is operating under memory
- * and disk constraints. The default map size is 1Tb.
- */
- static void *
- mdbstore_new (const char *id, size_t _unused)
- {
- (void) _unused;
- const char *path = mdbstore_path_from_id (id);
- if (!path) return NULL;
- MDBStore *store;
- CALLOC_GUARD (store, NULL);
- RCNL (mdb_env_create (&store->env));
- MDB_txn *txn = NULL;
- // Set map size.
- size_t mapsize;
- char *env_mapsize = getenv ("LSUP_MDB_MAPSIZE");
- if (env_mapsize == NULL) mapsize = DEFAULT_MAPSIZE;
- else sscanf (env_mapsize, "%lu", &mapsize);
- log_info (
- "Setting environment map size at %s to %lu Mb.",
- path, mapsize / 1024 / 1024);
- CHECK (mdb_env_set_mapsize (store->env, mapsize), fail);
- CHECK (mdb_env_set_maxdbs (store->env, N_DB), fail);
- CHECK (mdb_env_open (store->env, path, 0, ENV_FILE_MODE), fail);
- // Assign DB handles to store->dbi.
- CHECK (mdb_txn_begin (store->env, NULL, 0, &txn), fail);
- for (int i = 0; i < N_DB; i++)
- CHECK (mdb_dbi_open (
- txn, db_labels[i], db_flags[i], store->dbi + i), fail);
- // Bootstrap the permanent store with initial data.
- MDB_stat stat;
- CHECK (mdb_stat (txn, store->dbi[IDX_PFX_NS], &stat), fail);
- if (stat.ms_entries == 0) {
- log_debug ("Loading initial data into %s", path);
- // Load initial NS map.
- mdbstore_nsm_put (store, LSUP_default_nsm, txn);
- // Index default context.
- // Create a dummy iterator just to use the current txn.
- MDBIterator *it;
- CALLOC_GUARD (it, NULL);
- it->txn = txn;
- mdbstore_add_term (store, LSUP_default_ctx_buf, it);
- free (it);
- }
- store->flags |= LSSTORE_OPEN;
- mdb_txn_commit (txn);
- txn = NULL;
- return store;
- fail:
- if (txn) mdb_txn_abort (txn);
- mdb_env_close (store->env);
- return NULL;
- }
- static void
- mdbstore_free (void *h)
- {
- MDBStore *store = h;
- if (store->flags & LSSTORE_OPEN) {
- const char *path;
- mdb_env_get_path (store->env, &path);
- log_info ("Closing MDB env at %s.", path);
- mdb_env_close (store->env);
- }
- free (store);
- }
- #if 0
- static char *
- mdbstore_id (const void *h)
- {
- const MDBStore *store = h;
- const char *path;
- mdb_env_get_path (store->env, &path);
- return strcat ("file://", path);
- }
- #endif
- static LSUP_rc
- mdbstore_stat (const MDBStore *store, MDB_stat *stat)
- {
- if (!(store->flags & LSSTORE_OPEN)) return 0;
- MDB_txn *txn;
- mdb_txn_begin (store->env, NULL, MDB_RDONLY, &txn);
- if (mdb_stat (txn, store->dbi[IDX_SPO_C], stat) != MDB_SUCCESS)
- return LSUP_DB_ERR;
- mdb_txn_abort (txn);
- return LSUP_OK;
- }
- static size_t
- mdbstore_size (const void *h)
- {
- const MDBStore *store = h;
- // Size is calculated outside of any pending write txn.
- MDB_stat stat;
- if (mdbstore_stat (store, &stat) != LSUP_OK) return 0;
- return stat.ms_entries;
- }
- static LSUP_rc
- mdbstore_txn_begin (void *h, int flags, void **th)
- {
- MDBStore *store = h;
- RCCK (mdb_txn_begin (store->env, NULL, flags, (MDB_txn **) th));
- return LSUP_OK;
- }
- static LSUP_rc
- mdbstore_txn_commit (void *th)
- {
- RCCK (mdb_txn_commit ((MDB_txn *) th));
- return LSUP_OK;
- }
- static void
- mdbstore_txn_abort (void *th)
- { mdb_txn_abort ((MDB_txn *) th); }
- static void *
- mdbiter_txn (void *h)
- { return ((MDBIterator *) h)->txn; }
- /** @brief Begin an add loop.
- *
- * @sa #store_add_init_fn_t
- *
- * @param[in] th Previously opened MDB_txn handle, if the add loop shall be
- * run within an enclosing transaction. The transaction must be read-write.
- * The operation will always open a new transaction that is closed with
- * #mdbstore_add_done() or #mdbstore_add_abort(). If this parameter is not
- * NULL, the loop transaction will have the passed txn set as its parent.
- */
- static void *
- mdbstore_add_init (void *h, const LSUP_Buffer *sc, void *th)
- {
- MDBStore *store = h;
- /* An iterator is used here. Some members are a bit misused but it does
- * its job without having to define a very similar struct.
- */
- MDBIterator *it;
- MALLOC_GUARD (it, NULL);
- it->store = store;
- it->i = 0;
- mdb_txn_begin (store->env, (MDB_txn *) th, 0, &it->txn);
- if (sc) {
- // Store context if it's not the default one.
- it->luc = LSUP_buffer_hash (sc);
- // Insert t:st for context.
- //log_debug ("Adding context: %s", sc);
- it->key.mv_data = &it->luc;
- it->key.mv_size = KLEN;
- it->data.mv_data = sc->addr;
- it->data.mv_size = sc->size;
- int db_rc = mdb_put (
- it->txn, it->store->dbi[IDX_T_ST],
- &it->key, &it->data, MDB_NOOVERWRITE);
- if (db_rc != MDB_SUCCESS && db_rc != MDB_KEYEXIST) {
- log_error (LSUP_strerror (db_rc));
- mdb_txn_abort (it->txn);
- return NULL;
- }
- } else {
- log_debug ("No context passed to iterator, using default.");
- it->luc = LSUP_buffer_hash (LSUP_default_ctx_buf);
- }
- return it;
- }
- /*
- * NOTE: at the moment #mdbstore_remove() or another
- * #mdbstore_init() cannot be called between #mdbstore_add_init and
- * #mdbstore_add_abort or #mdbstore_add_done. FIXME
- *
- */
- static LSUP_rc
- mdbstore_add_iter (void *h, const LSUP_BufferTriple *sspo)
- {
- if (UNLIKELY (!h)) return LSUP_VALUE_ERR;
- MDBIterator *it = h;
- int db_rc = LSUP_NOACTION;
- LSUP_TripleKey spok = NULL_TRP;
- // Add triple terms.
- for (int i = 0; i < 3; i++) {
- LSUP_Buffer *st = LSUP_btriple_pos (sspo, i);
- spok[i] = LSUP_buffer_hash (st);
- it->key.mv_data = spok + i;
- it->key.mv_size = KLEN;
- it->data.mv_data = st->addr;
- it->data.mv_size = st->size;
- db_rc = mdb_put(
- it->txn, it->store->dbi[IDX_T_ST],
- &it->key, &it->data, MDB_NOOVERWRITE);
- if (db_rc != MDB_SUCCESS && db_rc != MDB_KEYEXIST) {
- LOG_RC (db_rc);
- return LSUP_DB_ERR;
- }
- }
- log_trace ("Inserting spok: {%lx, %lx, %lx}", spok[0], spok[1], spok[2]);
- log_trace ("Into context: %lx", it->luc);
- // Insert spo:c.
- it->key.mv_data = spok;
- it->key.mv_size = TRP_KLEN;
- // In triple mode, data is empty (= NULL_KEY).
- it->data.mv_data = &it->luc;
- it->data.mv_size = it->luc == NULL_KEY ? 0 : KLEN;
- db_rc = mdb_put(
- it->txn, it->store->dbi[IDX_SPO_C],
- &it->key, &it->data, MDB_NODUPDATA);
- if (db_rc == MDB_KEYEXIST) return LSUP_NOACTION;
- if (db_rc != MDB_SUCCESS) {
- log_error (
- "MDB error while inserting triple: %s", LSUP_strerror(db_rc));
- return LSUP_DB_ERR;
- }
- // Index.
- LSUP_rc rc = index_triple (it->store, OP_ADD, spok, it->luc, it->txn);
- if (rc == LSUP_OK) it->i++;
- return rc;
- }
- static LSUP_rc
- mdbstore_add_done (void *h)
- {
- MDBIterator *it = h;
- LSUP_rc rc = LSUP_OK;
- if (mdb_txn_commit (it->txn) != MDB_SUCCESS) {
- mdb_txn_abort (it->txn);
- rc = LSUP_TXN_ERR;
- }
- free (it);
- return rc;
- }
- static void
- mdbstore_add_abort (void *h)
- {
- MDBIterator *it = h;
- mdb_txn_abort (it->txn);
- free (it);
- }
- #if 0
- /* TODO deprecate. Use low-level instead and abstract at graph level. */
- static LSUP_rc
- mdbstore_add (
- void *h, const LSUP_Buffer *sc,
- const LSUP_BufferTriple strp[], const size_t ct, size_t *inserted)
- {
- MDBStore *store = h;
- MDBIterator *it = mdbstore_add_init (store, sc);
- if (UNLIKELY (!it)) return LSUP_DB_ERR;
- for (size_t i = 0; i < ct; i++) {
- LSUP_rc rc = mdbstore_add_iter (it, strp + i);
- if (UNLIKELY (rc < 0)) {
- mdbstore_add_abort (it);
- return rc;
- }
- }
- *inserted = it->i;
- return mdbstore_add_done (it);
- }
- #endif
- static LSUP_rc
- key_to_sterm (MDBIterator *it, const LSUP_Key key, LSUP_Buffer *sterm)
- {
- LSUP_rc rc = LSUP_NORESULT;
- int db_rc;
- MDB_val key_v, data_v;
- key_v.mv_data = (void*)&key;
- key_v.mv_size = KLEN;
- db_rc = mdb_get (it->txn, it->store->dbi[IDX_T_ST], &key_v, &data_v);
- sterm->flags |= LSUP_BUF_BORROWED;
- if (db_rc == MDB_SUCCESS) {
- sterm->addr = data_v.mv_data;
- sterm->size = data_v.mv_size;
- rc = LSUP_OK;
- } else if (db_rc == MDB_NOTFOUND) {
- sterm->addr = NULL;
- sterm->size = 0;
- } else rc = LSUP_DB_ERR;
- return rc;
- }
- static void *
- mdbstore_lookup (
- void *h, const LSUP_Buffer *ss, const LSUP_Buffer *sp,
- const LSUP_Buffer *so, const LSUP_Buffer *sc, void *th, size_t *ct)
- {
- LSUP_TripleKey spok = {
- LSUP_buffer_hash (ss),
- LSUP_buffer_hash (sp),
- LSUP_buffer_hash (so),
- };
- MDBIterator *it;
- CALLOC_GUARD (it, NULL);
- it->store = h;
- it->luc = LSUP_buffer_hash (sc);
- log_debug ("Lookup context: %lx", it->luc);
- if (ct) *ct = 0;
- uint8_t idx0, idx1;
- // Start RO transaction if not in a write txn already.
- if (th) it->txn = th;
- else {
- it->rc = mdb_txn_begin (it->store->env, NULL, MDB_RDONLY, &it->txn);
- if (it->rc != MDB_SUCCESS) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return NULL;
- }
- it->flags |= ITER_OPEN_TXN;
- }
- // Context index loop.
- if (UNLIKELY (mdb_cursor_open (
- it->txn, it->store->dbi[IDX_SPO_C], &it->ctx_cur) != MDB_SUCCESS))
- return NULL;
- /*
- * Lookup decision tree.
- */
- // s p o (all terms bound)
- if (spok[0] != NULL_KEY && spok[1] != NULL_KEY && spok[2] != NULL_KEY) {
- it->luk[0] = spok[0];
- it->luk[1] = spok[1];
- it->luk[2] = spok[2];
- PRCNL (lookup_3bound (it, ct));
- } else if (spok[0] != NULL_KEY) {
- it->luk[0] = spok[0];
- idx0 = 0;
- // s p ?
- if (spok[1] != NULL_KEY) {
- it->luk[1] = spok[1];
- idx1 = 1;
- PRCNL (lookup_2bound (idx0, idx1, it, ct));
- // s ? o
- } else if (spok[2] != NULL_KEY) {
- it->luk[1] = spok[2];
- idx1 = 2;
- PRCNL (lookup_2bound (idx0, idx1, it, ct));
- // s ? ?
- } else PRCNL (lookup_1bound (idx0, it, ct));
- } else if (spok[1] != NULL_KEY) {
- it->luk[0] = spok[1];
- idx0 = 1;
- // ? p o
- if (spok[2] != NULL_KEY) {
- it->luk[1] = spok[2];
- idx1 = 2;
- PRCNL (lookup_2bound (idx0, idx1, it, ct));
- // ? p ?
- } else PRCNL (lookup_1bound (idx0, it, ct));
- // ? ? o
- } else if (spok[2] != NULL_KEY) {
- it->luk[0] = spok[2];
- idx0 = 2;
- PRCNL (lookup_1bound (idx0, it, ct));
- // ? ? ? (all terms unbound)
- } else PRCNL (lookup_0bound (it, ct));
- return it;
- }
- /** @brief Get next iterator key.
- *
- * ckset is filled with an array of contexts that the triple appears
- * in, if not NULL.
- */
- static LSUP_rc
- mdbiter_next_key (MDBIterator *it)
- {
- if (UNLIKELY (!it)) return LSUP_VALUE_ERR;
- // Only advance if the previous it->rc wasn't already at the end.
- if (it->rc == MDB_NOTFOUND) return LSUP_END;
- if (UNLIKELY (it->rc != MDB_SUCCESS)) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- LSUP_rc rc;
- /* Retrieve current value and advance cursor to the next result.
- * it->rc is set to the result of the next iteration.
- */
- it->iter_op_fn (it);
- log_trace (
- "Found spok: {%lx, %lx, %lx}",
- it->spok[0], it->spok[1], it->spok[2]);
- MDB_val key, data;
- int db_rc;
- key.mv_size = TRP_KLEN;
- data.mv_data = &it->luc;
- data.mv_size = KLEN;
- if (it->luc) {
- rc = LSUP_NORESULT; // Intermediary value, will never be returned.
- while (rc == LSUP_NORESULT) {
- //log_debug ("begin ctx loop.");
- // If ctx is specified, look if the matching triple is associated
- // with it. If not, move on to the next triple.
- // The loop normally exits when a triple with matching ctx is found
- // (LSUP_OK), if there are no more triples (LSUP_END), or if there
- // is an error (LSUP_DB_ERR).
- key.mv_data = it->spok;
- db_rc = mdb_cursor_get (it->ctx_cur, &key, &data, MDB_GET_BOTH);
- if (db_rc == MDB_SUCCESS) {
- rc = LSUP_OK;
- log_trace ("Triple found for context.");
- } else if (db_rc == MDB_NOTFOUND) {
- log_trace ("No triples found for context.");
- if (it->rc == MDB_NOTFOUND) rc = LSUP_END;
- else it->iter_op_fn (it);
- } else {
- log_error ("Database error: %s", LSUP_strerror (db_rc));
- rc = LSUP_DB_ERR;
- }
- }
- } else rc = LSUP_OK;
- // Get all contexts for a triple.
- key.mv_data = it->spok;
- db_rc = mdb_cursor_get (it->ctx_cur, &key, &data, MDB_SET_KEY);
- if (db_rc != MDB_SUCCESS) {
- log_error ("No context found for triple!");
- return LSUP_DB_ERR;
- }
- size_t ct;
- db_rc = mdb_cursor_count (it->ctx_cur, &ct);
- if (db_rc != MDB_SUCCESS) return LSUP_DB_ERR;
- // 1 spare for sentinel. Always allocated even on zero matches.
- LSUP_Key *tmp_ck = realloc (it->ck, sizeof (*it->ck) * (ct + 1));
- if (!tmp_ck) return LSUP_MEM_ERR;
- it->ck = tmp_ck;
- size_t i = 0;
- do {
- //log_trace("Copying to slot #%lu @%p", i, it->ck + i);
- memcpy (it->ck + i++, data.mv_data, sizeof (*it->ck));
- } while (
- mdb_cursor_get (it->ctx_cur, &key, &data, MDB_NEXT_DUP)
- == MDB_SUCCESS);
- //log_trace ("setting sentinel @%p", it->ck + i);
- it->ck[i] = NULL_KEY;
- return rc;
- }
- static LSUP_rc
- mdbiter_next (
- void *h, LSUP_BufferTriple *sspo, LSUP_Buffer **ctx_p)
- {
- MDBIterator *it = h;
- LSUP_rc rc = mdbiter_next_key (it);
- if (rc == LSUP_OK) {
- if (sspo) {
- key_to_sterm (it, it->spok[0], sspo->s);
- key_to_sterm (it, it->spok[1], sspo->p);
- key_to_sterm (it, it->spok[2], sspo->o);
- // TODO error handling.
- }
- // Contexts for current triple.
- if (ctx_p) {
- // Preallocate.
- size_t i = 0;
- while (it->ck[i++]); // Include sentinel in count.
- LSUP_Buffer *ctx;
- log_trace("Allocating %lu context buffers.", i);
- ctx = malloc(i * sizeof (*ctx));
- if (!ctx) return LSUP_MEM_ERR;
- for (i = 0; it->ck[i]; i++)
- key_to_sterm (it, it->ck[i], ctx + i);
- memset (ctx + i, 0, sizeof (*ctx)); // Sentinel
- // TODO error handling.
- *ctx_p = ctx;
- }
- }
- return rc;
- }
- static void
- mdbiter_free (void *h)
- {
- MDBIterator *it = h;
- if (!it) return;
- if (it->cur) mdb_cursor_close (it->cur);
- if (it->ctx_cur) mdb_cursor_close (it->ctx_cur);
- if (it->flags & ITER_OPEN_TXN) mdb_txn_abort (it->txn);
- free (it->ck);
- free (it);
- }
- static LSUP_rc
- mdbstore_remove (
- void *h, const LSUP_Buffer *ss, const LSUP_Buffer *sp,
- const LSUP_Buffer *so, const LSUP_Buffer *sc, void *th, size_t *ct)
- {
- MDBStore *store = h;
- LSUP_rc
- rc = LSUP_NOACTION,
- db_rc = 0;
- LSUP_Key ck = NULL_KEY;
- if (sc == NULL) sc = LSUP_default_ctx_buf;
- ck = LSUP_buffer_hash (sc);
- MDB_txn *txn;
- mdb_txn_begin (store->env, (MDB_txn *) th, 0, &txn);
- MDB_cursor *dcur, *icur;
- mdb_cursor_open (txn, store->dbi[IDX_SPO_C], &dcur);
- mdb_cursor_open (txn, store->dbi[IDX_C_SPO], &icur);
- MDB_val spok_v, ck_v;
- spok_v.mv_size = TRP_KLEN;
- ck_v.mv_size = KLEN;
- ck_v.mv_data = &ck;
- // The lookup operates within the current (bottom) write transaction.
- MDBIterator *it = mdbstore_lookup (store, ss, sp, so, sc, txn, ct);
- if (UNLIKELY (!it)) return LSUP_DB_ERR;
- if (ct) log_debug ("Found %lu triples to remove.", *ct);
- while (mdbiter_next_key (it) == LSUP_OK) {
- spok_v.mv_data = it->spok;
- db_rc = mdb_cursor_get (dcur, &spok_v, &ck_v, MDB_GET_BOTH);
- if (db_rc == MDB_NOTFOUND) continue;
- if (UNLIKELY (db_rc != MDB_SUCCESS)) goto fail;
- log_trace (
- "Removing {%lx, %lx, %lx}",
- it->spok[0], it->spok[1], it->spok[2]);
- // Delete spo:c entry.
- db_rc = mdb_cursor_del (dcur, 0);
- if (UNLIKELY (db_rc != MDB_SUCCESS)) goto fail;
- // Restore ck address after each delete.
- spok_v.mv_data = it->spok;
- ck_v.mv_data = &ck;
- // Delete c:spo entry.
- db_rc = mdb_cursor_get (icur, &ck_v, &spok_v, MDB_GET_BOTH);
- if (db_rc == MDB_NOTFOUND) continue;
- if (UNLIKELY (db_rc != MDB_SUCCESS)) goto fail;
- db_rc = mdb_cursor_del (icur, 0);
- if (UNLIKELY (db_rc != MDB_SUCCESS)) goto fail;
- spok_v.mv_data = it->spok;
- ck_v.mv_data = &ck;
- // If there are no more contexts associated with this triple,
- // remove from indices.
- db_rc = mdb_cursor_get (dcur, &spok_v, NULL, MDB_SET);
- if (db_rc == MDB_SUCCESS) continue;
- if (UNLIKELY (db_rc != MDB_NOTFOUND)) goto fail;
- rc = index_triple (store, OP_REMOVE, it->spok, ck, txn);
- }
- mdbiter_free (it);
- if (UNLIKELY (mdb_txn_commit (txn) != MDB_SUCCESS)) {
- rc = LSUP_TXN_ERR;
- goto fail;
- }
- return rc;
- fail:
- mdb_txn_abort (txn);
- log_error ("Database error: %s", LSUP_strerror (db_rc));
- return rc == LSUP_TXN_ERR ? rc : LSUP_DB_ERR;
- }
- #if 0
- static int
- mdbstore_tkey_exists (MDBStore *store, LSUP_Key tkey)
- {
- int db_rc, rc;
- MDB_val key, data;
- key.mv_data = &tkey;
- key.mv_size = KLEN;
- MDB_txn *txn = NULL;
- mdb_txn_begin (store->env, NULL, MDB_RDONLY, &txn);
- MDB_cursor *cur = NULL;
- mdb_cursor_open (txn, store->dbi[IDX_T_ST], &cur);
- db_rc = mdb_cursor_get (cur, &key, &data, MDB_SET);
- if (db_rc == MDB_SUCCESS) rc = 1;
- else if (db_rc == MDB_NOTFOUND) rc = 0;
- else {
- log_error ("DB error: %s", LSUP_strerror (db_rc));
- rc = LSUP_DB_ERR;
- }
- if (cur) mdb_cursor_close (cur);
- if (txn) mdb_txn_abort (txn);
- return rc;
- }
- #endif
- /** @brief Add a term to the store.
- *
- * @param[in] h #MDBStore handle.
- *
- * @param[in] sterm Serialized term to store.
- *
- * @param[in] ith #MDBIterator handle. Only the transaction handle inside this
- * is used. It may be NULL, in which case a new transaction is opened and
- * closed for the operation.
- *
- * @return LSUP_OK on success; <0 on error.
- */
- static LSUP_rc
- mdbstore_add_term (void *h, const LSUP_Buffer *sterm, void *ith)
- {
- //log_trace ("Adding term to MDB store: %s", sterm->addr);
- MDBStore *store = h;
- int db_rc;
- MDB_val key, data;
- MDBIterator *it = ith;
- MDB_txn *txn;
- // If a transaction is active in the iterator, use it, otherwise open and
- // close a new one.
- bool borrowed_txn = (it && it->txn);
- if (borrowed_txn) txn = it->txn;
- else RCCK (mdb_txn_begin (store->env, NULL, 0, &txn));
- MDB_cursor *cur;
- CHECK (mdb_cursor_open (txn, store->dbi[IDX_T_ST], &cur), fail);
- LSUP_Key k = LSUP_buffer_hash (sterm);
- key.mv_data = &k;
- key.mv_size = sizeof (k);
- data.mv_data = sterm->addr;
- data.mv_size = sterm->size;
- db_rc = mdb_cursor_put (cur, &key, &data, MDB_NOOVERWRITE);
- if (db_rc != MDB_KEYEXIST) CHECK (db_rc, fail);
- if (!borrowed_txn) CHECK (db_rc = mdb_txn_commit (txn), fail);
- return LSUP_OK;
- fail:
- if (!borrowed_txn) mdb_txn_abort (txn);
- log_trace ("Aborted txn for adding term.");
- return LSUP_DB_ERR;
- }
- const LSUP_StoreInt mdbstore_int = {
- .name = "MDB Store",
- .features = LSUP_STORE_PERM | LSUP_STORE_CTX | LSUP_STORE_IDX
- | LSUP_STORE_TXN | LSUP_STORE_COW,
- .setup_fn = mdbstore_setup,
- .new_fn = mdbstore_new,
- .free_fn = mdbstore_free,
- .size_fn = mdbstore_size,
- .txn_begin_fn = mdbstore_txn_begin,
- .txn_commit_fn = mdbstore_txn_commit,
- .txn_abort_fn = mdbstore_txn_abort,
- .iter_txn_fn = mdbiter_txn,
- .add_init_fn = mdbstore_add_init,
- .add_iter_fn = mdbstore_add_iter,
- .add_abort_fn = mdbstore_add_abort,
- .add_done_fn = mdbstore_add_done,
- .add_term_fn = mdbstore_add_term,
- .lookup_fn = mdbstore_lookup,
- .lu_next_fn = mdbiter_next,
- .lu_free_fn = mdbiter_free,
- .remove_fn = mdbstore_remove,
- .nsm_put_fn = mdbstore_nsm_put,
- .nsm_get_fn = mdbstore_nsm_get,
- };
- /* * * Static functions. * * */
- /** @brief Index an added or removed triple.
- *
- * @param store[in] MDB store to index.
- * @param op[in] Store operation. One of OP_ADD or OP_REMOVE.
- * @param spok[in] Triple key to index.
- * @param ck[in] Context to index, may be NULL.
- * @param[in] th Transaction handle. This MUST be a valid pointer to an open
- * RW transaction.
- */
- static LSUP_rc
- index_triple(
- MDBStore *store, StoreOp op, LSUP_TripleKey spok, LSUP_Key ck,
- MDB_txn *txn)
- {
- int db_rc;
- LSUP_rc rc = LSUP_NOACTION;
- MDB_val v1, v2;
- log_trace ("Indexing triple: {%lx %lx %lx}", spok[0], spok[1], spok[2]);
- // Index c:spo.
- if (op == OP_REMOVE) {
- log_trace ("Indexing op: REMOVE");
- if (ck != NULL_KEY) {
- MDB_cursor *cur;
- v1.mv_data = &ck;
- v1.mv_size = KLEN;
- v2.mv_data = spok;
- v2.mv_size = TRP_KLEN;
- mdb_cursor_open (txn, store->dbi[IDX_C_SPO], &cur);
- if (mdb_cursor_get (cur, &v1, &v2, MDB_GET_BOTH) == MDB_SUCCESS) {
- db_rc = mdb_cursor_del (cur, 0);
- if (db_rc != MDB_SUCCESS) return LSUP_DB_ERR;
- rc = LSUP_OK;
- }
- mdb_cursor_close (cur);
- }
- } else if (op == OP_ADD) {
- log_trace ("Indexing op: ADD");
- if (ck != NULL_KEY) {
- v1.mv_data = &ck;
- v1.mv_size = KLEN;
- v2.mv_data = spok;
- v2.mv_size = TRP_KLEN;
- db_rc = mdb_put(
- txn, store->dbi[IDX_C_SPO],
- &v1, &v2, MDB_NODUPDATA);
- if (db_rc != MDB_SUCCESS) return LSUP_DB_ERR;
- if (db_rc != MDB_KEYEXIST) rc = LSUP_OK;
- }
- } else return LSUP_VALUE_ERR;
- LSUP_DoubleKey dbl_keys[3] = {
- {spok[1], spok[2]}, // po
- {spok[0], spok[2]}, // so
- {spok[0], spok[1]}, // sp
- };
- // Add terms to index.
- v1.mv_size = KLEN;
- v2.mv_size = DBL_KLEN;
- for (int i = 0; i < 3; i++) {
- MDB_dbi db1 = store->dbi[lookup_indices[i]]; // s:po, p:so, o:sp
- MDB_dbi db2 = store->dbi[lookup_indices[i + 3]]; // po:s, so:p, sp:o
- v1.mv_data = spok + i;
- v2.mv_data = dbl_keys[i];
- if (op == OP_REMOVE) {
- MDB_cursor *cur1, *cur2;
- mdb_cursor_open(txn, store->dbi[lookup_indices[i]], &cur1);
- db_rc = mdb_cursor_get (cur1, &v1, &v2, MDB_GET_BOTH);
- if (db_rc == MDB_SUCCESS) mdb_cursor_del (cur1, 0);
- mdb_cursor_close (cur1);
- // Restore pointers invalidated after delete.
- v1.mv_data = spok + i;
- v2.mv_data = dbl_keys[i];
- mdb_cursor_open(txn, store->dbi[lookup_indices[i + 3]], &cur2);
- db_rc = mdb_cursor_get (cur2, &v2, &v1, MDB_GET_BOTH);
- if (db_rc == MDB_SUCCESS) mdb_cursor_del (cur2, 0);
- // TODO error handling.
- rc = LSUP_OK;
- mdb_cursor_close (cur2);
- } else { // OP_ADD is guaranteed.
- // 1-bound index.
- log_trace ("Indexing in %s: ", db_labels[lookup_indices[i]]);
- log_trace (
- "%lx: %lx %lx", *(size_t*)(v1.mv_data),
- *(size_t*)(v2.mv_data), *(size_t*)(v2.mv_data) + 1);
- db_rc = mdb_put (txn, db1, &v1, &v2, MDB_NODUPDATA);
- if (db_rc == MDB_SUCCESS) rc = LSUP_OK;
- else if (db_rc != MDB_KEYEXIST) return LSUP_DB_ERR;
- // 2-bound index.
- log_trace ("Indexing in %s: ", db_labels[lookup_indices[i + 3]]);
- log_trace (
- "%lx %lx: %lx", *(size_t*)(v2.mv_data),
- *(size_t*)(v2.mv_data) + 1, *(size_t*)(v1.mv_data));
- db_rc = mdb_put (txn, db2, &v2, &v1, MDB_NODUPDATA);
- if (db_rc == MDB_SUCCESS) rc = LSUP_OK;
- else if (db_rc != MDB_KEYEXIST) return LSUP_DB_ERR;
- }
- }
- return rc;
- }
- /* * * Term-specific iterators. * * */
- /** @brief Advance 0-bound iterator.
- *
- * Cursor: spo:c
- */
- inline static void
- it_next_0bound (MDBIterator *it)
- {
- memcpy (it->spok, it->key.mv_data, sizeof (LSUP_TripleKey));
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_NEXT);
- }
- /** @brief Advance 1-bound iterator.
- *
- * Uses paged data in a nested loop.
- *
- * Cursor: s:po, p:so, or o:sp.
- */
- inline static void
- it_next_1bound (MDBIterator *it)
- {
- LSUP_DoubleKey *lu_dset = it->data.mv_data;
- it->spok[it->term_order[0]] = it->luk[0];
- it->spok[it->term_order[1]] = lu_dset[it->i][0];
- it->spok[it->term_order[2]] = lu_dset[it->i][1];
- log_trace (
- "Composed triple: {%lx %lx %lx}",
- it->spok[0], it->spok[1], it->spok[2]);
- // Ensure next block within the same page is not beyond the last.
- if (it->i < it->data.mv_size / DBL_KLEN - 1) {
- it->i ++;
- //log_debug ("Increasing page cursor to %lu.", it->i);
- //log_debug ("it->rc: %d", it->rc);
- } else {
- // If the last block in the page is being yielded,
- // move cursor to beginning of next page.
- it->i = 0;
- //log_debug ("Reset page cursor to %lu.", it->i);
- it->rc = mdb_cursor_get (
- it->cur, &it->key, &it->data, MDB_NEXT_MULTIPLE);
- }
- }
- /** @brief Advance 2-bound iterator.
- *
- * Uses paged data in a nested loop.
- *
- * Cursor: po:s, so:p, or sp:o.
- */
- inline static void
- it_next_2bound (MDBIterator *it)
- {
- LSUP_Key *lu_dset = it->data.mv_data;
- it->spok[it->term_order[0]] = it->luk[0];
- it->spok[it->term_order[1]] = it->luk[1];
- it->spok[it->term_order[2]] = lu_dset[it->i];
- // Ensure next block within the same page is not beyond the last.
- if (it->i < it->data.mv_size / KLEN - 1)
- it->i ++;
- else {
- // If the last block in the page is being yielded,
- // move cursor to beginning of next page.
- it->i = 0;
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_NEXT_MULTIPLE);
- }
- }
- /** @brief Advance 3-bound iterator.
- *
- * This is a special case of 0÷1 results; either there was one matching triple,
- * which was already set in the first result, or there was none, i.e. it->rc is
- * already MDB_NOTFOUND and this function will not be called.
- */
- inline static void
- it_next_3bound (MDBIterator *it)
- {
- it->rc = MDB_NOTFOUND;
- }
- /* * * Term-specific lookups. * * */
- inline static LSUP_rc
- lookup_0bound (MDBIterator *it, size_t *ct)
- {
- log_debug ("Looking up 0 bound terms.");
- if (ct) {
- if (it->luc != NULL_KEY) {
- // Look up by given context.
- it->rc = mdb_cursor_open (
- it->txn, it->store->dbi[IDX_C_SPO], &it->cur);
- it->key.mv_data = &it->luc;
- it->key.mv_size = KLEN;
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (it->rc == MDB_SUCCESS) mdb_cursor_count (it->cur, ct);
- mdb_cursor_close (it->cur);
- it->cur = NULL;
- } else {
- // Look up all contexts.
- MDB_stat stat;
- mdb_stat (it->txn, it->store->dbi[IDX_S_PO], &stat);
- *ct = stat.ms_entries;
- }
- log_debug ("Found %lu keys.", *ct);
- }
- it->rc = mdb_cursor_open (it->txn, it->store->dbi[IDX_SPO_C], &it->cur);
- if (it->rc != MDB_SUCCESS) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_FIRST);
- /*
- mdb_cursor_close (it->cur);
- it->cur = NULL;
- */
- it->iter_op_fn = it_next_0bound;
- if (it->rc != MDB_SUCCESS && it->rc != MDB_NOTFOUND) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- return LSUP_OK;
- }
- inline static LSUP_rc
- lookup_1bound (uint8_t idx0, MDBIterator *it, size_t *ct)
- {
- it->term_order = (const uint8_t*)lookup_ordering_1bound[idx0];
- log_debug ("Looking up 1 bound term: %lx", it->luk[0]);
- mdb_cursor_open (it->txn, it->store->dbi[lookup_indices[idx0]], &it->cur);
- it->key.mv_data = it->luk;
- it->key.mv_size = KLEN;
- if (ct) {
- // If a context is specified, the only way to count triples matching
- // the context is to loop over them.
- if (it->luc != NULL_KEY) {
- log_debug ("Counting in context: %lx", it->luc);
- MDBIterator *ct_it;
- MALLOC_GUARD (ct_it, LSUP_MEM_ERR);
- /*
- memcpy (ct_it, it, sizeof (*ct_it));
- */
- ct_it->store = it->store;
- ct_it->txn = it->txn;
- ct_it->ctx_cur = it->ctx_cur;
- ct_it->key = it->key;
- ct_it->data = it->data;
- ct_it->ck = NULL;
- ct_it->luk[0] = it->luk[0];
- ct_it->luc = it->luc;
- ct_it->i = 0;
- LSUP_rc rc = lookup_1bound (idx0, ct_it, NULL);
- if (rc < 0) return rc;
- LSUP_rc db_rc;
- while (LSUP_END != (db_rc = mdbiter_next_key (ct_it))) {
- if (UNLIKELY (db_rc < 0)) return db_rc;
- (*ct)++;
- }
- // Free the counter iterator without freeing the shared txn.
- if (ct_it->cur) mdb_cursor_close (ct_it->cur);
- free (ct_it->ck);
- free (ct_it);
- } else {
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (it->rc == MDB_SUCCESS) mdb_cursor_count (it->cur, ct);
- }
- }
- it->i = 0;
- it->iter_op_fn = it_next_1bound;
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (it->rc == MDB_SUCCESS)
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_GET_MULTIPLE);
- if (it->rc != MDB_SUCCESS && it->rc != MDB_NOTFOUND) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- return LSUP_OK;
- }
- inline static LSUP_rc
- lookup_2bound(uint8_t idx0, uint8_t idx1, MDBIterator *it, size_t *ct)
- {
- uint8_t luk1_offset, luk2_offset;
- MDB_dbi dbi = 0;
- // Establish lookup ordering with some awkward offset math.
- for (int i = 0; i < 3; i++) {
- if (
- (
- idx0 == lookup_ordering_2bound[i][0] &&
- idx1 == lookup_ordering_2bound[i][1]
- ) || (
- idx0 == lookup_ordering_2bound[i][1] &&
- idx1 == lookup_ordering_2bound[i][0]
- )
- ) {
- it->term_order = (const uint8_t*)lookup_ordering_2bound[i];
- if (it->term_order[0] == idx0) {
- luk1_offset = 0;
- luk2_offset = 1;
- } else {
- luk1_offset = 1;
- luk2_offset = 0;
- }
- dbi = it->store->dbi[lookup_indices[i + 3]];
- log_debug (
- "Looking up 2 bound in %s",
- db_labels[lookup_indices[i + 3]]);
- break;
- }
- }
- if (dbi == 0) {
- log_error (
- "Values %d and %d not found in lookup keys.",
- idx0, idx1);
- return LSUP_VALUE_ERR;
- }
- // Compose term keys in lookup key.
- LSUP_DoubleKey luk;
- luk[luk1_offset] = it->luk[0];
- luk[luk2_offset] = it->luk[1];
- it->key.mv_data = luk;
- it->key.mv_size = DBL_KLEN;
- mdb_cursor_open (it->txn, dbi, &it->cur);
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (ct) {
- // If a context is specified, the only way to count triples matching
- // the context is to loop over them.
- if (it->luc != NULL_KEY) {
- MDBIterator *ct_it;
- MALLOC_GUARD (ct_it, LSUP_MEM_ERR);
- ct_it->store = it->store;
- ct_it->txn = it->txn;
- ct_it->ctx_cur = it->ctx_cur;
- ct_it->ck = NULL;
- ct_it->luk[0] = it->luk[0];
- ct_it->luk[1] = it->luk[1];
- ct_it->luc = it->luc;
- ct_it->i = 0;
- lookup_2bound (idx0, idx1, ct_it, NULL);
- while (mdbiter_next_key (ct_it) != LSUP_END) (*ct) ++;
- // Free the counter iterator without freeing the shared txn.
- if (ct_it->cur) mdb_cursor_close (ct_it->cur);
- free (ct_it->ck);
- free (ct_it);
- } else {
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (it->rc == MDB_SUCCESS) mdb_cursor_count (it->cur, ct);
- }
- }
- it->i = 0;
- it->iter_op_fn = it_next_2bound;
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_SET);
- if (it->rc == MDB_SUCCESS)
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_GET_MULTIPLE);
- if (it->rc != MDB_SUCCESS && it->rc != MDB_NOTFOUND) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- return LSUP_OK;
- }
- inline static LSUP_rc
- lookup_3bound (MDBIterator *it, size_t *ct)
- {
- log_debug (
- "Looking up 3 bound: {%lx, %lx, %lx}",
- it->luk[0], it->luk[1], it->luk[2]);
- it->key.mv_data = it->luk;
- if (it->luc != NULL_KEY) {
- it->rc = mdb_cursor_open (
- it->txn, it->store->dbi[IDX_SPO_C], &it->cur);
- it->key.mv_size = TRP_KLEN;
- it->data.mv_data = &it->luc;
- it->data.mv_size = KLEN;
- } else {
- it->rc = mdb_cursor_open (it->txn, it->store->dbi[IDX_S_PO], &it->cur);
- it->key.mv_size = KLEN;
- it->data.mv_data = it->luk + 1;
- it->data.mv_size = DBL_KLEN;
- }
- it->rc = mdb_cursor_get (it->cur, &it->key, &it->data, MDB_GET_BOTH);
- if (it->rc != MDB_SUCCESS && it->rc != MDB_NOTFOUND) {
- log_error ("Database error: %s", LSUP_strerror (it->rc));
- return LSUP_DB_ERR;
- }
- mdb_cursor_close (it->cur);
- it->cur = NULL;
- if (ct && it->rc == MDB_SUCCESS) *ct = 1;
- it->iter_op_fn = it_next_3bound;
- memcpy (it->spok, it->luk, sizeof (LSUP_TripleKey));
- return LSUP_OK;
- }
|