|
@@ -1,5 +1,3 @@
|
|
|
-#include <sys/stat.h>
|
|
|
-#include <errno.h>
|
|
|
#include <ftw.h>
|
|
|
|
|
|
#include "store_mdb.h"
|
|
@@ -19,6 +17,16 @@
|
|
|
*/
|
|
|
#define N_DB 12
|
|
|
|
|
|
+/**
|
|
|
+ * Memory map size.
|
|
|
+ */
|
|
|
+#if !(defined __LP64__ || defined __LLP64__) || \
|
|
|
+ defined _WIN32 && !defined _WIN64
|
|
|
+ #define DEFAULT_MAPSIZE 1<<31 // 2Gb (limit for 32-bit systems)
|
|
|
+#else
|
|
|
+ #define DEFAULT_MAPSIZE 1UL<<40 // 1Tb
|
|
|
+#endif
|
|
|
+
|
|
|
#define DEFAULT_ENV_PATH "./mdb_store"
|
|
|
#define ENV_DIR_MODE 0750
|
|
|
#define ENV_FILE_MODE 0640
|
|
@@ -31,7 +39,6 @@ typedef char DbLabel[8];
|
|
|
typedef enum {
|
|
|
LSSTORE_INIT = 1, // Is the store environment set up on disk?
|
|
|
LSSTORE_OPEN = 3, // Is the environment open? Assumes init is set.
|
|
|
- LSSTORE_DB_CREATED = 5, // Are DBs created? Assumes init is set.
|
|
|
} StoreState;
|
|
|
|
|
|
|
|
@@ -61,6 +68,7 @@ struct MatchArgs {
|
|
|
void *ctx;
|
|
|
};
|
|
|
|
|
|
+static LSUP_Buffer *default_ctx = NULL; // Default context URI for quad store.
|
|
|
|
|
|
/**
|
|
|
* Main DBs. These are the master information containers.
|
|
@@ -166,7 +174,6 @@ static const uint8_t lookup_ordering_2bound[3][3] = {
|
|
|
/**
|
|
|
* Static prototypes.
|
|
|
*/
|
|
|
-static int dbi_init(LSUP_MDBStore *store);
|
|
|
static int index_triple(
|
|
|
LSUP_MDBStore *store, StoreOp op,
|
|
|
LSUP_TripleKey spok, LSUP_Key ck);
|
|
@@ -199,44 +206,44 @@ LSUP_store_setup(char **path/*, bool clear*/) // TODO clear
|
|
|
int rc;
|
|
|
|
|
|
// Set environment path.
|
|
|
- if (path == NULL || (*path = getenv("LSUP_STORE_PATH")) == NULL) {
|
|
|
- printf(
|
|
|
- "WARNING: `LSUP_STORE_PATH' environment variable is not set. "
|
|
|
- "A local directory will be used to store the LSUP data.");
|
|
|
+ if (path == NULL && (*path = getenv("LSUP_STORE_PATH")) == NULL) {
|
|
|
+ // FIXME This won't work for multiple graphs with different disk
|
|
|
+ // back ends. A random path generator needs to be used.
|
|
|
*path = DEFAULT_ENV_PATH;
|
|
|
+ fprintf(
|
|
|
+ stderr,
|
|
|
+ "WARNING: `LSUP_STORE_PATH' environment variable is not set. "
|
|
|
+ "The default location %s will be used as the graph store.\n",
|
|
|
+ *path);
|
|
|
}
|
|
|
|
|
|
// Verify that a writable directory exists or can be created.
|
|
|
struct stat path_stat;
|
|
|
|
|
|
- rc = stat(*path, &path_stat);
|
|
|
- if (rc == ENOENT) {
|
|
|
+ /*
|
|
|
+ // TODO clear
|
|
|
+ if (clear) {
|
|
|
+ rmrf(*path);
|
|
|
if (mkdir(*path, ENV_DIR_MODE) != 0) abort();
|
|
|
-
|
|
|
- } else if (!S_ISDIR(path_stat.st_mode)) {
|
|
|
- printf("%s is not a valid directory.\n", path);
|
|
|
- abort();
|
|
|
-
|
|
|
- /* TODO clear
|
|
|
- } else {
|
|
|
- if (clear) {
|
|
|
- rmrf(*path);
|
|
|
- if (mkdir(*path, ENV_DIR_MODE) != 0) abort();
|
|
|
- }
|
|
|
- */
|
|
|
}
|
|
|
+ */
|
|
|
+
|
|
|
+ if (mkdir_p(*path, ENV_DIR_MODE) != 0) abort();
|
|
|
|
|
|
// Open a temporary environment and txn to create the DBs.
|
|
|
MDB_env *env;
|
|
|
mdb_env_create(&env);
|
|
|
+
|
|
|
+ mdb_env_set_maxdbs(env, N_DB);
|
|
|
mdb_env_open(env, *path, 0, ENV_FILE_MODE);
|
|
|
|
|
|
MDB_txn *txn;
|
|
|
mdb_txn_begin(env, NULL, 0, &txn);
|
|
|
for (int i = 0; i < N_DB; i++) {
|
|
|
+ TRACE("Creating DB %s", db_labels[i]);
|
|
|
MDB_dbi dbi;
|
|
|
- mdb_dbi_open(txn, db_labels[i], db_flags[i] | MDB_CREATE, &dbi);
|
|
|
- mdb_dbi_close(env, dbi);
|
|
|
+ rc = mdb_dbi_open(txn, db_labels[i], db_flags[i] | MDB_CREATE, &dbi);
|
|
|
+ if (rc != MDB_SUCCESS) return rc;
|
|
|
}
|
|
|
|
|
|
mdb_txn_commit(txn);
|
|
@@ -246,11 +253,15 @@ LSUP_store_setup(char **path/*, bool clear*/) // TODO clear
|
|
|
}
|
|
|
|
|
|
|
|
|
-LSUP_rc
|
|
|
-LSUP_store_open(
|
|
|
- LSUP_MDBStore *store, const char *path, LSUP_Buffer *default_ctx)
|
|
|
+LSUP_MDBStore *
|
|
|
+LSUP_store_init(const char *path, LSUP_Buffer *default_ctx)
|
|
|
{
|
|
|
- if(store->state & LSSTORE_OPEN) return LSUP_NOACTION;
|
|
|
+ int rc;
|
|
|
+ LSUP_MDBStore *store;
|
|
|
+ CRITICAL(store = malloc(sizeof(LSUP_MDBStore)));
|
|
|
+
|
|
|
+ rc = mdb_env_create(&store->env);
|
|
|
+ TRACE("create rc: %d", rc);
|
|
|
|
|
|
if (default_ctx == NULL) store->default_ctx = NULL;
|
|
|
else {
|
|
@@ -263,29 +274,37 @@ LSUP_store_open(
|
|
|
char *env_mapsize = getenv("LSUP_MDB_MAPSIZE");
|
|
|
|
|
|
if (env_mapsize == NULL) {
|
|
|
- mapsize = 0x10000000000; // 1Tb
|
|
|
+ mapsize = DEFAULT_MAPSIZE;
|
|
|
} else {
|
|
|
sscanf(env_mapsize, "%lu", &mapsize);
|
|
|
}
|
|
|
- mdb_env_set_mapsize(store->env, mapsize);
|
|
|
+ TRACE("mapsize rc: %d", rc);
|
|
|
+ if(rc != MDB_SUCCESS) return NULL;
|
|
|
|
|
|
- mdb_env_set_maxdbs(store->env, N_DB);
|
|
|
+ rc = mdb_env_set_maxdbs(store->env, N_DB);
|
|
|
+ if(rc != MDB_SUCCESS) return NULL;
|
|
|
|
|
|
- int rc = mdb_env_open(store->env, path, 0, ENV_FILE_MODE);
|
|
|
+ rc = mdb_env_open(store->env, path, 0, ENV_FILE_MODE);
|
|
|
+ if (rc != MDB_SUCCESS) return NULL;
|
|
|
|
|
|
// Assign DB handles to store->dbi.
|
|
|
MDB_txn *txn;
|
|
|
mdb_txn_begin(store->env, NULL, 0, &txn);
|
|
|
for (int i = 0; i < N_DB; i++) {
|
|
|
- mdb_dbi_open(
|
|
|
+ rc = mdb_dbi_open(
|
|
|
txn, db_labels[i], db_flags[i], store->dbi + i);
|
|
|
+ if (rc != MDB_SUCCESS) {
|
|
|
+ mdb_txn_abort(txn);
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
mdb_txn_commit(txn);
|
|
|
|
|
|
- store->state = LSSTORE_OPEN;
|
|
|
+ store->state |= LSSTORE_OPEN;
|
|
|
+ store->txn = NULL;
|
|
|
|
|
|
- return rc;
|
|
|
+ return store;
|
|
|
}
|
|
|
|
|
|
|
|
@@ -313,14 +332,13 @@ LSUP_store_size(LSUP_MDBStore *store)
|
|
|
LSUP_rc
|
|
|
LSUP_store_add(
|
|
|
LSUP_MDBStore *store, const LSUP_Buffer *sc,
|
|
|
- const LSUP_SerTerm **data, const size_t data_size)
|
|
|
+ const LSUP_SerTriple *data, const size_t data_size)
|
|
|
{
|
|
|
-
|
|
|
MDB_val key_v, data_v;
|
|
|
|
|
|
bool txn_pending = false;
|
|
|
if (!store->txn) {
|
|
|
- mdb_txn_begin(store->env, NULL, MDB_RDONLY, &store->txn);
|
|
|
+ mdb_txn_begin(store->env, NULL, 0, &store->txn);
|
|
|
txn_pending = true;
|
|
|
}
|
|
|
|
|
@@ -334,7 +352,7 @@ LSUP_store_add(
|
|
|
ck = LSUP_sterm_to_key(sc);
|
|
|
|
|
|
// Insert t:st for context.
|
|
|
- TRACE("Adding context: %s", sc);
|
|
|
+ //TRACE("Adding context: %s", sc);
|
|
|
key_v.mv_data = &ck;
|
|
|
key_v.mv_size = KLEN;
|
|
|
data_v.mv_data = sc->addr;
|
|
@@ -345,74 +363,62 @@ LSUP_store_add(
|
|
|
&key_v, &data_v, MDB_NOOVERWRITE);
|
|
|
}
|
|
|
|
|
|
- int rc = LSUP_NOACTION;
|
|
|
- for (size_t i = 0; i < data_size && rc == LSUP_OK; i++) {
|
|
|
- const LSUP_SerTerm *sspo = data[i];
|
|
|
- /*
|
|
|
- LSUP_SerTerm sspo[3];
|
|
|
-
|
|
|
- CHECK(LSUP_term_serialize(spo->s, sspo), _add_free_sterm1);
|
|
|
- CHECK(LSUP_term_serialize(spo->p, sspo + 1), _add_free_sterm2);
|
|
|
- CHECK(LSUP_term_serialize(spo->s, sspo + 2), _add_free_sterms);
|
|
|
- */
|
|
|
-
|
|
|
+ LSUP_rc rc = LSUP_NOACTION;
|
|
|
+ int db_rc;
|
|
|
+ for (size_t i = 0; i < data_size; i++) {
|
|
|
+ const LSUP_SerTriple *sspo = data + i;
|
|
|
LSUP_TripleKey spok = NULL_TRP;
|
|
|
|
|
|
// Add triple.
|
|
|
- TRACE("Inserting spok: {%lx, %lx, %lx}", spok[0], spok[1], spok[2]);
|
|
|
-
|
|
|
- int put_rc[2] = {LSUP_OK, MDB_KEYEXIST};
|
|
|
- // Insert t:st for s, p, o
|
|
|
for (int j = 0; j < 3; j++) {
|
|
|
- spok[i] = LSUP_sterm_to_key(sspo + i);
|
|
|
+ LSUP_SerTerm *st = LSUP_ser_triple_term_by_pos(sspo, j);
|
|
|
+
|
|
|
+ printf("Inserting term: ");
|
|
|
+ LSUP_buffer_print(st);
|
|
|
+ printf("\n");
|
|
|
|
|
|
- key_v.mv_data = spok + i;
|
|
|
+ spok[j] = LSUP_sterm_to_key(st);
|
|
|
+
|
|
|
+ key_v.mv_data = spok + j;
|
|
|
key_v.mv_size = KLEN;
|
|
|
- data_v.mv_data = (sspo + i)->addr;
|
|
|
- data_v.mv_size = (sspo + i)->size;
|
|
|
-
|
|
|
- MCHECK(
|
|
|
- mdb_put(
|
|
|
- store->txn, store->dbi[IDX_T_ST],
|
|
|
- &key_v, &data_v, MDB_NOOVERWRITE),
|
|
|
- put_rc, _add_close_txn);
|
|
|
+ data_v.mv_data = st->addr;
|
|
|
+ data_v.mv_size = st->size;
|
|
|
+
|
|
|
+ db_rc = mdb_put(
|
|
|
+ store->txn, store->dbi[IDX_T_ST],
|
|
|
+ &key_v, &data_v, MDB_NOOVERWRITE);
|
|
|
+ if (db_rc == MDB_SUCCESS) rc = LSUP_OK;
|
|
|
+ else if (db_rc != MDB_KEYEXIST) goto _add_close_txn;
|
|
|
}
|
|
|
|
|
|
- // Insert spo:c
|
|
|
+ TRACE("Inserting spok: {%lx, %lx, %lx}", spok[0], spok[1], spok[2]);
|
|
|
+
|
|
|
+ // Insert spo:c.
|
|
|
key_v.mv_data = spok;
|
|
|
key_v.mv_size = TRP_KLEN;
|
|
|
- // In triple mode, data is empty.
|
|
|
- data_v.mv_data = ck == NULL_KEY ? NULL : &ck;
|
|
|
+
|
|
|
+ // In triple mode, data is empty (= NULL_KEY).
|
|
|
+ data_v.mv_data = &ck;
|
|
|
data_v.mv_size = ck == NULL_KEY ? 0 : KLEN;
|
|
|
|
|
|
- MCHECK(
|
|
|
- mdb_put(
|
|
|
- store->txn, store->dbi[IDX_SPO_C],
|
|
|
- &key_v, &data_v, MDB_NODUPDATA),
|
|
|
- put_rc, _add_close_txn);
|
|
|
|
|
|
- // Index.
|
|
|
- PCHECK(index_triple(store, OP_ADD, spok, ck), _add_close_txn);
|
|
|
+ db_rc = mdb_put(
|
|
|
+ store->txn, store->dbi[IDX_SPO_C],
|
|
|
+ &key_v, &data_v, MDB_NODUPDATA);
|
|
|
+ if (db_rc == MDB_SUCCESS) rc = LSUP_OK;
|
|
|
+ else if (db_rc != MDB_KEYEXIST) goto _add_close_txn;
|
|
|
|
|
|
- // Free serialized terms.
|
|
|
-/*
|
|
|
-_add_free_sterms:
|
|
|
- LSUP_buffer_done(sspo + 2);
|
|
|
-_add_free_sterm2:
|
|
|
- LSUP_buffer_done(sspo + 1);
|
|
|
-_add_free_sterm1:
|
|
|
- LSUP_buffer_done(sspo);
|
|
|
-*/
|
|
|
+ // Index.
|
|
|
+ PCHECK(index_triple(store, OP_ADD, spok, ck), db_rc, _add_close_txn);
|
|
|
}
|
|
|
|
|
|
_add_close_txn:
|
|
|
// Only return commit rc if it fails.
|
|
|
if (txn_pending) {
|
|
|
- if (rc >= LSUP_OK) {
|
|
|
- int txn_rc;
|
|
|
- if((txn_rc = mdb_txn_commit(store->txn)) != MDB_SUCCESS) {
|
|
|
+ if (rc == LSUP_OK) {
|
|
|
+ if((db_rc = mdb_txn_commit(store->txn)) != MDB_SUCCESS) {
|
|
|
mdb_txn_abort(store->txn);
|
|
|
- rc = txn_rc;
|
|
|
+ rc = db_rc;
|
|
|
}
|
|
|
} else mdb_txn_abort(store->txn);
|
|
|
|
|
@@ -641,34 +647,6 @@ static int rmrf(char *path)
|
|
|
*/
|
|
|
|
|
|
|
|
|
-/**
|
|
|
- * @brief Open and allocate DB handles in an array.
|
|
|
- *
|
|
|
- * @param bool create [in]: If true, the DBs are created. This is only needed
|
|
|
- * on bootstrap.
|
|
|
- */
|
|
|
-static LSUP_rc dbi_init(LSUP_MDBStore *store)
|
|
|
-{
|
|
|
- bool db_created = store->state & LSSTORE_DB_CREATED;
|
|
|
- MDB_txn *txn;
|
|
|
- unsigned int txn_flags = db_created ? MDB_RDONLY : 0;
|
|
|
- unsigned int create_flag = db_created ? 0 : MDB_CREATE;
|
|
|
-
|
|
|
- mdb_txn_begin(store->env, NULL, txn_flags, &txn);
|
|
|
-
|
|
|
- for (int i = 0; i < N_DB; i++) {
|
|
|
- mdb_dbi_open(
|
|
|
- txn, db_labels[i], db_flags[i] | create_flag, store->dbi + i);
|
|
|
- }
|
|
|
-
|
|
|
- store->state |= LSSTORE_DB_CREATED;
|
|
|
-
|
|
|
- mdb_txn_commit(txn);
|
|
|
-
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
/* TODO
|
|
|
inline static int
|
|
|
check_txn_open(MDB_txn *txn, bool write)
|
|
@@ -732,9 +710,10 @@ index_triple(
|
|
|
v1.mv_size = KLEN;
|
|
|
v2.mv_size = DBL_KLEN;
|
|
|
|
|
|
+ int db_rc;
|
|
|
for (int i = 0; i < 3; i++) {
|
|
|
- MDB_dbi db1 = lookup_indices[i]; // s:po, p:so, o:sp
|
|
|
- MDB_dbi db2 = lookup_indices[i + 3]; // po:s, so:p, sp:o
|
|
|
+ MDB_dbi db1 = store->dbi[lookup_indices[i]]; // s:po, p:so, o:sp
|
|
|
+ MDB_dbi db2 = store->dbi[lookup_indices[i + 3]]; // po:s, so:p, sp:o
|
|
|
|
|
|
v1.mv_data = spok + i;
|
|
|
v2.mv_data = dbl_keys[i];
|
|
@@ -761,11 +740,16 @@ index_triple(
|
|
|
mdb_cursor_close(cur2);
|
|
|
|
|
|
} else { // OP_ADD is guaranteed.
|
|
|
- mdb_put(store->txn, db1, &v1, &v2, MDB_NODUPDATA);
|
|
|
- mdb_put(store->txn, db2, &v2, &v1, MDB_NODUPDATA);
|
|
|
+ CHECK(
|
|
|
+ mdb_put(store->txn, db1, &v1, &v2, MDB_NODUPDATA),
|
|
|
+ db_rc, _index_triple_exit);
|
|
|
+ CHECK(
|
|
|
+ mdb_put(store->txn, db2, &v2, &v1, MDB_NODUPDATA),
|
|
|
+ db_rc, _index_triple_exit);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+_index_triple_exit:
|
|
|
return rc;
|
|
|
}
|
|
|
|