lsup_rdf/include/store_mdb.h

/** @file store_mdb.h
 *
 * @brief LMDB graph store backend.
 *
 * This module stores triples in a LMDB embedded store, optionally organized
 * into named graphs. The store is optimized and indexed for fast lookup of any
 * number of bound terms.
 *
 * The store must be first initialized once, to create the environment files
 * and folders as well as the internal databases, then it must be opened once
 * per session. Within that session multiple R/W operations can be performed
 * using transactions.
 *
 * Note that, even though the terms "graph", "context", etc. are used, no code
 * in this module checks for valid RDF data. In theory any term can be any
 * binary data. This allows using the store for non-RDF graph data.
 *
 * TODO more doc
 */


#ifndef _LSUP_STORE_MDB_H
#define _LSUP_STORE_MDB_H

#include "lmdb.h"
#include "triple.h"

typedef char DbLabel[8];
typedef struct MDBStore LSUP_MDBStore;

typedef LSUP_rc (*mdb_store_match_fn_t)(const LSUP_TripleKey spok, void *data);

// TODO Introduce compile-time LSUP_BIG_STORE option to define two store
// options: false: 64-bit hashes, uint32 keys, max 4G entries; true:
// 128-bit hashes, size_t keys, max MAX_SIZE entries, larger and slower.
// Ideally this could be specified at runtime to handle different stores with
// different sizes, but code could become more complex.


/** @brief Create the MDB environment and databases on disk.
 *
 * This function takes care of creaating the environment path if not existing,
 * and checking that it's a writable directory. If the path is not specified
 * in the LSUP_STORE_PATH environment variable, a default directory is used.
 *
 * @param[in,out] path Path of the suggested directory to use. It may be NULL,
 *  in which case it will be set either to the environment variable
 *  LSUP_STORE_PATH, or if that is not set, a default local path.
 */
LSUP_rc LSUP_store_setup(char **path/*, bool clear*/);


/** @brief Open an MDB store.
 *
 * The store must have been set up with #LSUP_store_setup.
 *
 * Some environment variables affect various store parameters:
 *
 * - LSUP_MDB_MAPSIZE Long int specifying the size of the memory map. Usually
 *   it is not necessary to modify this, unless one is operating under memory
 *   and disk constraints. The default map size is 1Tb.
 *
 * @param[in,out] store Uninitialized store struct pointer.
 *
 * @param[in] path MDB environment path. This must be the path given by
 * #LSUP_store_setup.
 *
 * @param[in] default_ctx Serialized URI to be used as a default context for
 *  triples inserted without a context specified. If NULL, the store operates
 *  in triple mode.
 */
LSUP_MDBStore *
LSUP_store_new(const char *path, const LSUP_Buffer *default_ctx);


/** @brief Close a store and free its handle.
 *
 * @param[in] store Store pointer.
 *
 */
void LSUP_store_free(LSUP_MDBStore *store);


/** @brief Print stats about a store and its databases.
 *
 * TODO
 *
 * @param store[in] The store to get stats for.
 */
LSUP_rc LSUP_store_stats(LSUP_MDBStore *store);


/** @brief Store size.
 *
 * @param store[in] The store to calculate size of.
 *
 * @return Number of stored SPO triples across all contexts.
 */
size_t LSUP_store_size(LSUP_MDBStore *store);


/** @brief Add a batch of triples with optional context to the store.
 *
 * @param store[in] The store to add to.
 *
 * @param sc[in] Context as a serialized term. If this is NULL, and the
 *  default context is not NULL, triples will be added to the default context
 *  for the store, If the default context for the store is NULL, regardless of
 *  the value of sc, triples will be added with no context.

 * @param data[in] Triples to be inserted as a 2D array of triples in the shape
 * of data[n][3], where n is the value of data_size.
 *
 * @param data_size[in] Number of triples to be inserted.
 */
LSUP_rc LSUP_store_add(
        struct MDBStore *store, const LSUP_Buffer *sc,
        const LSUP_SerTriple *data, const size_t data_size);


/** @brief Look up matching triples and optional context.
 *
 * This function may return a count of matches and/or run a callback function
 * on each of the matching triples.
 *
 * @param store[in] The store to be queried.
 *
 * @param sspoc Array of 4 serialized term pointers representing the s, p, o, c
 * terms. Any and all of these may be NULL, which indicates an unbound query
 * term. Stores with context not set will always ignore the fourth term.
 *
 * @param ct[out] If not NULL, this will be populated with the number of
 *  entries found. This can be used when calling this function once to
 *  pre-allocate some memory based on the results obtained, then again to
 *  perform an operation on the allocated memory.
 *
 * @param callback[in] This function is applied to the matching
 *  triple keys. It may be NULL if the function is called only to count the
 *  matches; which in most cases will be much faster.
 *
 * @param data[in,out] Arbitrary data to be used in the callback function.
 *
 * @return LSUP_OK if entries were found, LSUP_NORESULT if none were found; if
 *  a callback was applied, and an error (<0) code was returned in any of the
 *  interactions, that code is returned and the callback execution is stopped.
 */
LSUP_rc LSUP_store_lookup(
        LSUP_MDBStore *store, LSUP_SerTerm *sspoc[], size_t *ct,
        mdb_store_match_fn_t callback_fn, void *data);



/** @brief Contexts that a triple key appears in.
 *
 * This function is most conveniently used by a callback to #LSUP_store_lookup
 * because it handles triple keys.
 *
 * @param store[in] The store to be queried.
 *
 * @param spok[in] Triple key to look up.
 *
 * @param ck[out] Pointer to an array of contexts. Memory is allocated by this
 * function and must be freed by the caller.
 *
 * @param ct[out] Number of contexts found.
 */
LSUP_rc LSUP_store_triple_contexts(
        LSUP_MDBStore *store, LSUP_Key spok[], LSUP_Key **ck, size_t *ct);

#endif