lsup_rdf/include/store.h

/** @file store.h
 *
 * @brief Store back end interfaces.
 *
 * The basic interfaces for store and store iterator implementations are
 * defined here. New store implementations should include this header and
 * implement four basic elements:
 *
 * - A structure representing the store back end and one for the iterator.
 *   These structures will be opaque to all external modules and their layouts
 *   are entirely up to the implementer.
 *
 * - The LSUP_StoreInt and LSUP_StoreIteratorInt interfaces as well as the
 *   functions defined in the interfaces necessary to interact with the store.
 *
 * See the `store_htable.{c,h}` and `store_mdb.{c,h}` files for examples of
 * fully functioning implementations.
 *
 * The #LSUP_Store structure defines a store back end for raw buffer triples.
 * Nothing in the store should hint at RDF triples—it should accept and output
 * exclusively raw byte buffers (LSUP_Buffer). A store may have any of the
 * `LSUP_STORE_*` faeture flags which should be reflected by how its members
 * are implemented.
 *
 * The #LSUP_StoreIterator structure defines a store iterator implementation
 * that is designed to work with the store implementation. It must define a
 * "next" function for retrieving a triple in a lookup and a "free" function
 * to deallocate the iterator after use. Functions for creating iterators for
 * lookup and incremental addition are defined in the associated store.
 */


#ifndef _LSUP_STORE_H
#define _LSUP_STORE_H

#include "namespace.h"
#include "data/bootstrap.h"


/*
 * Store feature flags.
 *
 * NOTE: LSUP_STORE_PERM need only be set by an implementation based on whether
 * its path is on a default temporary dir (e.g. LSUP_MDB_RAMDISK_PATH). If this
 * flag is not set, it means the data will be cleared before the next execution
 * of the program. However, its being set does not guarantee the persistence of
 * the medium (i.e. a "permanent" store may have been created ad hoc on a
 * tempfs).
 */
#define LSUP_STORE_PERM     1<<0    /// Store is on a permanent location.
#define LSUP_STORE_CTX      1<<1    /// Store supports contexts (quads).
#define LSUP_STORE_IDX      1<<2    /// Store is fully SPO(C)-indexed.
#define LSUP_STORE_TXN      1<<3    /// Supports manual transaction handling.
#define LSUP_STORE_NET      1<<4    /// Store is over a network protocol.


/*
 * Iterator function types.
 */

/** @brief Prototype: yield the matching triples and advance the iterator.
 *
 * NOTE: Iterators keep transactions open. Don't hold on to them longer than
 * necessary.
 *
 * NOTE: The memory pointed to by the individual LSUP_Buffer pointers is
 * owned by the database. It must not be written to or freed. To modify
 * the data or use them beyond the caller's scope, this memory must be copied.
 *
 * @param[in] it Opaque iterator handle obtained with the store's #lookup_fn.
 *
 * @param[out] sspo #LSUP_BufferTriple to be populated with three serialized
 * terms if found, NULL if not found. Internal callers (e.g. counters) may pass
 * NULL if they don't need the serialized triples.
 *
 * @param[out] ctx If not NULL, it is populated with a NULL-terminated array of
 *  LSUP_Buffer structs, one for each context associated with the matching
 *  triple. These contexts are the same regardless of the context filter used
 *  in the lookup. The array is freed with a simple #free(). This parameter
 *  is ignored by implementations without the LSUP_STORE_CTX feature.
 *
 *  To iterate over the context array, use this loop:
 *
 *      size_t i = 0;
 *      while (ctx[i].addr)
 *          do_something(ctx + i++); // Buffer data are RO.
 *
 * @return LSUP_OK if results were found; LSUP_END if no (more) results were
 * found; LSUP_DB_ERR if a backend error occurred.
 */
typedef LSUP_rc (*iter_next_fn_t)(
        void *it, LSUP_BufferTriple *sspo, LSUP_Buffer **ctx);


/** @brief Prototype: free an iterator allocated by a lookup.
 *
 * @param it[in] Iterator pointer. It will be set to NULL after freeing.
 */
typedef void (*iter_free_fn_t)(void * it);


/*
 * Store function types.
 */

/** @brief Prototype: create any environment necessary for the store to work.
 *
 * @param[in] clear Whether to remove a previous environment at this location.
 *
 * @param[in,out] path Path of the suggested directory to use. It may be NULL,
 *  in which case it will be set either to the environment variable
 *  LSUP_STORE_PATH, or if that is not set, a default local path.
 */
typedef LSUP_rc (*store_setup_fn_t)(const char *path, bool clear);


/** @brief Prototype: create a new store.
 *
 * @param[in] Location for the new store. This depends on the implementation,
 *  which may ignore it (e.g. an in-memory store). Implementations should
 *  provide instructions on how to use and interpret this parameter.
 *
 * @param[in] default_ctx Default context IRI (serialized). It should only be
 *  considered by implementations with the LSUP_STORE_CTX feature.
 *
 * @return New store handle.
 */
typedef void * (*store_new_fn_t)(
        const char *location, const LSUP_Buffer *default_ctx);


/** @brief Prototype: free store handle.
 *
 * @param[in] store Store handle.
 *
 */
typedef void (*store_free_fn_t)(void *store);


/** @brief Prototype: get store size.
 *
 * @param store[in] The store to calculate size of.
 *
 * @return Number of stored SPO triples (across all contexts if supported).
 */
typedef size_t (*store_size_fn_t)(void *store);


/** @brief Print stats about a store.
 *
 * TODO
 *
 * @param store[in] The store to get stats for.
 */
/* TODO
typedef LSUP_rc (*store_stat_fn_t)(void *store, void *stat);
*/


/** @brief Initialize bulk triple load.
 *
 * This is the first step of a bulk load. It is best used when the data at hand
 * need to be pre-processed, which can be done in the same loop as the next
 * step to keep memory usage low.
 *
 * @param store[in] The store to add to.
 *
 * @param sc[in] Context as a serialized term. If this is NULL, and the
 *  default context is not NULL, triples will be added to the default context
 *  for the store, If the default context for the store is NULL, regardless of
 *  the value of sc, triples will be added with no context. Only meaningful
 *  for stores with the LSUP_STORE_CTX feature.
 *
 * @return Iterator handle to be passed to the following load steps.
 */
typedef void * (*store_add_init_fn_t)(void *store, const LSUP_Buffer * sc);


/** @brief Add one triple into the store.
 *
 * This must be called after #add_init_fn, using the iterator
 * yielded by that function. It may be called multiple times and must be
 * followed by #add_done_fn or #add_abort_fn (if supported).
 *
 * @param it[in] Iterator obtained by #LSUP_mdbstore_add_init.
 *  The following members are of interest:
 *  it->i stores the total number of records inserted.
 *
 * @param sspo[in] Serialized triple to be added.
 *
 * @return LSUP_OK if the triple was inserted; LSUP_NOACTION if the triple
 *  already existed; LSUP_DB_ERR if an MDB error occurred.
 */
typedef LSUP_rc (*store_add_iter_fn_t)(void *it, const LSUP_BufferTriple * sspo);


/** @brief Abort an add loop and free iterator.
 *
 * Usually called on an irrecoverable error from #add_iter_fn. None of the
 * successful inserts in the same loop is retained.
 *
 * @param it[in] Iterator obtained by #LSUP_mdbstore_add_init.
 */
typedef void (*store_add_abort_fn_t)(void *it);


/** @brief Finalize an add loop and free iterator.
 *
 * This must be called after #add_iter_fn.
 *
 * @param it[in] Iterator obtained by #LSUP_mdbstore_add_init.
 */
typedef LSUP_rc (*store_add_done_fn_t)(void *it);


/** @brief Add a single term to the store.
 *
 * @param[in] store Store handle.
 *
 * @param[in] sterm Serialized term to store.
 */
typedef LSUP_rc (*store_add_term_fn_t)(void *store, const LSUP_Buffer *sterm);


/** @brief Prototype: look up triples by pattern matching.
 *
 * This function may return a count of matches and/or an iterator of results as
 * serialized triples.
 *
 * Any and all of the terms may be NULL, which indicates an unbound query
 * term. Stores with context not set or witout context support will always
 * ignore the fourth term.
 *
 * @param[in] store The store to be queried.
 *
 * @param[in] ss Buffer representing the serialized s term.
 *
 * @param[in] sp Buffer representing the serialized p term.
 *
 * @param[in] so Buffer representing the serialized o term.
 *
 * @param[in] sc Serialized context to limit search to. It may be NULL, in
 * which case search is done in all contexts. Note that triples inserted
 * without context are assigned the *default* context for the store.
 *
 * @param[out] ct If not NULL, this will be populated with the number of
 *  entries found. It is very inexpensive to set for lookups without context,
 *  much less so for 1-bound and 2-bound context lookups, in which cases it
 *  should be set only if needed.
 *
 * @return Iterator handle that will be populated with a result iterator. This
 * is always created even if no matches are found and must be freed with
 * #LSUP_mdbiter_free() after use. If matches are found, the iterator points to
 * the first result which can be retrieved with #iter_next_fn().
 */
typedef void * (*store_lookup_fn_t)(
        void *store,
        const LSUP_Buffer *ss, const LSUP_Buffer *sp, const LSUP_Buffer *so,
        const LSUP_Buffer *sc, size_t *ct);


/** @brief Prototype: delete triples by pattern matching.
 *
 * The ss, sp, so, sc terms act as a matching pattern as documented in
 * #store_lookup_fn. if not NULL, ct yields the number of triples actually
 * deleted.
 */
typedef LSUP_rc (*store_remove_fn_t)(
        void *store,
        const LSUP_Buffer *ss, const LSUP_Buffer *sp, const LSUP_Buffer *so,
        const LSUP_Buffer *sc, size_t *ct);


/** @brief Put an in-memory namespace map into a permanent back end.
 *
 * This is only available in stores with the LSUP_STORE_PERM feature.
 *
 * Existing prefixes and namespaces are not updated. Thus, if the following are
 * already stored:
 *
 * ns1: <urn:ns:a#>
 * ns2: <urn:ns:b#>
 *
 * Neither of the following will be inserted:
 *
 * ns3: <urn:ns:a#>
 * ns2: <urn:ns:c#>
 *
 * @param[in] store MDB store to update.
 *
 * @param[out] nsm Namespace map handle to store.
 *
 * @return LSUP_OK if all terms were updated; LSUP_CONFLICT if one or more
 *  namespaces or terms were not updated because they already existed; <0 if
 *  an error occurred.
 */
typedef LSUP_rc (*store_nsm_put_fn_t)(void *store, const LSUP_NSMap * nsm);


/** @brief Get the store's namespace prefix map.
 *
 * @param[in] store MDB store to query.
 *
 * @return NS map or NULL on error.
 */
typedef LSUP_NSMap * (*store_nsm_get_fn_t)(void *store);


/*
 * Iterface type definitions.
 */

/** @brief Store interface.
 */
typedef struct store_t {
    // Basic properties.
    char                name[16];       ///> Store type name.
    int                 features;       ///> Feature flags.

    // Allocation, setup and deallocation.
    store_setup_fn_t    setup_fn;       ///> Optional function called before
                                        ///>    new_fn.
    store_new_fn_t      new_fn;         ///> Create a new store instance.
    store_free_fn_t     free_fn;        ///> Free the store.

    // Metadata.
    store_size_fn_t     size_fn;        ///> Number of triples in the store.
    //store_stat_fn       stat_fn;        ///> Store statistics. Optional.

    // Addition.
    store_add_init_fn_t add_init_fn;    ///> Initialize add iteration.
    store_add_iter_fn_t add_iter_fn;    ///> Add one triple.
    store_add_abort_fn_t add_abort_fn;  ///> Abort (roll back) the add process.
                                        ///>    Only available in stores with
                                        ///>    LSUP_STORE_TXN feature.
                                        ///>    Optional.
    store_add_done_fn_t add_done_fn;    ///> Complete the add process.
    store_add_term_fn_t add_term_fn;    ///> Add (index) a term to the store.
                                        ///>    Only available in stores with
                                        ///>    LSUP_STORE_IDX feature.
                                        ///>    Optional.

    // Look up.
    store_lookup_fn_t   lookup_fn;      ///> Look up triples by pattern.

    // Removal.
    store_remove_fn_t   remove_fn;      ///> Remove triples by pattern.

    // Namespace prefix mapping.
    store_nsm_put_fn_t  nsm_put_fn;     ///> Add a namespace/prefix pair to
                                        ///>    the prefix map.
                                        ///>    Only available (and mandatory)
                                        ///>    in stores with the
                                        ///>    LSUP_STORE_IDX feature.
    store_nsm_get_fn_t  nsm_get_fn;     ///> Get a namespace/prefix from
                                        ///>    the prefix map.
                                        ///>    Only available (and mandatory)
                                        ///>    in stores with the
                                        ///>    LSUP_STORE_IDX feature.
} LSUP_StoreInt;


/** @brief Store iterator interface.
 */
typedef struct store_it_t {
    char                name[16];       ///> Iterator type name.

    iter_next_fn_t      iter_fn;        ///> Advance to next step and yield
                                        ///>    one triple key if found.
    iter_free_fn_t      free_fn;        ///> Free the iterator.
} LSUP_StoreIteratorInt;


/*
 * Template for a new store and iterator implementation.
 * These should be placed in the .c file where the interface functions are
 * defined, and declared as `extern` in the related .h file.

const LSUP_StoreInt my_store_int = {
    .name           = "My Store", // Truncated to 15 chars.
    .features       = LSUP_STORE_PERM | LSUP_STORE_IDX,

    .setup_fn       = my_setup_fn,
    .new_fn         = my_new_fn,
    .free_fn        = my_free_fn,

    .size_fn        = my_size_fn,

    .add_init_fn    = my_init_fn,
    .add_iter_fn    = my_iter_fn,
    .add_abort_fn   = my_add_abort_fn,
    .add_done_fn    = my_add_done_fn,
    .add_term_fn    = my_add_term_fn,

    .lookup_fn      = my_lookup_fn,

    .remove_fn      = my_remove_fn,

    .nsm_put_fn     = my_nsm_put_fn,
    .nsm_get_fn     = my_nsm_get_fn,
};

const LSUP_StoreIteratorInt my_it_int = {
    .name           = "My Iterator",

    .iter_next_fn_t = my_iter_next_fn,
    .iter_free_fn_t = my_iter_free_fn,
};
*/

#endif  /* _LSUP_STORE_H */