store_mdb.h 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. /** @file store_mdb.h
  2. *
  3. * @brief LMDB graph store backend.
  4. *
  5. * This module stores triples in a LMDB embedded store, optionally organized
  6. * into named graphs. The store is optimized and indexed for fast lookup of any
  7. * number of bound terms.
  8. *
  9. * The store must be first initialized once, to create the environment files
  10. * and folders as well as the internal databases, then it must be opened once
  11. * per session. Within that session multiple R/W operations can be performed
  12. * using transactions.
  13. *
  14. * Note that, even though the terms "graph", "context", etc. are used, no code
  15. * in this module checks for valid RDF data. In theory any term can be any
  16. * binary data. This allows using the store for non-RDF graph data.
  17. *
  18. * TODO more doc
  19. */
  20. #ifndef _LSUP_STORE_MDB_H
  21. #define _LSUP_STORE_MDB_H
  22. #include "lmdb.h"
  23. #include "triple.h"
  24. typedef char DbLabel[8];
  25. typedef struct MDBStore LSUP_MDBStore;
  26. typedef struct MDBIterator LSUP_MDBIterator;
  27. typedef LSUP_rc (*store_match_fn_t)(const LSUP_TripleKey spok, void *data);
  28. // TODO Introduce compile-time LSUP_BIG_STORE option to define two store
  29. // options: false: 64-bit hashes, uint32 keys, max 4G entries; true:
  30. // 128-bit hashes, size_t keys, max MAX_SIZE entries, larger and slower.
  31. // Ideally this could be specified at runtime to handle different stores with
  32. // different sizes, but code could become more complex.
  33. /** @brief Create the MDB environment and databases on disk.
  34. *
  35. * This function takes care of creaating the environment path if not existing,
  36. * and checking that it's a writable directory. If the path is not specified
  37. * in the LSUP_STORE_PATH environment variable, a default directory is used.
  38. *
  39. * @param[in,out] path Path of the suggested directory to use. It may be NULL,
  40. * in which case it will be set either to the environment variable
  41. * LSUP_STORE_PATH, or if that is not set, a default local path.
  42. */
  43. LSUP_rc LSUP_store_setup(char **path/*, bool clear*/);
  44. /** @brief Open an MDB store.
  45. *
  46. * The store must have been set up with #LSUP_store_setup.
  47. *
  48. * Some environment variables affect various store parameters:
  49. *
  50. * - LSUP_MDB_MAPSIZE Long int specifying the size of the memory map. Usually
  51. * it is not necessary to modify this, unless one is operating under memory
  52. * and disk constraints. The default map size is 1Tb.
  53. *
  54. * @param[in,out] store Uninitialized store struct pointer.
  55. *
  56. * @param[in] path MDB environment path. This must be the path given by
  57. * #LSUP_store_setup.
  58. *
  59. * @param[in] default_ctx Serialized URI to be used as a default context for
  60. * triples inserted without a context specified. If NULL, the store operates
  61. * in triple mode.
  62. */
  63. LSUP_MDBStore *
  64. LSUP_store_new(const char *path, const LSUP_Buffer *default_ctx);
  65. /** @brief Close a store and free its handle.
  66. *
  67. * @param[in] store Store pointer.
  68. *
  69. */
  70. void LSUP_store_free(LSUP_MDBStore *store);
  71. /** @brief Print stats about a store and its databases.
  72. *
  73. * TODO
  74. *
  75. * @param store[in] The store to get stats for.
  76. */
  77. LSUP_rc LSUP_store_stats(LSUP_MDBStore *store);
  78. /** @brief Store size.
  79. *
  80. * @param store[in] The store to calculate size of.
  81. *
  82. * @return Number of stored SPO triples across all contexts.
  83. */
  84. size_t LSUP_store_size(LSUP_MDBStore *store);
  85. /** @brief Add a batch of triples with optional context to the store.
  86. *
  87. * @param store[in] The store to add to.
  88. *
  89. * @param sc[in] Context as a serialized term. If this is NULL, and the
  90. * default context is not NULL, triples will be added to the default context
  91. * for the store, If the default context for the store is NULL, regardless of
  92. * the value of sc, triples will be added with no context.
  93. * @param data[in] Triples to be inserted as a 2D array of triples in the shape
  94. * of data[n][3], where n is the value of data_size.
  95. *
  96. * @param data_size[in] Number of triples to be inserted.
  97. */
  98. LSUP_rc LSUP_store_add(
  99. struct MDBStore *store, const LSUP_Buffer *sc,
  100. const LSUP_SerTriple *data, const size_t data_size);
  101. /** @brief Look up matching triples and optional context.
  102. *
  103. * This function may return a count of matches and/or an iterator of results as
  104. * serialized triples.
  105. *
  106. * @param store[in] The store to be queried.
  107. *
  108. * @param sspoc Array of 4 serialized term pointers representing the s, p, o, c
  109. * terms. Any and all of these may be NULL, which indicates an unbound query
  110. * term. Stores with context not set will always ignore the fourth term.
  111. *
  112. * @param it[out] Pointer to a pointer to an #LSUP_MDBIterator that will be
  113. * populated with a result iterator. This is always created even if no matches
  114. * are found and must be freed with #LSUP_store_it_free after use. If matches
  115. * are found, the iterator points to the first result which can be retrieved
  116. * with #LSUP_store_it_next.
  117. *
  118. * @param ct[out] If not NULL, this will be populated with the number of
  119. * entries found. It is very inexpensive to set for lookups without context,
  120. * much less so for 1-bound and 2-bound context lookups, in which cases it
  121. * should be set only if needed.
  122. *
  123. * @return LSUP_OK if entries were found, LSUP_NORESULT if none were found.
  124. */
  125. LSUP_rc LSUP_store_lookup(
  126. LSUP_MDBStore *store, LSUP_SerTerm *sspoc[],
  127. LSUP_MDBIterator **it, size_t *ct);
  128. /** @brief Get iterator results and advance the cursor.
  129. *
  130. * This function also checks if the matching triple is associated with a
  131. * context, if one was specified. If no associated contexts are found, the next
  132. * triple is searched, until the end of the results.
  133. *
  134. * NOTE: Iterators keep LMDB cursors and (read only) transactions open. Don't
  135. * hold on to them longer than necessary.
  136. *
  137. * NOTE: The memory pointed to by the individual LSUP_SerTerm pointers is
  138. * owned by the database. It must not be written to or freed. To modify
  139. * the data or use them beyond the caller's scope, this memory must be copied.
  140. *
  141. * @param it[in] Opaque iterator handle obtained with #LSUP_store_lookup.
  142. *
  143. * @param sspo[out] #LSUP_SerTriple to be populated with three serialized terms
  144. * if found, NULL if not found.
  145. *
  146. * @return LSUP_OK if results were found; LSUP_END otherwise. TODO handle
  147. * errors.
  148. */
  149. LSUP_rc LSUP_store_it_next(LSUP_MDBIterator *it, LSUP_SerTerm **sspo);
  150. /** @brief Free an iterator allocated by a lookup.
  151. *
  152. * @param it[in] Iterator pointer. It will be set to NULL after freeing.
  153. */
  154. void LSUP_store_it_free(struct MDBIterator *it);
  155. /** @brief Contexts that a triple key appears in.
  156. *
  157. * This function is most conveniently used by a callback to #LSUP_store_lookup
  158. * because it handles triple keys.
  159. *
  160. * @param store[in] The store to be queried.
  161. *
  162. * @param spok[in] Triple key to look up.
  163. *
  164. * @param ck[out] Pointer to an array of contexts. Memory is allocated by this
  165. * function and must be freed by the caller.
  166. *
  167. * @param ct[out] Number of contexts found.
  168. */
  169. LSUP_rc LSUP_store_triple_contexts(
  170. LSUP_MDBStore *store, LSUP_Key spok[], LSUP_Key **ck, size_t *ct);
  171. #endif