store_mdb.h 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. /** @file store_mdb.h
  2. *
  3. * @brief LMDB graph store backend.
  4. *
  5. * This module stores triples in a LMDB embedded store, optionally organized
  6. * into named graphs. The store is optimized and indexed for fast lookup of any
  7. * number of bound terms.
  8. *
  9. * The store must be first initialized once, to create the environment files
  10. * and folders as well as the internal databases, then it must be opened once
  11. * per session. Within that session multiple R/W operations can be performed
  12. * using transactions.
  13. *
  14. * Note that, even though the terms "graph", "context", etc. are used, no code
  15. * in this module checks for valid RDF data. In theory any term can be any
  16. * binary data. This allows using the store for non-RDF graph data.
  17. *
  18. * TODO more doc
  19. */
  20. #ifndef _LSUP_STORE_MDB_H
  21. #define _LSUP_STORE_MDB_H
  22. #include "lmdb.h"
  23. #include "triple.h"
  24. typedef char DbLabel[8];
  25. typedef struct MDBStore LSUP_MDBStore;
  26. typedef struct MDBIterator LSUP_MDBIterator;
  27. typedef LSUP_rc (*store_match_fn_t)(const LSUP_TripleKey spok, void *data);
  28. // TODO Introduce compile-time LSUP_BIG_STORE option to define two store
  29. // options: false: 64-bit hashes, uint32 keys, max 4G entries; true:
  30. // 128-bit hashes, size_t keys, max MAX_SIZE entries, larger and slower.
  31. // Ideally this could be specified at runtime to handle different stores with
  32. // different sizes, but code could become more complex.
  33. /** @brief Create the MDB environment and databases on disk.
  34. *
  35. * This function takes care of creaating the environment path if not existing,
  36. * and checking that it's a writable directory. If the path is not specified
  37. * in the LSUP_STORE_PATH environment variable, a default directory is used.
  38. *
  39. * TODO Add clear parameter.
  40. *
  41. * @param[in,out] path Path of the suggested directory to use. It may be NULL,
  42. * in which case it will be set either to the environment variable
  43. * LSUP_STORE_PATH, or if that is not set, a default local path.
  44. */
  45. LSUP_rc LSUP_mdbstore_setup(char **path/*, bool clear*/);
  46. /** @brief Open an MDB store.
  47. *
  48. * The store must have been set up with #LSUP_mdbstore_setup.
  49. *
  50. * Some environment variables affect various store parameters:
  51. *
  52. * - LSUP_MDB_MAPSIZE Long int specifying the size of the memory map. Usually
  53. * it is not necessary to modify this, unless one is operating under memory
  54. * and disk constraints. The default map size is 1Tb.
  55. *
  56. * @param[in,out] store Uninitialized store struct pointer.
  57. *
  58. * @param[in] path MDB environment path. This must be the path given by
  59. * #LSUP_mdbstore_setup.
  60. *
  61. * @param[in] default_ctx Serialized URI to be used as a default context for
  62. * triples inserted without a context specified. If NULL, the store operates
  63. * in triple mode.
  64. */
  65. LSUP_rc
  66. LSUP_mdbstore_new(
  67. const char *path, const LSUP_Buffer *default_ctx,
  68. LSUP_MDBStore **store);
  69. /** @brief Close a store and free its handle.
  70. *
  71. * @param[in] store Store pointer.
  72. *
  73. */
  74. void LSUP_mdbstore_free(LSUP_MDBStore *store);
  75. /** @brief Print stats about a store and its databases.
  76. *
  77. * TODO
  78. *
  79. * @param store[in] The store to get stats for.
  80. */
  81. LSUP_rc LSUP_mdbstore_stats(LSUP_MDBStore *store);
  82. /** @brief Store size.
  83. *
  84. * @param store[in] The store to calculate size of.
  85. *
  86. * @return Number of stored SPO triples across all contexts.
  87. */
  88. size_t LSUP_mdbstore_size(LSUP_MDBStore *store);
  89. /** @brief Initialize bulk triple load.
  90. *
  91. * This is the first step of a bulk load. It is best used when the data at hand
  92. * need to be pre-processed, which can be done in the same loop as the next
  93. * step to keep memory usage low.
  94. *
  95. * @param store[in] The store to add to.
  96. *
  97. * @param sc[in] Context as a serialized term. If this is NULL, and the
  98. * default context is not NULL, triples will be added to the default context
  99. * for the store, If the default context for the store is NULL, regardless of
  100. * the value of sc, triples will be added with no context.
  101. *
  102. * @param it[out] Pointer to an iterator pointer to be passed to the following
  103. * load steps.
  104. */
  105. void
  106. LSUP_mdbstore_add_init(
  107. LSUP_MDBStore *store, const LSUP_Buffer *sc, LSUP_MDBIterator **it);
  108. /** @brief Add one triple into the store.
  109. *
  110. * This must be called after #LSUP_mdbstore_add_init, using the iterator
  111. * yielded by that function. It may be called multiple times and must be
  112. * followed by #LSUP_mdbstore_add_done.
  113. *
  114. * @param it[in] Iterator obtained by #LSUP_mdbstore_add_init.
  115. *
  116. * @param sspo[in] Serialized triple to be added.
  117. */
  118. LSUP_rc
  119. LSUP_mdbstore_add_iter(struct MDBIterator *it, const LSUP_SerTriple *sspo);
  120. /** @brief Finalize an add loop.
  121. *
  122. * This must be called after #LSUP_mdbstore_add_iter.
  123. *
  124. * @param it[in] Iterator obtained by #LSUP_mdbstore_add_init.
  125. *
  126. * @param inserted[out] If not NULL this is populated with the number of
  127. * triples effectively inserted.
  128. */
  129. LSUP_rc
  130. LSUP_mdbstore_add_done(LSUP_MDBIterator *it, size_t *inserted);
  131. /** @brief Add a batch of triples with optional context to the store.
  132. *
  133. * This is a shortcut for calling #LSUP_mdbstore_add_init,
  134. * #LSUP_mdbstore_add_iter and #LSUP_mdbstore_add_done in a sequence
  135. * when an array of pre-serialized triples is available.
  136. *
  137. * @param store[in] The store to add to.
  138. *
  139. * @param sc[in] Context as a serialized term. If this is NULL, and the
  140. * default context is not NULL, triples will be added to the default context
  141. * for the store. If the default context for the store is NULL, regardless of
  142. * the value of sc, triples will be added with no context.
  143. * @param data[in] Triples to be inserted as a 2D array of triples in the shape
  144. * of data[n][3], where n is the value of data_size.
  145. *
  146. * @param inserted[out] If not NULL, it will be filled with the count of
  147. * effectively inserted triples.
  148. *
  149. * @param data_size[in] Number of triples to be inserted.
  150. */
  151. LSUP_rc LSUP_mdbstore_add(
  152. struct MDBStore *store, const LSUP_Buffer *sc,
  153. const LSUP_SerTriple strp[], const size_t ct, size_t *inserted);
  154. LSUP_rc
  155. LSUP_mdbstore_remove(
  156. LSUP_MDBStore *store, const LSUP_SerTriple *sspo,
  157. const LSUP_Buffer *sc, size_t *ct);
  158. /** @brief Look up matching triples and optional context.
  159. *
  160. * This function may return a count of matches and/or an iterator of results as
  161. * serialized triples.
  162. *
  163. * @param store[in] The store to be queried.
  164. *
  165. * @param sspo Serialized triple representing the s, p, o
  166. * terms. Any and all of these may be NULL, which indicates an unbound query
  167. * term. Stores with context not set will always ignore the fourth term.
  168. *
  169. * @param sc Serialized context to limit search to. It may be NULL, in which
  170. * case search is done in all contexts. Note that triples inserted without
  171. * context are assigned the *default* context, indicated by the "default_ctx"
  172. * member of the store struct.
  173. *
  174. * @param it[out] Pointer to a pointer to an #LSUP_MDBIterator that will be
  175. * populated with a result iterator. This is always created even if no matches
  176. * are found and must be freed with #LSUP_mdbiter_free after use. If matches
  177. * are found, the iterator points to the first result which can be retrieved
  178. * with #LSUP_mdbiter_next.
  179. *
  180. * @param ct[out] If not NULL, this will be populated with the number of
  181. * entries found. It is very inexpensive to set for lookups without context,
  182. * much less so for 1-bound and 2-bound context lookups, in which cases it
  183. * should be set only if needed.
  184. *
  185. * @return LSUP_OK if entries were found, LSUP_NORESULT if none were found.
  186. */
  187. LSUP_rc LSUP_mdbstore_lookup(
  188. LSUP_MDBStore *store, const LSUP_SerTriple *sspo,
  189. const LSUP_Buffer *sc, LSUP_MDBIterator **it, size_t *ct);
  190. /** @brief Yield the matching triples and advance the iterator.
  191. *
  192. * This function also checks if the matching triple is associated with a
  193. * context, if one was specified. If no associated contexts are found, the next
  194. * triple is searched, until the end of the results.
  195. *
  196. * NOTE: Iterators keep LMDB cursors and (read only) transactions open. Don't
  197. * hold on to them longer than necessary.
  198. *
  199. * NOTE: The memory pointed to by the individual LSUP_Buffer pointers is
  200. * owned by the database. It must not be written to or freed. To modify
  201. * the data or use them beyond the caller's scope, this memory must be copied.
  202. *
  203. * @param it[in] Opaque iterator handle obtained with #LSUP_mdbstore_lookup.
  204. *
  205. * @param sspo[out] #LSUP_SerTriple to be populated with three serialized terms
  206. * if found, NULL if not found. Internal callers may pass NULL if they don't
  207. * need the serialized triples.
  208. *
  209. * @return LSUP_OK if results were found; LSUP_END if no (more) results were
  210. * found; LSUP_DB_ERR if a MDB_* error occurred.
  211. */
  212. LSUP_rc LSUP_mdbiter_next(LSUP_MDBIterator *it, LSUP_SerTriple *sspo);
  213. /** @brief Free an iterator allocated by a lookup.
  214. *
  215. * @param it[in] Iterator pointer. It will be set to NULL after freeing.
  216. */
  217. void LSUP_mdbiter_free(struct MDBIterator *it);
  218. /** @brief Contexts that a triple key appears in.
  219. *
  220. * This function is most conveniently used by a callback to #LSUP_mdbstore_lookup
  221. * because it handles triple keys.
  222. *
  223. * @param store[in] The store to be queried.
  224. *
  225. * @param spok[in] Triple key to look up.
  226. *
  227. * @param ck[out] Pointer to an array of contexts. Memory is allocated by this
  228. * function and must be freed by the caller.
  229. *
  230. * @param ct[out] Number of contexts found.
  231. */
  232. LSUP_rc LSUP_mdbstore_triple_contexts(
  233. LSUP_MDBStore *store, LSUP_Key spok[], LSUP_Key **ck, size_t *ct);
  234. #endif