WIP start refactoring Keyset.

lakesuperior/model/base.pxd

@@ -2,4 +2,19 @@ from lakesuperior.cy_include cimport cytpl as tpl
 
 ctypedef tpl.tpl_bin Buffer
 
+# NOTE This may change in the future, e.g. if a different key size is to
+# be forced.
+ctypedef size_t KeyIdx
+
+ctypedef KeyIdx Key[1]
+ctypedef KeyIdx DoubleKey[2]
+ctypedef KeyIdx TripleKey[3]
+ctypedef KeyIdx QuadKey[4]
+
+cdef enum:
+    KLEN = sizeof(Key)
+    DBL_KLEN = sizeof(DoubleKey)
+    TRP_KLEN = sizeof(TripleKey)
+    QUAD_KLEN = sizeof(QuadKey)
+
 cdef bytes buffer_dump(Buffer* buf)

lakesuperior/model/structures/keyset.pxd

@@ -1,12 +1,27 @@
-cdef class Keyset:
+from lakesuperior.cy_includes cimport collections as cc
+from lakesuperior.model.base cimport (
+    KeyIdx, Key, DoubleKey, TripleKey, Buffer
+)
+cdef class BaseKeyset:
     cdef:
-        readonly unsigned char *data
-        readonly unsigned char itemsize
+        readonly cc.Array data
         readonly size_t ct, size
         size_t _cur
+        cc.ArrayConf conf
 
         void resize(self, size_t ct) except *
         unsigned char *get_item(self, i)
         bint iter_next(self, unsigned char** val)
         bint contains(self, const void *val)
 
+
+cdef class Keyset(BaseKeyset):
+    cdef size_t get_itemsize()
+
+
+cdef class DoubleKeyset(BaseKeyset):
+    cdef size_t get_itemsize()
+
+
+cdef class TripleKeyset(BaseKeyset):
+    cdef size_t get_itemsize()

lakesuperior/model/structures/keyset.pyx

@@ -1,7 +1,12 @@
 from libc.string cimport memcmp
 from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
 
-cdef class Keyset:
+from lakesuperior.cy_includes cimport collections as cc
+from lakesuperior.model.base cimport (
+    KeyIdx, Key, DoubleKey, TripleKey, Buffer
+)
+
+cdef class BaseKeyset:
     """
     Pre-allocated result set.
 
@@ -17,29 +22,27 @@ cdef class Keyset:
     ``size``: Total size, in bytes, of the data set. This is the product of
         ``itemsize`` and ``ct``.
     """
-    def __cinit__(self, size_t ct, unsigned char itemsize):
+    def __cinit__(self, size_t ct):
         """
         Initialize and allocate memory for the data set.
 
         :param size_t ct: Number of elements to be accounted for.
-        :param unsigned char itemsize: Size of an individual item.
-            Note that the ``itemsize`` is an unsigned char,
-            i.e. an item can be at most 255 bytes. This is for economy reasons,
-            since many multiplications are done between ``itemsize`` and other
-            char variables.
         """
-        self.ct = ct
-        self.itemsize = itemsize
-        self.size = self.itemsize * self.ct
+        self.conf.capacity = ct
+        self.itemsize = self.get_itemsize() # Set this in concrete classes
+        self.size = self.itemsize * self.conf.capacity
+
+        cc.array_conf_init(&self.conf)
+        self.conf.capacity = self.conf.capacity
+        cc.array_init_conf(&self.data
+        if not self.data:
+            raise MemoryError()
         self._cur = 0
 
         #logger.debug('Got malloc sizes: {}, {}'.format(ct, itemsize))
         #logger.debug(
         #    'Allocating {0} ({1}x{2}) bytes of Keyset data...'.format(
-        #        self.size, self.ct, self.itemsize))
-        self.data = <unsigned char *>PyMem_Malloc(ct * itemsize)
-        if not self.data:
-            raise MemoryError()
+        #        self.size, self.conf.capacity, self.itemsize))
         #logger.debug('...done allocating @ {0:x}.'.format(
         #        <unsigned long>self.data))
 
@@ -53,65 +56,14 @@ cdef class Keyset:
         """
         #logger.debug(
         #    'Releasing {0} ({1}x{2}) bytes of Keyset @ {3:x}...'.format(
-        #        self.size, self.ct, self.itemsize,
+        #        self.size, self.conf.capacity, self.itemsize,
         #        <unsigned long>self.data))
         PyMem_Free(self.data)
         #logger.debug('...done releasing.')
 
 
-    cdef void resize(self, size_t ct) except *:
-        """
-        Resize the result set. Uses ``PyMem_Realloc``.
-
-        Note that resizing to a smaller size does not copy or reallocate the
-        data, resizing to a larger size does.
-
-        Also, note that only the number of items can be changed, the item size
-        cannot.
-
-        :param size_t ct: Number of items in the result set.
-        """
-        cdef unsigned char *tmp
-        self.ct = ct
-        self.size = self.itemsize * self.ct
-
-        #logger.debug(
-        #    'Resizing Keyset to {0} ({1}x{2}) bytes @ {3:x}...'.format(
-        #        self.itemsize * ct, ct, self.itemsize,
-        #        <unsigned long>self.data))
-        tmp = <unsigned char *>PyMem_Realloc(self.data, ct * self.itemsize)
-        if not tmp:
-            raise MemoryError()
-        #logger.debug('...done resizing.')
-
-        self.data = tmp
-
-
     # Access methods.
 
-    def to_tuple(self):
-        """
-        Return the data set as a Python tuple.
-
-        :rtype: tuple
-        """
-        return tuple(
-                self.data[i: i + self.itemsize]
-                for i in range(0, self.size, self.itemsize))
-
-
-    def get_item_obj(self, i):
-        """
-        Get an item at a given index position.
-
-        :rtype: bytes
-        """
-        if i >= self.ct:
-            raise ValueError(f'Index {i} out of range.')
-
-        return self.get_item(i)[: self.itemsize]
-
-
     def iter_init(self):
         """
         Reset the cursor to the initial position.
@@ -149,7 +101,7 @@ cdef class Keyset:
         :return: True if a value was found, False if the end of the buffer
             has been reached.
         """
-        if self._cur >= self.ct:
+        if self._cur >= self.conf.capacity:
             val = NULL
             return False
 
@@ -171,3 +123,17 @@ cdef class Keyset:
                 return True
         return False
 
+
+class Keyset(BaseKeyset):
+    cdef size_t get_itemsize():
+        return KLEN
+
+
+class DoubleKeyset(BaseKeyset):
+    cdef size_t get_itemsize():
+        return DBL_KLEN
+
+
+class TripleKeyset(BaseKeyset):
+    cdef size_t get_itemsize():
+        return TRP_KLEN

lakesuperior/store/ldp_rs/lmdb_triplestore.pxd

@@ -1,26 +1,14 @@
 cimport lakesuperior.cy_include.cylmdb as lmdb
 cimport lakesuperior.cy_include.cytpl as tpl
 
-from lakesuperior.model.base cimport Buffer
+from lakesuperior.model.base cimport (
+    KeyIdx, Key, DoubleKey, TripleKey, Buffer
+)
 from lakesuperior.model.graph.graph cimport SimpleGraph
 from lakesuperior.model.structures.keyset cimport Keyset
 from lakesuperior.store.base_lmdb_store cimport BaseLmdbStore
 
-# NOTE This may change in the future, e.g. if a different key size is to
-# be forced.
-ctypedef size_t KeyIdx
-
-ctypedef KeyIdx Key[1]
-ctypedef KeyIdx DoubleKey[2]
-ctypedef KeyIdx TripleKey[3]
-ctypedef KeyIdx QuadKey[4]
-
 cdef enum:
-    KLEN = sizeof(Key)
-    DBL_KLEN = sizeof(DoubleKey)
-    TRP_KLEN = sizeof(TripleKey)
-    QUAD_KLEN = sizeof(QuadKey)
-
     IDX_OP_ADD = 1
     IDX_OP_REMOVE = -1
 

lakesuperior/store/ldp_rs/lmdb_triplestore.pyx

@@ -13,14 +13,17 @@ from libc.stdlib cimport free
 from libc.string cimport memcpy
 
 cimport lakesuperior.cy_include.cylmdb as lmdb
-from lakesuperior.model.base cimport buffer_dump
+from lakesuperior.model.base cimport (
+    KLEN, DBL_KLEN, TRP_KLEN, QUAD_KLEN,
+    KeyIdx, Key, DoubleKey, TripleKey,
+    Buffer, buffer_dump
+)
 from lakesuperior.model.graph.graph cimport SimpleGraph, Imr
 from lakesuperior.model.graph.term cimport Term
 from lakesuperior.model.graph.triple cimport BufferTriple
 
 from lakesuperior.store.base_lmdb_store cimport (
         BaseLmdbStore, data_v, dbi, key_v)
-from lakesuperior.model.base cimport Buffer
 from lakesuperior.model.graph.term cimport (
         deserialize_to_rdflib, serialize_from_rdflib)
 from lakesuperior.model.structures.keyset cimport Keyset
@@ -1355,7 +1358,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
 
                     i += 1
 
-            return ret.to_tuple()
+            # FIXME This needs to get the triples and convert them.
+            return ret
 
         finally:
             #pass