Revert to Keyset as a simple C array (not cc.Array).

lakesuperior/model/structures/keyset.pxd

@@ -1,19 +1,16 @@
-cimport lakesuperior.cy_include.collections as cc
-
 from lakesuperior.model.base cimport (
     KeyIdx, Key, DoubleKey, TripleKey, Buffer
 )
-
-ctypedef bint (*key_cmp_fn_t)(
-    const TripleKey* spok, const KeyIdx* k1, const KeyIdx* k2
-)
-
 cdef class Keyset:
     cdef:
-        readonly size_t ct, size
-        cc.Array* data
-        cc.ArrayConf conf
+        TripleKey* data
+        size_t ct
+        size_t _cur # Index cursor used to look up values.
+        size_t _free_i # Index of next free slot.
 
-        Keyset lookup(
-            self, const KeyIdx* sk, const KeyIdx* pk, const KeyIdx* ok
-        )
+        void seek(self, size_t idx=*)
+        size_t tell(self)
+        bint get_at(self, size_t i, TripleKey* item)
+        bint get_next(self, TripleKey* item)
+        void add(self, const TripleKey* val) except *
+        bint contains(self, const TripleKey* val)

lakesuperior/model/structures/keyset.pyx

@@ -1,30 +1,44 @@
-from libc.string cimport memcmp
-from libc.stdlib cimport free
+import logging
 
-cimport lakesuperior.cy_include.collections as cc
-cimport lakesuperior.model.structures.callbacks as cb
+from libc.string cimport memcmp, memcpy
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+
+from lakesuperior.model.base cimport TripleKey, TRP_KLEN
+
+
+logger = logging.getLogger(__name__)
 
-from lakesuperior.model.base cimport (
-    TRP_KLEN, KeyIdx, Key, DoubleKey, TripleKey, Buffer
-)
 
 cdef class Keyset:
     """
     Pre-allocated result set.
+
+    Data in the set are stored as a 1D contiguous array of characters.
+    Access to elements at an arbitrary index position is achieved by using the
+    ``itemsize`` property multiplied by the index number.
+
+    Key properties:
+
+    ``ct``: number of elements in the set.
+    ``itemsize``: size of each element, in bytes. All elements have the same
+        size.
+    ``size``: Total size, in bytes, of the data set. This is the product of
+        ``itemsize`` and ``ct``.
     """
-    def __cinit__(self, size_t ct=1):
+    def __cinit__(self, size_t ct=0):
         """
         Initialize and allocate memory for the data set.
 
         :param size_t ct: Number of elements to be accounted for.
         """
-        cc.array_conf_init(&self.conf)
-        self.conf.capacity = ct or 1
-        self.conf.exp_factor = .5
+        self.ct = ct
+        self.data = <TripleKey*>PyMem_Malloc(self.ct * TRP_KLEN)
+        logger.info(f'data address: 0x{<size_t>self.data:02x}')
+        if ct and not self.data:
+            raise MemoryError('Error allocating Keyset data.')
 
-        cc.array_new_conf(&self.conf, &self.data)
-        if not self.data:
-            raise MemoryError()
+        self._cur = 0
+        self._free_i = 0
 
 
     def __dealloc__(self):
@@ -34,74 +48,89 @@ cdef class Keyset:
         This is called when the Python instance is garbage collected, which
         makes it handy to safely pass a Keyset instance across functions.
         """
-        if self.data:
-            free(self.data)
+        #logger.debug(
+        #    'Releasing {0} ({1}x{2}) bytes of Keyset @ {3:x}...'.format(
+        #        self.size, self.conf.capacity, self.itemsize,
+        #        <unsigned long>self.data))
+        PyMem_Free(self.data)
+        #logger.debug('...done releasing.')
 
 
     # Access methods.
 
-    cdef Keyset lookup(
-            self, const KeyIdx* sk, const KeyIdx* pk, const KeyIdx* ok
-    ):
+    cdef void seek(self, size_t idx=0):
         """
-        Look up triple keys.
+        Place the cursor at a certain index, 0 by default.
+        """
+        self._cur = idx
+
 
-        This works in a similar way that the ``SimpleGraph`` and ``LmdbStore``
-        methods work.
+    cdef size_t tell(self):
+        """
+        Tell the position of the cursor in the keyset.
+        """
+        return self._cur
 
-        Any and all the terms may be NULL. A NULL term is treated as unbound.
 
-        :param const KeyIdx* sk: s key pointer.
-        :param const KeyIdx* pk: p key pointer.
-        :param const KeyIdx* ok: o key pointer.
+    cdef bint get_at(self, size_t i, TripleKey* item):
         """
-        cdef:
-            void* cur
-            cc.ArrayIter it
-            TripleKey spok
-            Keyset ret
-            KeyIdx* k1 = NULL
-            KeyIdx* k2 = NULL
-            key_cmp_fn_t cmp_fn
+        Get an item at a given index position. Cython-level method.
 
-        cc.array_iter_init(&it, self.data)
+        :rtype: TripleKey
+        """
+        if i >= self._free_i:
+            return False
 
-        if sk and pk and ok: # s p o
-            pass # TODO
+        self._cur = i
+        item[0] = self.data[i]
 
-        elif sk:
-            k1 = sk
-            if pk: # s p ?
-                k2 = pk
-                cmp_fn = cb.lookup_skpk_cmp_fn
+        return True
 
-            elif ok: # s ? o
-                k2 = ok
-                cmp_fn = cb.lookup_skok_cmp_fn
 
-            else: # s ? ?
-                cmp_fn = cb.lookup_sk_cmp_fn
+    cdef bint get_next(self, TripleKey* item):
+        """
+        Populate the current value and advance the cursor by 1.
 
-        elif pk:
-            k1 = pk
-            if ok: # ? p o
-                k2 = ok
-                cmp_fn = cb.lookup_pkok_cmp_fn
+        :param void *val: Addres of value returned. It is NULL if
+            the end of the buffer was reached.
 
-            else: # ? p ?
-                cmp_fn = cb.lookup_pk_cmp_fn
+        :rtype: bint
+        :return: True if a value was found, False if the end of the buffer
+            has been reached.
+        """
+        if self._cur >= self._free_i:
+            return False
+
+        item[0] = self.data[self._cur]
+        self._cur += 1
+
+        return True
 
-        elif ok: # ? ? o
-            k1 = ok
-            cmp_fn = cb.lookup_ok_cmp_fn
 
-        else: # ? ? ?
-            return self # TODO Placeholder. This should actually return a copy.
+    cdef void add(self, const TripleKey* val) except *:
+        """
+        Add a triple key to the array.
+        """
+        logger.info('Adding triple to key set.')
+        logger.info(f'triple: {val[0][0]} {val[0][1]} {val[0][2]}')
+        logger.info(f'_free_i: {self._free_i}')
+
+        if self._free_i >= self.ct:
+            raise MemoryError('No slots left in key set.')
 
-        ret = Keyset(256) # TODO Totally arbitrary.
-        while cc.array_iter_next(&it, &cur) != cc.CC_ITER_END:
-            if cmp_fn(<TripleKey*>spok, k1, k2):
-                if cc.array_add(ret.data, spok) != cc.CC_OK:
-                    raise RuntimeError('Error adding triple key.')
+        self.data[self._free_i] = val[0]
+
+        self._free_i += 1
+
+
+    cdef bint contains(self, const TripleKey* val):
+        """
+        Whether a value exists in the set.
+        """
+        cdef TripleKey stored_val
 
-        return ret
+        self.seek()
+        while self.get_next(&stored_val):
+            if memcmp(val, stored_val, TRP_KLEN) == 0:
+                return True
+        return False

lakesuperior/store/ldp_rs/lmdb_triplestore.pxd

@@ -37,7 +37,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         void _index_triple(self, int op, TripleKey spok) except *
         Keyset triple_keys(self, tuple triple_pattern, context=*)
         void _all_term_keys(self, term_type, cc.HashSet** tkeys) except *
-        void lookup_term(self, const Key tk, Buffer* data) except *
+        void lookup_term(self, const KeyIdx* tk, Buffer* data) except *
         Keyset _lookup(self, tuple triple_pattern)
         Keyset _lookup_1bound(self, unsigned char idx, Key luk)
         Keyset _lookup_2bound(

lakesuperior/store/ldp_rs/lmdb_triplestore.pyx

@@ -380,11 +380,9 @@ cdef class LmdbTriplestore(BaseLmdbStore):
 
     cpdef void _remove(self, tuple triple_pattern, context=None) except *:
         cdef:
-            unsigned char spok[TRP_KLEN]
-            void* cur
-            cc.ArrayIter it
-            Key ck
             lmdb.MDB_val spok_v, ck_v
+            TripleKey spok_cur
+            Key ck
 
         #logger.debug('Removing triple: {}'.format(triple_pattern))
         if context is not None:
@@ -404,13 +402,13 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         try:
             spok_v.mv_size = TRP_KLEN
             # If context was specified, remove only associations with that context.
-            cc.array_iter_init(&it, match_set.data)
+            match_set.seek()
             if context is not None:
                 #logger.debug('Removing triples in matching context.')
                 ck_v.mv_data = ck
                 ck_v.mv_size = KLEN
-                while cc.array_iter_next(&it, &cur) != cc.CC_ITER_END:
-                    spok_v.mv_data = cur
+                while match_set.get_next(&spok_cur):
+                    spok_v.mv_data = spok_cur
                     # Delete spo:c entry.
                     try:
                         _check(lmdb.mdb_cursor_get(
@@ -436,20 +434,19 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                         # association is present.
 
                         # spok_v has changed on mdb_cursor_del. Restore.
-                        spok_v.mv_data = cur
+                        spok_v.mv_data = spok_cur
                         try:
                             _check(lmdb.mdb_cursor_get(
                                 dcur, &spok_v, NULL, lmdb.MDB_SET))
                         except KeyNotFoundError:
-                            self._index_triple(IDX_OP_REMOVE, <TripleKey>cur)
-                    i += 1
+                            self._index_triple(IDX_OP_REMOVE, <TripleKey>spok_cur)
 
             # If no context is specified, remove all associations.
             else:
                 #logger.debug('Removing triples in all contexts.')
                 # Loop over all SPO matching the triple pattern.
-                while cc.array_iter_next(&it, &cur) != cc.CC_ITER_END:
-                    spok_v.mv_data = cur
+                while match_set.get_next(&spok_cur):
+                    spok_v.mv_data = spok_cur
                     # Loop over all context associations for this SPO.
                     try:
                         _check(lmdb.mdb_cursor_get(
@@ -459,9 +456,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                         continue
                     else:
                         ck = <Key>ck_v.mv_data
-                        logger.debug(f'Removing {<TripleKey>cur} from main.')
+                        logger.debug(f'Removing {<TripleKey>spok_cur} from main.')
                         while True:
-
                             # Delete c:spo association.
                             try:
                                 _check(lmdb.mdb_cursor_get(
@@ -471,7 +467,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                             else:
                                 lmdb.mdb_cursor_del(icur, 0)
                                 # Restore the pointer to the deleted SPO.
-                                spok_v.mv_data = cur
+                                spok_v.mv_data = spok_cur
                             # Move on to next associated context.
                             try:
                                 _check(lmdb.mdb_cursor_get(
@@ -486,10 +482,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                             pass
                         else:
                             lmdb.mdb_cursor_del(dcur, lmdb.MDB_NODUPDATA)
-                            self._index_triple(IDX_OP_REMOVE, <TripleKey>cur)
+                            self._index_triple(IDX_OP_REMOVE, <TripleKey>spok_cur)
                             #ck_v.mv_data = ck # Unnecessary?
-                    finally:
-                        i += 1
 
         finally:
             #logger.debug('Closing spo:c in _remove.')
@@ -697,9 +691,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         """
         cdef:
             size_t i = 0
-            void* it_cur
+            TripleKey it_cur
             lmdb.MDB_val key_v, data_v
-            cc.ArrayIter it
 
         # This sounds strange, RDFLib should be passing None at this point,
         # but anyway...
@@ -714,8 +707,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         cur = self._cur_open('spo:c')
         try:
             key_v.mv_size = TRP_KLEN
-            cc.array_iter_init(&it, rset.data)
-            while cc.array_iter_next(&it, &it_cur) != cc.CC_ITER_END:
+            rset.seek()
+            while rset.get_next(&it_cur):
                 logger.info(f'it_cur address: {<size_t>it_cur:02x}')
                 logger.info('it_cur: {}'.format(
                     (<unsigned char*>it_cur)[:TRP_KLEN]))
@@ -728,9 +721,9 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                 logger.info('smome bogus memory exercise.')
                 logger.info('smome')
                 logger.info('Checking contexts for triples: {} {} {}'.format(
-                    (<TripleKey>it_cur)[0],
-                    (<TripleKey>it_cur)[1],
-                    (<TripleKey>it_cur)[2],
+                    it_cur[0],
+                    it_cur[1],
+                    it_cur[2],
                 ))
                 contexts = []
                 # This shall never be MDB_NOTFOUND.
@@ -783,12 +776,14 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         in.
         """
         cdef:
-            void* spok
-            size_t cur = 0
-            Buffer* buffers
-            BufferTriple* btrp
+            Buffer buffers[3]
+            BufferTriple btrp
             SimpleGraph gr
-            cc.ArrayIter it
+            TripleKey spok
+
+        btrp.s = buffers
+        btrp.p = buffers + 1
+        btrp.o = buffers + 2
 
         gr = Imr(uri=uri) if uri else SimpleGraph()
 
@@ -797,27 +792,17 @@ cdef class LmdbTriplestore(BaseLmdbStore):
 
         match = self.triple_keys(triple_pattern, context)
         logger.info(f'Matches in graph_lookup: {match.ct}')
-        btrp = <BufferTriple*>gr.pool.alloc(match.ct, sizeof(BufferTriple))
-        buffers = <Buffer*>gr.pool.alloc(3 * match.ct, sizeof(Buffer))
-
-        cc.array_iter_init(&it, match.data)
-        while cc.array_iter_next(&it, &spok):
-            btrp[cur].s = buffers + cur * 3
-            btrp[cur].p = buffers + cur * 3 + 1
-            btrp[cur].o = buffers + cur * 3 + 2
-
-            logger.info('Looking up key: {}'.format((<unsigned char*>spok)[:KLEN]))
-            self.lookup_term(<KeyIdx*>spok, buffers + cur * 3)
-            logger.info(f'Found triple s: {buffer_dump(btrp[cur].s)}')
-            logger.info('Looking up key: {}'.format((<unsigned char*>spok)[KLEN:DBL_KLEN]))
-            self.lookup_term(<KeyIdx*>(spok + KLEN), buffers + cur * 3 + 1)
-            logger.info(f'Found triple p: {buffer_dump(btrp[cur].p)}')
-            logger.info('Looking up key: {}'.format((<unsigned char*>spok)[DBL_KLEN:TRP_KLEN]))
-            self.lookup_term(<KeyIdx*>(spok + DBL_KLEN), buffers + cur * 3 + 2)
-            logger.info(f'Found triple o: {buffer_dump(btrp[cur].o)}')
-
-            gr.add_triple(btrp + cur, copy)
-            cur += 1
+        #btrp = <BufferTriple*>gr.pool.alloc(match.ct, sizeof(BufferTriple))
+        #buffers = <Buffer*>gr.pool.alloc(3 * match.ct, sizeof(Buffer))
+
+        match.seek()
+        while match.get_next(&spok):
+            self.lookup_term(spok, buffers)
+            self.lookup_term(spok + 1, buffers + 1)
+            self.lookup_term(spok + 2, buffers + 2)
+            #logger.info(f'Found triple: {buffer_dump(btrp.s)} {buffer_dump(btrp.p)} {buffer_dump(btrp.o)}')
+
+            gr.add_triple(&btrp, True)
 
         return gr
 
@@ -837,8 +822,6 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         # TODO: Improve performance by allowing passing contexts as a tuple.
         cdef:
             size_t ct = 0, i = 0
-            void* cur
-            cc.ArrayIter it
             lmdb.MDB_cursor *icur
             lmdb.MDB_val key_v, data_v
             Key tk, ck
@@ -882,7 +865,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                         #logger.debug('spok / ck pair not found.')
                         return Keyset()
                     ret = Keyset(1)
-                    cc.array_add(ret.data, &spok)
+                    ret.add(&spok)
 
                     return ret
 
@@ -909,7 +892,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                     while True:
                         #logger.debug('Data page: {}'.format(
                         #        (<unsigned char *>data_v.mv_data)[: data_v.mv_size]))
-                        cc.array_add(ret.data, data_v.mv_data)
+                        spok = <TripleKey>data_v.mv_data
+                        ret.add(&spok)
 
                         try:
                             _check(lmdb.mdb_cursor_get(
@@ -930,10 +914,10 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                     data_v.mv_size = TRP_KLEN
 
                     flt_res = Keyset(res.ct)
-                    cc.array_iter_init(&it, res.data)
-                    while cc.array_iter_next(&it, &cur) != cc.CC_ITER_END:
+                    res.seek()
+                    while res.get_next(&spok):
+                        data_v.mv_data = spok
                         #logger.debug('Checking row #{}'.format(flt_j))
-                        data_v.mv_data = cur
                         #logger.debug('Checking c:spo {}, {}'.format(
                         #    (<unsigned char *>key_v.mv_data)[: key_v.mv_size],
                         #    (<unsigned char *>data_v.mv_data)[: data_v.mv_size]))
@@ -945,7 +929,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                         except KeyNotFoundError:
                             continue
                         else:
-                            cc.array_add(flt_res.data, cur)
+                            flt_res.add(&spok)
 
                     return flt_res
             finally:
@@ -971,7 +955,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         :return: Matching triple keys.
         """
         cdef:
-            size_t ct = 0, i = 0
+            size_t ct = 0
             lmdb.MDB_stat db_stat
             lmdb.MDB_val spok_v, ck_v
             TripleKey spok
@@ -1006,7 +990,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                         return Keyset()
 
                     matches = Keyset(1)
-                    cc.array_add(matches.data, &spok)
+                    matches.add(&spok)
                     return matches
 
                 # s p ?
@@ -1052,7 +1036,8 @@ cdef class LmdbTriplestore(BaseLmdbStore):
             _check(lmdb.mdb_cursor_get(
                     dcur, &key_v, &data_v, lmdb.MDB_FIRST))
             while True:
-                cc.array_add(ret.data, key_v.mv_data)
+                spok = <TripleKey>key_v.mv_data
+                ret.add(&spok)
 
                 try:
                     _check(lmdb.mdb_cursor_get(
@@ -1060,8 +1045,6 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                 except KeyNotFoundError:
                     break
 
-                i += 1
-
             return ret
         finally:
             self._cur_close(dcur)
@@ -1122,9 +1105,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                     logger.info('Assembled triple in lookup_1bound: {} {} {}'.format(
                         spok[0], spok[1], spok[2]))
 
-                    r = cc.array_add(ret.data, &spok)
-                    if r != cc.CC_OK:
-                        raise RuntimeError(f'Error adding to keyset: {r}.')
+                    ret.add(&spok)
                     logger.info(f'ret count: {ret.ct}')
 
                 try:
@@ -1234,7 +1215,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
                     spok[term_order[1]] = luk[1]
                     spok[term_order[2]] = <KeyIdx>(data_v.mv_data + src_pos)
 
-                    cc.array_add(ret.data, spok)
+                    ret.add(&spok)
                     #src_pos = KLEN * j
                     #ret_pos = (ret_offset + ret.itemsize * j)
                     ##logger.debug('Page offset: {}'.format(pg_offset))
@@ -1432,7 +1413,7 @@ cdef class LmdbTriplestore(BaseLmdbStore):
 
     # Key conversion methods.
 
-    cdef inline void lookup_term(self, const Key tk, Buffer* data) except *:
+    cdef inline void lookup_term(self, const KeyIdx* tk, Buffer* data) except *:
         """
         look up a term by key.
 
@@ -1446,13 +1427,13 @@ cdef class LmdbTriplestore(BaseLmdbStore):
         key_v.mv_size = KLEN
 
         _check(
-                lmdb.mdb_get(
-                    self.txn, self.get_dbi('t:st'), &key_v, &data_v
-                ),
-                f'Error getting data for key \'{tk[0]}\'.')
+            lmdb.mdb_get(
+                self.txn, self.get_dbi('t:st'), &key_v, &data_v
+            ),
+            f'Error getting data for key \'{tk[0]}\'.'
+        )
         data.addr = data_v.mv_data
         data.sz = data_v.mv_size
-        #logger.info('Found term: {}'.format(buffer_dump(data)))
 
 
     cdef object from_key(self, const Key tk):