Pass all LmdbStore tests.

conftest.py

@@ -28,7 +28,6 @@ def db(app):
     Set up and tear down test triplestore.
     '''
     db = app.rdfly
-    import pdb; pdb.set_trace()
     if hasattr(db.store, 'begin'):
         with TxnManager(db.store, True) as txn:
             db.bootstrap()

doc/notes/performance.txt

@@ -1,6 +1,27 @@
-# Performance benchmark notes
+# Performance Benchmark Notes
 
-Sleepycat back end: Duchamp VIAF dataset(343 triples)
+## Environment
+
+### Hardware
+
+- Dell Precison M3800 Laptop
+- 8x Intel(R) Core(TM) i7-4712HQ CPU @ 2.30GHz
+- 12Gb RAM
+- SSD
+
+### Software
+
+- Arch Linux OS
+- glibc 2.26-11
+- python 3.5.4
+- lmdb 0.9.21-1
+- db (BerkeleyDB) 5.3.28-3
+
+### Sample Data Set
+
+Modified Duchamp VIAF dataset (343 triples; changed all subjects to `<>`)
+
+## Sleepycat Back End Test
 
 10K PUTs to new resources under the same container:
 
@@ -9,3 +30,19 @@ Sleepycat back end: Duchamp VIAF dataset(343 triples)
 3.4M triples total in repo at the end of the process
 
 Retrieval of parent resource (11400 triples), pipe to /dev/null: 3.6"
+
+## LMDB Back End Test
+
+10K PUTs to new resources under the same container:
+
+~29' running time
+0.178" per resource
+3.4M triples total in repo at the end of the process
+
+Some gaps every ~40-50 requests, probably blocking transactions or disk
+flush
+
+Database size: 633 Mb
+
+Retrieval of parent resource (11400 triples), pipe to /dev/null: 3.48"
+

lakesuperior/store_layouts/ldp_rs/lmdb_store.py

@@ -224,6 +224,8 @@ class LmdbStore(Store):
         'spo:c',
         # This has empty values and is used to keep track of empty contexts.
         'c:',
+        # Prefix to namespace: 1:1
+        'pfx:ns',
     )
     idx_keys = (
         # Namespace to prefix: 1:1
@@ -457,9 +459,14 @@ class LmdbStore(Store):
                     # Index.
                     icur.put(thash, keys[i])
 
+        # Add context in cnotext DB.
+        ck = keys[3]
+        with self.cur('c:') as cur:
+            if not cur.set_key(ck):
+                cur.put(ck, b'')
+
         # Add triple:context association.
         spok = self.SEP_BYTE.join(keys[:3])
-        ck = keys[3]
         with self.cur('spo:c') as cur:
             if not cur.set_key_dup(spok, ck):
                 cur.put(spok, ck)
@@ -562,8 +569,8 @@ class LmdbStore(Store):
         Get an iterator of all prefix: namespace bindings.
         '''
         with self.cur('pfx:ns') as cur:
-            bindings = iter(cur)
-            return ((b2s(pfx), Namespace(b2s(ns))) for pfx, ns in bindings)
+            for pfx, ns in iter(cur):
+                yield (b2s(pfx), Namespace(b2s(ns)))
 
 
     def contexts(self, triple=None):
@@ -578,8 +585,8 @@ class LmdbStore(Store):
                 for ctx in cur.iternext_dup():
                     yield self._from_key(ctx)[0]
         else:
-            with self.cur('c:spo') as cur:
-                for ctx in cur.iternext_nodup():
+            with self.cur('c:') as cur:
+                for ctx in cur.iternext(values=False):
                     yield self._from_key(ctx)[0]
 
 
@@ -620,11 +627,11 @@ class LmdbStore(Store):
                 with self.data_env.begin(write=True) as wtxn:
                     with wtxn.cursor(self.dbs['t:st']) as cur:
                         ck = self._append(cur, (pk_c,))[0]
+                    with wtxn.cursor(self.dbs['c:']) as cur:
+                        cur.put(ck, b'')
                 with self.idx_env.begin(write=True) as wtxn:
                     with wtxn.cursor(self.dbs['th:t']) as cur:
                         cur.put(c_hash, ck)
-                    with wtxn.cursor(self.dbs['c:']) as cur:
-                        cur.put(ck, b'')
 
 
     def remove_graph(self, graph):
@@ -793,7 +800,7 @@ class LmdbStore(Store):
                 pk_t = cur.get(k)
                 terms.append(self._unpickle(pk_t))
 
-        return terms
+        return tuple(terms)
 
 
     def _to_key(self, obj):

tests/store/test_lmdb_store.py

@@ -16,6 +16,10 @@ def store():
     rmtree('/tmp/test_lmdbstore')
 
 
+def _clean(res):
+    return {r[0] for r in res}
+
+
 @pytest.mark.usefixtures('store')
 class TestStoreInit:
     '''
@@ -52,20 +56,25 @@ class TestStoreInit:
             pass
         assert not store.is_txn_open
 
-        with TxnManager(store) as txn:
-            raise RuntimeError
-        assert not store.is_txn_open
+        try:
+            with TxnManager(store) as txn:
+                raise RuntimeError
+        except RuntimeError:
+            assert not store.is_txn_open
 
 
     def test_rollback(self, store):
         '''
         Test rolling back a transaction.
         '''
-        with TxnManager(store, True) as txn:
-            store.add((
-                URIRef('urn:nogo:s'), URIRef('urn:nogo:p'),
-                URIRef('urn:nogo:o')))
-            raise RuntimeError # This should roll back the transaction.
+        try:
+            with TxnManager(store, True) as txn:
+                store.add((
+                    URIRef('urn:nogo:s'), URIRef('urn:nogo:p'),
+                    URIRef('urn:nogo:o')))
+                raise RuntimeError # This should roll back the transaction.
+        except RuntimeError:
+            pass
 
         with TxnManager(store) as txn:
             res = set(store.triples((None, None, None)))
@@ -86,12 +95,13 @@ class TestBasicOps:
         with TxnManager(store, True) as txn:
             store.add(trp)
 
-            #import pdb; pdb.set_trace()
             res1 = set(store.triples((None, None, None)))
             res2 = set(store.triples(trp))
             assert len(res1) == 1
             assert len(res2) == 1
-            assert trp in res1 & res2
+            clean_res1 = _clean(res1)
+            clean_res2 = _clean(res2)
+            assert trp in clean_res1 & clean_res2
 
 
     def test_triple_match_1bound(self, store):
@@ -102,11 +112,11 @@ class TestBasicOps:
             res1 = set(store.triples((URIRef('urn:test:s'), None, None)))
             res2 = set(store.triples((None, URIRef('urn:test:p'), None)))
             res3 = set(store.triples((None, None, URIRef('urn:test:o'))))
-            assert res1 == {(
+            assert _clean(res1) == {(
                 URIRef('urn:test:s'), URIRef('urn:test:p'),
                 URIRef('urn:test:o'))}
-            assert res2 == res1
-            assert res3 == res2
+            assert _clean(res2) == _clean(res1)
+            assert _clean(res3) == _clean(res2)
 
 
     def test_triple_match_2bound(self, store):
@@ -120,10 +130,10 @@ class TestBasicOps:
                 (URIRef('urn:test:s'), None, URIRef('urn:test:o'))))
             res3 = set(store.triples(
                 (None, URIRef('urn:test:p'), URIRef('urn:test:o'))))
-            assert res1 == {(
+            assert _clean(res1) == {(
                 URIRef('urn:test:s'), URIRef('urn:test:p'), URIRef('urn:test:o'))}
-            assert res2 == res1
-            assert res3 == res2
+            assert _clean(res2) == _clean(res1)
+            assert _clean(res3) == _clean(res2)
 
 
     def test_triple_no_match(self, store):
@@ -229,7 +239,7 @@ class TestContext:
 
         with TxnManager(store, True) as txn:
             store.add_graph(gr_uri)
-            assert gr_uri in store.contexts()
+            assert gr_uri in set(store.contexts())
 
 
     def test_empty_context(self, store):
@@ -241,7 +251,7 @@ class TestContext:
         with TxnManager(store, True) as txn:
             store.add_graph(gr_uri)
             assert gr_uri in store.contexts()
-            store.ermove_graph(gr_uri)
+            store.remove_graph(gr_uri)
             assert gr_uri not in store.contexts()
 
 
@@ -262,19 +272,21 @@ class TestContext:
             store.add(trp3, gr2_uri)
             store.add(trp3)
 
-            assert len(set(store.triples((None, None, None)))) == 2
+            assert len(set(store.triples((None, None, None)))) == 3
+            assert len(set(store.triples((None, None, None),
+                store.DEFAULT_GRAPH_URI))) == 2
             assert len(set(store.triples((None, None, None), gr_uri))) == 2
             assert len(set(store.triples((None, None, None), gr2_uri))) == 1
 
             assert gr2_uri in store.contexts()
-            assert trp1 not in store.triples((None, None, None))
-            assert trp1 not in store.triples((None, None, None),
-                    store.DEFAULT_GRAPH_URI)
-            assert trp2 in store.triples((None, None, None), gr_uri)
-            assert trp2 in store.triples((None, None, None))
-            assert trp3 in store.triples((None, None, None), gr2_uri)
-            assert trp3 in store.triples((None, None, None),
-                    store.DEFAULT_GRAPH_URI)
+            assert trp1 in _clean(store.triples((None, None, None)))
+            assert trp1 not in _clean(store.triples((None, None, None),
+                    store.DEFAULT_GRAPH_URI))
+            assert trp2 in _clean(store.triples((None, None, None), gr_uri))
+            assert trp2 in _clean(store.triples((None, None, None)))
+            assert trp3 in _clean(store.triples((None, None, None), gr2_uri))
+            assert trp3 in _clean(store.triples((None, None, None),
+                    store.DEFAULT_GRAPH_URI))
 
 
     def test_delete_from_ctx(self, store):