Implement and wire methods for checksum lifecycle.

lakesuperior/model/ldpr.py

@@ -23,6 +23,7 @@ from lakesuperior.exceptions import (
     InvalidResourceError, RefIntViolationError, ResourceNotExistsError,
     ServerManagedTermError, TombstoneError)
 from lakesuperior.store.ldp_rs.rsrc_centric_layout import VERS_CONT_LABEL
+from lakesuperior.store.ldp_rs.metadata_store import MetadataStore
 from lakesuperior.toolbox import Toolbox
 
 
@@ -293,9 +294,10 @@ class Ldpr(metaclass=ABCMeta):
         # First verify that the instance IMR options correspond to the
         # "canonical" representation.
         if (
-                self.imr_options.get('incl_srv_mgd')
-                and not self.imr_options.get('incl_inbound')
-                and imr_options.get('incl_children')):
+                hasattr(self, '_imr_options')
+                and self._imr_options.get('incl_srv_mgd')
+                and not self._imr_options.get('incl_inbound')
+                and self._imr_options.get('incl_children')):
             gr = self.imr
         else:
             gr = rdfly.get_imr(
@@ -304,13 +306,21 @@ class Ldpr(metaclass=ABCMeta):
 
 
     @property
-    def digest(self):
+    def rsrc_digest(self):
         """
         Cryptographic digest of a resource.
 
         :rtype: str
         """
-        return self.canonical_graph.graph_digest()
+        # This RDFLib function seems to be based on an in-depth study of the
+        # topic of graph checksums; however the output is puzzling because it
+        # returns **65** hexadecimal characters, which are one too many to be
+        # a SHA256 and an odd number that cannot be converted to bytes.
+        # Therefore the string version is being converted to bytes for
+        # storage. See https://github.com/RDFLib/rdflib/issues/825
+        digest = self.canonical_graph.graph_digest()
+
+        return format(digest, 'x').encode('ascii')
 
 
     @property
@@ -750,7 +760,10 @@ class Ldpr(metaclass=ABCMeta):
         rdfly.modify_rsrc(self.uid, remove_trp, add_trp)
 
         # Calculate checksum (asynchronously).
-        Thread(target=self._update_checksum).run()
+        cksum_action = (
+                self._delete_checksum if ev_type == RES_DELETED
+                else self._update_checksum)
+        Thread(target=cksum_action).run()
 
         # Clear IMR buffer.
         if hasattr(self, '_imr'):
@@ -771,8 +784,14 @@ class Ldpr(metaclass=ABCMeta):
         """
         Save the resource checksum in a dedicated metadata store.
         """
-        pass
+        MetadataStore().update_checksum(self.uri, self.rsrc_digest)
+
 
+    def _delete_checksum(self):
+        """
+        Delete the resource checksum from the metadata store.
+        """
+        MetadataStore().delete_checksum(self.uri)
 
 
     def _enqueue_msg(self, ev_type, remove_trp=None, add_trp=None):

lakesuperior/store/base_lmdb_store.py

@@ -123,6 +123,8 @@ class BaseLmdbStore(metaclass=ABCMeta):
         Transaction context manager.
 
         :param bool write: Whether a write transaction is to be opened.
+
+        :rtype: lmdb.Transaction
         """
         try:
             txn = self.dbenv.begin(write=write)
@@ -152,12 +154,18 @@ class BaseLmdbStore(metaclass=ABCMeta):
 
         :rtype: lmdb.Cursor
         """
-        if txn is None:
-            txn = self.txn(write=write)
         db = None if index is None else self.dbs[index]
-        with txn as _txn:
-            try:
+
+        if txn is None:
+            with self.txn(write=write) as _txn:
                 cur = _txn.cursor(db)
                 yield cur
-            finally:
                 cur.close()
+        else:
+            try:
+                cur = txn.cursor(db)
+                yield cur
+            finally:
+                if cur:
+                    cur.close()
+                    cur = None

lakesuperior/store/ldp_rs/metadata_store.py

@@ -25,3 +25,38 @@ class MetadataStore(BaseLmdbStore):
     path = path.join(
         env.app_globals.config['application']['store']['ldp_rs']['location'],
         'metadata')
+
+
+    def get_checksum(self, uri):
+        """
+        Get the checksum of a resource.
+
+        :param str uri: Resource URI (``info:fcres...``).
+        :rtype: bytes
+        """
+        with self.cur(index='checksums') as cur:
+            cksum = cur.get(uri.encode('utf-8'))
+
+        return cksum
+
+
+    def update_checksum(self, uri, cksum):
+        """
+        Update the stored checksum of a resource.
+
+        :param str uri: Resource URI (``info:fcres...``).
+        :param bytes cksum: Checksum bytestring.
+        """
+        with self.cur(index='checksums', write=True) as cur:
+            cur.put(uri.encode('utf-8'), cksum)
+
+
+    def delete_checksum(self, uri):
+        """
+        Delete the stored checksum of a resource.
+
+        :param str uri: Resource URI (``info:fcres...``).
+        """
+        with self.cur(index='checksums', write=True) as cur:
+            if cur.set_key(uri.encode('utf-8')):
+                cur.delete()

tests/api/test_resource_api.py

@@ -13,6 +13,7 @@ from lakesuperior.exceptions import (
         TombstoneError)
 from lakesuperior.globals import RES_CREATED, RES_UPDATED
 from lakesuperior.model.ldpr import Ldpr
+from lakesuperior.store.ldp_rs.metadata_store import MetadataStore
 
 
 @pytest.fixture(scope='module')
@@ -456,6 +457,39 @@ class TestResourceCRUD:
                 rsrc_api.resurrect('{}/child{}'.format(uid, i))
 
 
+    def test_checksum(self):
+        """
+        Verify that a checksum is created and updated appropriately.
+        """
+        mds = MetadataStore()
+        root_cksum1 = mds.get_checksum(nsc['fcres']['/'])
+        uid = '/test_checksum'
+        rsrc_api.create_or_replace(uid)
+
+        mds = MetadataStore()
+        root_cksum2 = mds.get_checksum(nsc['fcres']['/'])
+        cksum1 = mds.get_checksum(nsc['fcres'][uid])
+
+        assert len(cksum1)
+        assert root_cksum1 != root_cksum2
+
+        rsrc_api.update(
+                uid,
+                'DELETE {} INSERT {<> a <http://ex.org/ns#Hello> .} WHERE {}')
+
+        mds = MetadataStore()
+        cksum2 = mds.get_checksum(nsc['fcres'][uid])
+
+        assert cksum1 != cksum2
+
+        rsrc_api.delete(uid)
+
+        mds = MetadataStore()
+        cksum3 = mds.get_checksum(nsc['fcres'][uid])
+
+        assert cksum3 is None
+
+
 
 @pytest.mark.usefixtures('db')
 class TestResourceVersioning:

tests/store/test_metadata_store.py

@@ -0,0 +1,81 @@
+import pytest
+
+from hashlib import sha256
+
+from lakesuperior.store.ldp_rs.metadata_store import MetadataStore
+
+
+class TestMetadataStore:
+    """
+    Tests for the LMDB Metadata store.
+    """
+    def test_put_checksum(self):
+        """
+        Put a new checksum.
+        """
+        uri = 'info:fcres/test_checksum'
+        cksum = sha256(b'Bogus content')
+        mds = MetadataStore()
+        with mds.cur(index='checksums', write=True) as cur:
+            cur.put(uri.encode('utf-8'), cksum.digest())
+
+        with mds.cur(index='checksums') as cur:
+            assert cur.get(uri.encode('utf-8')) == cksum.digest()
+
+
+    def test_separate_txn(self):
+        """
+        Open a transaction and put a new checksum.
+
+        Same as test_put_checksum but wrapping the cursor in a separate
+        transaction. This is really to test the base store which is an abstract
+        class.
+        """
+        uri = 'info:fcres/test_checksum_separate'
+        cksum = sha256(b'More bogus content.')
+        mds = MetadataStore()
+        with mds.txn(True) as txn:
+            with mds.cur(index='checksums', txn=txn) as cur:
+                cur.put(uri.encode('utf-8'), cksum.digest())
+
+        with mds.txn() as txn:
+            with mds.cur(index='checksums', txn=txn) as cur:
+                assert cur.get(uri.encode('utf-8')) == cksum.digest()
+
+
+    def test_exception(self):
+        """
+        Test exceptions within cursor and transaction contexts.
+        """
+        uri = 'info:fcres/test_checksum_exception'
+        cksum = sha256(b'More bogus content.')
+        mds = MetadataStore()
+
+        class CustomError(Exception):
+            pass
+
+        with pytest.raises(CustomError):
+            with mds.txn() as txn:
+                raise CustomError()
+
+        with pytest.raises(CustomError):
+            with mds.txn() as txn:
+                with mds.cur(index='checksums', txn=txn) as cur:
+                    raise CustomError()
+
+        with pytest.raises(CustomError):
+            with mds.cur(index='checksums') as cur:
+                raise CustomError()
+
+        with pytest.raises(CustomError):
+            with mds.txn(write=True) as txn:
+                raise CustomError()
+
+        with pytest.raises(CustomError):
+            with mds.txn(write=True) as txn:
+                with mds.cur(index='checksums', txn=txn) as cur:
+                    raise CustomError()
+
+        with pytest.raises(CustomError):
+            with mds.cur(index='checksums', write=True) as cur:
+                raise CustomError()