Stats UI; change open() signature for LmdbStore.

doc/notes/TODO

@@ -73,10 +73,10 @@
 
 # Alpha 6
 
-- [ ] Basic admin tools
+- [D] Optimize queries
+- [D] Optimize store
+- [W] Basic admin tools
 - [ ] Term query API
-- [ ] Optimize queries
-- [ ] Optimize store
 - [ ] Complete revert to version and resurrect
 
 # Alpha 7

lakesuperior/app.py

@@ -6,6 +6,7 @@ from logging.config import dictConfig
 
 from flask import Flask
 
+from lakesuperior.endpoints.admin import admin
 from lakesuperior.endpoints.ldp import ldp
 from lakesuperior.endpoints.main import main
 from lakesuperior.endpoints.query import query
@@ -43,6 +44,7 @@ def create_app(app_conf, logging_conf):
         'url_prefix': 'rest'
     })
     app.register_blueprint(query, url_prefix='/query')
+    app.register_blueprint(admin, url_prefix='/admin')
 
     # Initialize RDF store connector.
     conn_mod_name = app_conf['store']['ldp_rs']['connector']['module']

lakesuperior/endpoints/templates/base.html

@@ -36,6 +36,13 @@
                     <li><a href="/query/sparql">SPARQL Query</a></li>
                   </ul>
                 </li>
+                <li class="dropdown">
+                  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Administration<span class="caret"></span></a>
+                  <ul class="dropdown-menu">
+                    <li><a href="/admin/stats">Statistics</a></li>
+                    <li><a href="/admin/tools">Tools</a></li>
+                  </ul>
+                </li>
               </ul>
               <form class="navbar-form navbar-left">
                 <div class="form-group">

lakesuperior/store_layouts/ldp_rs/lmdb_store.py

@@ -1,5 +1,6 @@
 import hashlib
 import logging
+import os
 
 from contextlib import ContextDecorator, ExitStack
 from multiprocessing import Process
@@ -286,11 +287,11 @@ class LmdbStore(Store):
 
 
     def __init__(self, path, identifier=None):
+        self.path = path
         self.__open = False
 
         self.identifier = identifier or URIRef(pathname2url(abspath(path)))
         super().__init__(path)
-        self.path = path
 
         self._pickle = self.node_pickler.dumps
         self._unpickle = self.node_pickler.loads
@@ -324,7 +325,7 @@ class LmdbStore(Store):
         return self.__open
 
 
-    def open(self, path, create=True):
+    def open(self, configuration=None, create=True):
         '''
         Open the database.
 
@@ -336,7 +337,7 @@ class LmdbStore(Store):
         This method is called outside of the main transaction. All cursors
         are created separately within the transaction.
         '''
-        self._init_db_environments(path, create)
+        self._init_db_environments(create)
         if self.data_env == NO_STORE:
             return NO_STORE
         self.__open = True
@@ -352,12 +353,34 @@ class LmdbStore(Store):
             raise RuntimeError('Store must be opened first.')
         logger.info('Beginning a {} transaction.'.format(
             'read/write' if write else 'read-only'))
+
         self.data_txn = self.data_env.begin(buffers=True, write=write)
         self.idx_txn = self.idx_env.begin(buffers=False, write=write)
 
         self.is_txn_rw = write
 
 
+    def stats(self):
+        '''
+        Gather statistics about the database.
+        '''
+        stats = {
+            'data_db_stats': {
+                db_label: self.data_txn.stat(self.dbs[db_label])
+                for db_label in self.data_keys},
+
+            'idx_db_stats': {
+                db_label: self.idx_txn.stat(self.dbs[db_label])
+                for db_label in self.idx_keys},
+
+            'data_db_size': os.stat(self.data_env.path()).st_size,
+            'idx_db_size': os.stat(self.idx_env.path()).st_size,
+            'num_triples': len(self),
+        }
+
+        return stats
+
+
     @property
     def is_txn_open(self):
         '''
@@ -813,7 +836,7 @@ class LmdbStore(Store):
             yield from self._lookup(triple_pattern)
 
 
-    def _init_db_environments(self, path, create=True):
+    def _init_db_environments(self, create=True):
         '''
         Initialize the DB environment.
 
@@ -821,15 +844,16 @@ class LmdbStore(Store):
         (these may be even further split up depending on performance
         considerations).
 
-        @param path The base path to contain the databases.
         @param create (bool) If True, the environment and its databases are
         created.
         '''
+        path = self.path
         if not exists(path):
             if create is True:
                 makedirs(path)
             else:
                 return NO_STORE
+
         self.data_env = lmdb.open(path + '/main', subdir=False, create=create,
                 map_size=self.MAP_SIZE, max_dbs=4, readahead=False)
         self.idx_env = lmdb.open(path + '/index', subdir=False, create=create,

lakesuperior/store_layouts/ldp_rs/rsrc_centric_layout.py

@@ -184,7 +184,7 @@ class RsrcCentricLayout:
         store.destroy(store.path)
 
         self._logger.info('Initializing the graph store with system data.')
-        store.open(store.path)
+        store.open()
         with TxnManager(store, True):
             with open('data/bootstrap/rsrc_centric_layout.sparql', 'r') as f:
                 self.ds.update(f.read())
@@ -216,6 +216,20 @@ class RsrcCentricLayout:
         return self._parse_construct(qry, init_bindings=bindings)
 
 
+    def count_rsrc(self):
+        '''
+        Return a count of first-class resources, subdivided in "live" and
+        historic snapshots.
+        '''
+        with TxnManager(self.ds.store) as txn:
+            main = set(
+                    self.ds.graph(META_GR_URI)[ : nsc['foaf'].primaryTopic : ])
+            hist = set(
+                    self.ds.graph(HIST_GR_URI)[ : nsc['foaf'].primaryTopic : ])
+
+        return {'main': len(main), 'hist': len(hist)}
+
+
     def raw_query(self, qry_str):
         '''
         Perform a straight query to the graph store.