Merge pull request #38 from scossu/packaging

Packaging

.travis.yml

@@ -3,7 +3,6 @@ python:
   - "3.5"
   - "3.6"
 install:
-  - pip install -r requirements.txt
-  - coilmq&
+  - pip install -e .
 script:
-- pytest tests
+  - python setup.py test

README.rst

@@ -31,20 +31,17 @@ Dependencies
 Installation steps
 ~~~~~~~~~~~~~~~~~~
 
-1. Create a virtualenv in a project folder:
-   ``virtualenv -p <python 3.5+ exec path> <virtualenv folder>``
-2. Activate the virtualenv: ``source <path_to_virtualenv>/bin/activate``
-3. Clone this repo:
-   ``git clone https://github.com/scossu/lakesuperior.git``
-4. ``cd`` into repo folder
-5. Install dependencies: ``pip install -r requirements.txt``
-6. Start your STOMP broker, e.g.: ``coilmq &``. If you have another
+#. Create a virtualenv in a project folder:
+   ``python3 -m venv <virtualenv folder>``
+#. Activate the virtualenv: ``source <path_to_virtualenv>/bin/activate``
+#. Install dependencies: ``pip install -r requirements.txt``
+#. Start your STOMP broker, e.g.: ``coilmq &``. If you have another
    queue manager listening to port 61613 you can either configure a
    different port on the application configuration, or use the existing
    message queue.
-7. Run ``./lsup-admin bootstrap`` to initialize the binary and graph
-   stores
-8. Run ``./fcrepo``.
+#. Run ``lsup-admin bootstrap`` to initialize the binary and graph
+   stores.
+#. Run ``fcrepo``.
 
 Contributing
 ------------

docker/docker_entrypoint

@@ -2,8 +2,9 @@
 
 mkdir -p /data/log /data/run /data/bootstrap
 cp ./data/bootstrap/* /data/bootstrap
+pip install -e .
 coilmq &
 if [ ! -d /data/ldpnr_store ] && [ ! -d /data/ldprs_store ]; then
-  echo yes | ./lsup-admin bootstrap
+  echo yes | lsup-admin bootstrap
 fi
-exec ./fcrepo
+exec fcrepo

docker/etc/gunicorn.py

@@ -1,31 +0,0 @@
-# See: http://docs.gunicorn.org/en/stable/settings.html
-
-# Directory where to store logs, PIDfile, etc.
-_data_dir = '/data/'
-
-# Set app_mode to either 'prod', 'test' or 'dev'.
-# 'prod' is normal running mode. 'test' is used for running test suites.
-# 'dev' is similar to normal mode but with reload and debug enabled.
-_app_mode = 'dev'
-
-
-bind = "0.0.0.0:8000"
-workers = 4
-worker_class = 'gevent'
-max_requests = 512
-
-#user = "user"
-#group = "group"
-
-raw_env = 'APP_MODE={}'.format(_app_mode)
-
-# Set this to the directory containing logs, etc.
-# The path must end with a slash.
-#chdir = "/usr/local/lakesuperior/"
-
-daemon = _app_mode=='prod'
-pidfile = _data_dir + "run/fcrepo.pid"
-reload = _app_mode=='dev'
-
-accesslog = _data_dir + "log/gunicorn-access.log"
-errorlog = _data_dir + "log/gunicorn-error.log"

docker/etc/gunicorn.yml

@@ -0,0 +1,24 @@
+# Set up main GUnicorn options.
+# See: http://docs.gunicorn.org/en/stable/settings.html
+
+# Commented values are the application defaults.
+
+# Directory where the WSGI server data are stored.
+data_dir: 'data'
+
+# Set app_mode to either 'prod', 'test' or 'dev'.
+# 'prod' is normal running mode. 'test' is used for running test suites.
+# 'dev' is similar to normal mode but with reload and debug enabled.
+app_mode: 'dev'
+
+#listen_addr: '0.0.0.0'
+#listen_port: 8000
+#workers: 4
+#worker_class: 'gevent'
+#max_requests: 512
+
+#user: ''
+#group: ''
+
+#preload_app: True
+

docs/setup.rst

@@ -49,26 +49,14 @@ Installation steps
 #. Create a virtualenv in a project folder:
    ``python3 -m venv <virtualenv folder>``
 #. Activate the virtualenv: ``source <path_to_virtualenv>/bin/activate``
-#. Clone this repo:
-   ``git clone https://github.com/scossu/lakesuperior.git``
-#. ``cd`` into repo folder
 #. Install dependencies: ``pip install -r requirements.txt``
-
-   - (Optional) For a development server, install additional dependencies:
-     ``pip install -r requirements_dev.txt``. These include some heavyweight
-     packages needed for development and testing but not for regular operation.
-
-#. Start your STOMP broker, e.g.: ``coilmq &``.
-
-   - If you have another
-     queue manager listening to port 61613 you can either configure a
-     different port on the application configuration, or use the existing
-     message queue.
-
-#. Make sure that the ``lsup-admin`` and ``fcrepo`` files are executable.
-#. Run ``./lsup-admin bootstrap`` to initialize the binary and graph
+#. Start your STOMP broker, e.g.: ``coilmq &``. If you have another
+   queue manager listening to port 61613 you can either configure a
+   different port on the application configuration, or use the existing
+   message queue.
+#. Run ``lsup-admin bootstrap`` to initialize the binary and graph
    stores.
-#. Run ``./fcrepo``.
+#. Run ``fcrepo``.
 
 Configuration
 -------------
@@ -79,14 +67,14 @@ the ``data`` directory.
 
 To change the default configuration you should:
 
-1. Copy the ``etc.skeleton`` folder to a separate location
-2. Set the configuration folder location in the environment:
+#. Copy the ``etc.skeleton`` folder to a separate location
+#. Set the configuration folder location in the environment:
    ``export FCREPO_CONFIG_DIR=<your config dir location>`` (you can add
    this line at the end of your virtualenv ``activate`` script)
-3. Configure the application
-4. Bootstrap the app or copy the original data folders to the new
+#. Configure the application
+#. Bootstrap the app or copy the original data folders to the new
    location if any loction options changed
-5. (Re)start the server: ``./fcrepo``
+#. (Re)start the server: ``./fcrepo``
 
 The configuration options are documented in the files.
 

etc.defaults/gunicorn.py

@@ -1,32 +0,0 @@
-# See: http://docs.gunicorn.org/en/stable/settings.html
-
-# Directory where to store logs, PIDfile, etc.
-_data_dir = 'data/'
-
-# Set app_mode to either 'prod', 'test' or 'dev'.
-# 'prod' is normal running mode. 'test' is used for running test suites.
-# 'dev' is similar to normal mode but with reload and debug enabled.
-_app_mode = 'dev'
-
-
-bind = "0.0.0.0:8000"
-workers = 4
-worker_class = 'gevent'
-max_requests = 512
-
-#user = "user"
-#group = "group"
-
-raw_env = 'APP_MODE={}'.format(_app_mode)
-
-# Set this to the directory containing logs, etc.
-# The path must end with a slash.
-#chdir = "/usr/local/lakesuperior/"
-
-daemon = _app_mode=='prod'
-pidfile = _data_dir + "run/fcrepo.pid"
-reload = _app_mode=='dev'
-
-accesslog = _data_dir + "log/gunicorn-access.log"
-errorlog = _data_dir + "log/gunicorn-error.log"
-

etc.defaults/gunicorn.yml

@@ -0,0 +1,24 @@
+# Set up main GUnicorn options.
+# See: http://docs.gunicorn.org/en/stable/settings.html
+
+# Commented values are the application defaults.
+
+# Directory where the WSGI server data are stored.
+data_dir: 'data'
+
+# Set app_mode to either 'prod', 'test' or 'dev'.
+# 'prod' is normal running mode. 'test' is used for running test suites.
+# 'dev' is similar to normal mode but with reload and debug enabled.
+app_mode: 'dev'
+
+#listen_addr: '0.0.0.0'
+#listen_port: 8000
+#workers: 4
+#worker_class: 'gevent'
+#max_requests: 512
+
+#user: ''
+#group: ''
+
+#preload_app: True
+

fcrepo

@@ -1,5 +0,0 @@
-#!/bin/bash
-default_conf_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/etc.defaults"
-conf_dir=${FCREPO_CONFIG_DIR:-$default_conf_dir}
-
-gunicorn -c "${conf_dir}/gunicorn.py" server:fcrepo --preload

lsup-admin → lsup_admin.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import click
 import click_log
 import json

profiler.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 import logging
 from logging.config import dictConfig
 from werkzeug.contrib.profiler import ProfilerMiddleware
@@ -18,9 +16,11 @@ options = {
 
 from lakesuperior.app import create_app
 
-if __name__ == '__main__':
+def run():
     fcrepo = create_app(config['application'])
     fcrepo.wsgi_app = ProfilerMiddleware(fcrepo.wsgi_app, **options)
     fcrepo.config['PROFILE'] = True
     fcrepo.run(debug = True)
 
+if __name__ == '__main__':
+    run()

requirements.txt

@@ -1,14 +1,2 @@
-CoilMQ==1.0.1
-Flask==0.12.2
-HiYaPyCo==0.4.11
-PyYAML==3.12
-arrow==0.10.0
-click==6.7
-click-log==0.2.1
-gevent==1.2.2
-gunicorn==19.7.1
-lmdb==0.93
-rdflib==4.2.2
-requests-toolbelt==0.8.0
-requests==2.18.4
-stomp.py==4.1.20
+--index-url https://pypi.python.org/lakesuperior/
+-e .

requirements_dev.txt

@@ -1,6 +0,0 @@
-Pillow==4.3.0
-numpy==1.14.1
-pytest==3.2.2
-pytest-flask==0.10.0
-sphinx-rtd-theme==0.2.4
-wheel==0.30.0a0

server.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 import logging
 from logging.config import dictConfig
 

setup.cfg

@@ -0,0 +1,6 @@
+[aliases]
+test = pytest
+
+[tool:pytest]
+addopts = --verbose
+#python_files = tests

setup.py

@@ -0,0 +1,131 @@
+"""
+LAKEsuperior setup script.
+
+Proudly ripped from https://github.com/pypa/sampleproject/blob/master/setup.py
+"""
+
+import sys
+
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+# To use a consistent encoding
+from codecs import open
+from os import path
+
+here = path.abspath(path.dirname(__file__))
+
+# ``pytest_runner`` is referenced in ``setup_requires``.
+# See https://github.com/pytest-dev/pytest-runner#conditional-requirement
+needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv)
+pytest_runner = ['pytest-runner'] if needs_pytest else []
+
+
+# Get the long description from the README file
+with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name='lakesuperior',
+    version='1.0.0a91',
+
+    description='A Linked Data Platform repository sever.',
+    long_description=long_description,
+    long_description_content_type='text/x-rst; charset=UTF-8',
+
+    url='https://lakesuperior.readthedocs.io',
+
+    author='Stefano Cossu <@scossu>',
+    #author_email='',  # Optional
+    license='Apache License Version 2.0',
+
+    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+
+        'Environment :: Console',
+        'Environment :: Web Environment',
+
+        'Framework :: Flask',
+
+        'Intended Audience :: Developers',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: Science/Research',
+
+        'License :: OSI Approved :: Apache Software License',
+
+        'Natural Language :: English',
+
+        'Operating System :: MacOS',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX :: Linux',
+
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+
+        'Topic :: Database :: Database Engines/Servers',
+    ],
+
+    keywords='repository linked-data',
+
+    python_requires='~=3.5',
+
+    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
+
+    # Great reference read about dependency management:
+    # https://caremad.io/posts/2013/07/setup-vs-requirement/
+    install_requires=[
+        'CoilMQ',
+        'Flask',
+        'HiYaPyCo',
+        'PyYAML',
+        'arrow',
+        'click',
+        'click-log',
+        'gevent',
+        'gunicorn',
+        'lmdb',
+        'rdflib',
+        'requests',
+        'requests-toolbelt',
+        'sphinx-rtd-theme',
+        'stomp.py',
+    ],
+
+    setup_requires=[] + pytest_runner,
+    tests_require=[
+        'Pillow',
+        'numpy',
+        'pytest',
+        'pytest-flask',
+    ],
+
+    #extras_require={},
+    #package_data={},
+    #data_files=[],
+
+    entry_points={
+        'console_scripts': [
+            'lsup-admin=lsup_admin:admin',
+            'profiler=profiler:run',
+            'fcrepo=wsgi:run',
+        ],
+    },
+
+    # List additional URLs that are relevant to your project as a dict.
+    #
+    # This field corresponds to the "Project-URL" metadata fields:
+    # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use
+    #
+    # Examples listed include a pattern for specifying where the package tracks
+    # issues, where the source is hosted, where to say thanks to the package
+    # maintainers, and where to support the project financially. The key is
+    # what's used to render the link text on PyPI.
+    project_urls={  # Optional
+        'Source Code': 'https://github.com/scossu/lakesuperior/',
+        'Documentation': 'https://lakesuperior.readthedocs.io',
+        'Discussion': 'https://groups.google.com/forum/#!forum/lakesuperior',
+        'Bug Reports': 'https://github.com/scossu/lakesuperior/issues',
+    }
+)
+

wsgi.py

@@ -0,0 +1,69 @@
+import multiprocessing
+import yaml
+
+from os import environ, path
+
+import gunicorn.app.base
+
+from server import fcrepo
+
+
+default_config_dir = '{}/etc.defaults'.format(
+        path.dirname(path.abspath(__file__)))
+config_dir = environ.get('FCREPO_CONFIG_DIR', default_config_dir)
+config_file = '{}/gunicorn.yml'.format(config_dir)
+
+with open(config_file, 'r') as fh:
+    config = yaml.load(fh, yaml.SafeLoader)
+
+listen_addr = config.get('listen_addr', '0.0.0.0')
+listen_port = config.get('listen_port', 8000)
+preload_app = config.get('preload_app', True)
+app_mode = config.get('app_mode', 'prod')
+data_dir = path.realpath(config.get('data_dir'))
+
+def default_workers():
+    return (multiprocessing.cpu_count() * 2) + 1
+
+options = {
+    'bind': '{}:{}'.format(listen_addr, listen_port),
+    'workers': config.get('workers', default_workers()),
+    'worker_class': config.get('worker_class', 'gevent'),
+    'max_requests': config.get('max_requests', 512),
+
+    'user': config.get('user'),
+    'group': config.get('group'),
+
+    'raw_env': 'APP_MODE={}'.format(app_mode),
+
+    'preload_app': preload_app,
+    'daemon': app_mode=='prod',
+    'reload': app_mode=='dev' and not preload_app,
+
+    'pidfile': '{}/run/fcrepo.pid'.format(data_dir),
+    'accesslog': '{}/log/gunicorn-access.log'.format(data_dir),
+    'errorlog': '{}/log/gunicorn-error.log'.format(data_dir),
+}
+
+
+class WsgiApp(gunicorn.app.base.BaseApplication):
+
+    def __init__(self, app, options={}):
+        self.options = options
+        self.application = app
+        super(WsgiApp, self).__init__()
+
+    def load_config(self):
+        for key, value in self.options.items():
+            self.cfg.set(key.lower(), value)
+
+    def load(self):
+        return self.application
+
+
+def run():
+    WsgiApp(fcrepo, options).run()
+
+
+if __name__ == '__main__':
+    run()