Almost fix Python build and many other fixes.

MANIFEST.in

@@ -0,0 +1,3 @@
+graft include
+include cpython/*.h ext/hashmap/*.h ext/tpl/src/*.h ext/log/src/*.h ext/openldap/libraries/liblmdb/*.h
+include src/codec/Makefile

Makefile

@@ -21,7 +21,7 @@ MASSIF_DUMP = /tmp/lsup_massif.out
 INCLUDE_BASE ::= . -Iinclude -I$(MDB_DIR) -I$(XXHASH_DIR) \
 	-Iext/tpl/src -Iext/hashmap -Iext/log/src
 INCLUDE ::= -I$(INCLUDE_BASE)
-_CFLAGS ::= -Wall -fPIC -MMD -DLOG_USE_COLOR $(INCLUDE)
+_CFLAGS ::= -std=gnu11 -Wall -fPIC -MMD $(INCLUDE)
 CFLAGS = $(_CFLAGS) -O3
 DBG_CFLAGS = $(_CFLAGS) -Itest -O0 -g3 -DDEBUG
 # NOTE: -luuid is a Linux system library. Other OS's might need a different
@@ -244,16 +244,16 @@ profile: bin/profile ## Run a profiling session. Output can be inspected with KC
 
 
 .PHONY: footprint
-footprint: bin/perftest ## Measure memory footprint of storing 100K triples.
+footprint: bin/perftest ## Measure memory footprint by generating and storing 100K triples.
 	LSUP_MDB_MAPSIZE=80000000 valgrind --tool=massif \
 		--massif-out-file=$(MASSIF_DUMP) bin/perftest 100000
 	@echo "Memory stats file written at $(MASSIF_DUMP). Open it with "\
-		"massiftool or similar."
+		"massif-visualizer or similar."
 
 
 .PHONY: pytest
-pytest: ## Run a test suite for the Python package.
-	pip3 install --user .
+pytest: codec_dbg ## Run a test suite for the Python package.
+	pip3 install --user -e .
 	python3 test/cpython_test.py
 
 

cpython/py_graph.h

@@ -23,20 +23,21 @@
  */
 typedef struct {
     PyObject_HEAD
-    LSUP_CodecIterator *it;
-    unsigned char *line;
+    void *it;
+    const LSUP_Codec *codec;
+    char *line;
 } StringIteratorObject;
 
 
 static void
 StringIterator_dealloc (StringIteratorObject *it_obj)
-{ it_obj->it->codec->encode_graph_done (it_obj->it); }
+{ it_obj->codec->encode_graph_done (it_obj->it); }
 
 
 static PyObject *
 StringIterator_next (StringIteratorObject *it_obj)
 {
-    LSUP_rc rc = it_obj->it->codec->encode_graph_iter (
+    LSUP_rc rc = it_obj->codec->encode_graph_iter (
             it_obj->it, &it_obj->line);
     if (rc != LSUP_OK) {
         if (rc != LSUP_END)
@@ -91,7 +92,7 @@ GraphIterator_dealloc (GraphIteratorObject *it_obj)
 static PyObject *
 GraphIterator_next (GraphIteratorObject *it_obj)
 {
-    LSUP_rc rc = LSUP_graph_iter_next (it_obj->it, it_obj->spo);
+    LSUP_rc rc = LSUP_graph_iter_next (it_obj->it, &it_obj->spo);
     if (rc != LSUP_OK) {
         if (rc != LSUP_END)
             PyErr_SetString (PyExc_ValueError, "Error encoding graph.");
@@ -310,6 +311,7 @@ Graph_new_from_rdf (PyTypeObject *cls, PyObject *args)
 
     const LSUP_Codec *codec;
     if (strcmp(type, "nt") == 0) codec = &nt_codec;
+    else if (strcmp (type, "ttl") == 0) codec = &ttl_codec;
     // TODO other codecs here.
     else {
             PyErr_SetString (PyExc_ValueError, "Unsupported codec.");
@@ -558,23 +560,22 @@ Graph_encode (PyObject *self, PyObject *args)
 
     if (! PyArg_ParseTuple (args, "s", &type)) return NULL;
 
-    const LSUP_Codec *codec;
-    if (strcmp(type, "nt") == 0) codec = &nt_codec;
+    // Initialize the generator object.
+    StringIteratorObject *it_obj = PyObject_New (
+            StringIteratorObject, &StringIteratorType);
+    if (!it_obj) return NULL;
+
+    if (strcmp (type, "nt") == 0) it_obj->codec = &nt_codec;
+    else if (strcmp (type, "ttl") == 0) it_obj->codec = &ttl_codec;
     // TODO other codecs here.
     else {
         PyErr_SetString (PyExc_ValueError, "Unsupported codec.");
         return NULL;
     }
 
-    LSUP_CodecIterator *it = codec->encode_graph_init (
+    it_obj->it = it_obj->codec->encode_graph_init (
             ((GraphObject *)self)->ob_struct);
 
-    // Initialize the generator object.
-    StringIteratorObject *it_obj = PyObject_New (
-            StringIteratorObject, &StringIteratorType);
-    if (!it_obj) return NULL;
-
-    it_obj->it = it;
     it_obj->line = NULL;
 
     Py_INCREF (it_obj);

cpython/py_term.h

@@ -214,8 +214,7 @@ Term_lit_get_lang (TermObject *self, void *closure)
 {
     if (
             self->ob_struct->type != LSUP_TERM_LT_LITERAL
-            || ! self->ob_struct->lang
-            || strlen (self->ob_struct->lang) == 0)
+            || self->ob_struct->lang[0] == '\0')
         Py_RETURN_NONE;
 
     return PyUnicode_FromString (self->ob_struct->lang);

docs/dev/deps.dot

@@ -5,48 +5,48 @@ digraph "source tree" {
     fontsize="16";
     fontname="Helvetica";
 	clusterrank="local";
-	"py_namespace" -> "namespace"
-	"py_graph" -> "py_triple"
-	"namespace" -> "hashmap"
-	"lsup_rdf" -> "codec_nt"
-	"term" -> "buffer"
-	"core" -> "lmdb"
-	"codec_nt" -> "parser_nt"
+	"term" -> "namespace"
+	"profile" -> "lsup_rdf"
 	"store_htable" -> "buffer"
+	"grammar_ttl" -> "codec"
+	"graph" -> "environment"
 	"lsup_rdf" -> "codec_ttl"
+	"store_interface" -> "environment"
+	"lsup_rdf" -> "codec_nt"
+	"namespace" -> "hashmap"
+	"core" -> "lmdb"
+	"environment" -> "term"
 	"py_triple" -> "py_term"
-	"parser_ttl" -> "codec"
+	"core" -> "xxhash"
+	"store_mdb" -> "buffer"
+	"py_namespace" -> "namespace"
+	"store_htable" -> "store_interface"
+	"term" -> "buffer"
+	"graph" -> "store"
 	"py_lsup_rdf" -> "py_graph"
-	"parser_nt" -> "tokens_nt"
 	"store_htable" -> "hashmap"
-	"buffer" -> "core"
-	"py_graph" -> "codec_nt"
-	"store_mdb" -> "lmdb"
-	"environment" -> "bootstrap"
-	"environment" -> "term"
-	"graph" -> "term"
+	"py_term" -> "term"
+	"core" -> "log"
+	"parser_ttl" -> "codec"
+	"py_graph" -> "codec_ttl"
 	"store" -> "store_htable"
-	"store_interface" -> "environment"
-	"store_htable" -> "store_interface"
-	"parser_ttl" -> "tokens_ttl"
+	"codec_nt" -> "parser_nt"
+	"buffer" -> "core"
 	"py_term" -> "py_namespace"
-	"grammar_ttl" -> "codec"
-	"grammar_nt" -> "codec"
-	"graph" -> "environment"
-	"py_term" -> "term"
-	"store_mdb" -> "buffer"
-	"graph" -> "store"
+	"store_mdb" -> "store_interface"
+	"parser_nt" -> "tokens_nt"
 	"store" -> "store_mdb"
-	"term" -> "namespace"
-	"core" -> "xxhash"
-	"profile" -> "lsup_rdf"
 	"parser_nt" -> "codec"
-	"py_graph" -> "graph"
-	"codec" -> "graph"
-	"py_graph" -> "codec_ttl"
-	"store_mdb" -> "store_interface"
+	"parser_ttl" -> "tokens_ttl"
+	"graph" -> "term"
+	"environment" -> "bootstrap"
+	"store_mdb" -> "lmdb"
 	"term" -> "tpl"
+	"codec" -> "graph"
 	"codec_ttl" -> "parser_ttl"
+	"py_graph" -> "graph"
+	"py_graph" -> "py_triple"
+	"grammar_nt" -> "codec"
+	"py_graph" -> "codec_nt"
 	"namespace" -> "core"
-	"core" -> "log"
 }

include/core.h

@@ -1,6 +1,10 @@
 #ifndef _LSUP_CORE_H
 #define _LSUP_CORE_H
 
+#ifndef NOCOLOR
+#define LOG_USE_COLOR
+#endif
+
 #include <ctype.h>
 #include <dirent.h>
 #include <inttypes.h>
@@ -17,7 +21,7 @@
 #include "xxhash.h"
 
 
-#define LSUP_VERSION "1.0a1"
+#define LSUP_VERSION "1.0a2"
 
 
 // Logging and debugging.

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=60"]
+build-backend = "setuptools.build_meta"

setup.py

@@ -1,46 +1,64 @@
 from glob import glob
-from os import path
-from setuptools import Extension, setup
+from os import environ
+from setuptools import Extension, find_packages, setup
 from setuptools.command.install import install
-from subprocess import check_output, CalledProcessError
+from subprocess import check_output
 
 
-ROOT_DIR = path.dirname(path.realpath(__file__))
-MOD_DIR = path.join(ROOT_DIR, 'cpython')
-SRC_DIR = path.join(ROOT_DIR, 'src')
-CODEC_DIR = path.join(SRC_DIR, 'codec')
-INCL_DIR = path.join(ROOT_DIR, 'include')
-EXT_DIR = path.join(ROOT_DIR, 'ext')
+ROOT_DIR = "."
+SRC_DIR = f"{ROOT_DIR}/src"
+CODEC_DIR = f"{SRC_DIR}/codec"
+CPY_DIR = f"{ROOT_DIR}/cpython"
+INCL_DIR = f"{ROOT_DIR}/include"
+EXT_DIR = f"{ROOT_DIR}/ext"
 
-LEXER = 're2c'
-PARSER = 'lemon'
+LEXER = "re2c"
+PARSER = "lemon"
 
 
 sources = (
-    glob(path.join(SRC_DIR, '*.c')) +
-    glob(path.join(CODEC_DIR, '*_grammar.c')) +
-    glob(path.join(SRC_DIR, 'codec', '*_parser.c')) +
-    glob(path.join(MOD_DIR, '*.c')) +
+    glob(f"{SRC_DIR}/*.c") +
+    glob(f"{CODEC_DIR}/grammar_*.c") +
+    glob(f"{CODEC_DIR}/parser_*.c") +
+    glob(f"{CODEC_DIR}/codec_*.c") +
+    glob(f"{CPY_DIR}/*.c") +
     [
-        path.join(EXT_DIR, 'openldap', 'libraries', 'liblmdb', 'mdb.c'),
-        path.join(EXT_DIR, 'openldap', 'libraries', 'liblmdb', 'midl.c'),
-        path.join(EXT_DIR, 'xxHash', 'xxhash.c'),
-        path.join(EXT_DIR, 'hashmap', 'hashmap.c'),
-        path.join(EXT_DIR, 'tpl', 'src', 'tpl.c'),
-        path.join(EXT_DIR, 'log', 'src', 'log.c'),
+        f"{EXT_DIR}/openldap/libraries/liblmdb/mdb.c",
+        f"{EXT_DIR}/openldap/libraries/liblmdb/midl.c",
+        f"{EXT_DIR}/xxHash/xxhash.c",
+        f"{EXT_DIR}/hashmap/hashmap.c",
+        f"{EXT_DIR}/tpl/src/tpl.c",
+        f"{EXT_DIR}/log/src/log.c",
     ]
 )
 
-debug = True
+include_dirs = [
+    # ROOT_DIR,
+    INCL_DIR,
+    f"{INCL_DIR}/codec",
+    CPY_DIR,
+    f"{EXT_DIR}/hashmap",
+    f"{EXT_DIR}/tpl/src",
+    f"{EXT_DIR}/log/src",
+]
+
+debug = bool(environ.get("DEBUG"))
 
 compile_args = [
-    '-DLOG_USE_COLOR',
-    # '-std=c99',
+    "-std=gnu11",
+    "-Wall",
+    # "-Wextra",
+    "-fPIC",
+    "-MMD",
 ]
 if debug:
-    compile_args.extend(['-DDEBUG', '-g3', '-O0'])
+    print("Compiling with debug flags.")
+    compile_args.extend(["-DDEBUG", "-g3", "-O1"])
 else:
-    compile_args.extend(['-g0', '-O3'])
+    compile_args.extend(["-g0", "-O3"])
+
+# with open(path.join(ROOT_DIR, "README.md"), "r") as fh:
+#     long_description = fh.read()
 
 
 class LSUPInstallCmd(install):
@@ -52,30 +70,9 @@ class LSUPInstallCmd(install):
     """
 
     def run(self):
-        # Run grammar and parser generators.
-        try:
-            lexer_ex_path = check_output(['which', LEXER])
-        except CalledProcessError:
-            raise SystemError(f'Lexer program `{LEXER}` is not installed.')
-
-        try:
-            parser_ex_path = check_output(['which', PARSER])
-        except CalledProcessError:
-            raise SystemError(f'Lexer program `{PARSER}` is not installed.')
-
-        print("Generating grammar.")
-        for fpath in glob(path.join(CODEC_DIR, '*_grammar.y')):
-            check_output([
-                parser_ex_path, fpath, 'q', '-m',
-                '-T' + fpath.join(CODEC_DIR, 'lempar.c'), f'-d{CODEC_DIR}'
-            ])
-
-        print("Generating parser.")
-        for fpath in glob(path.join(CODEC_DIR, '*_lexer.re')):
-            check_output([
-                lexer_ex_path, fpath, '-o',
-                fpath.replace('_lexer.re', '_parser.c'), '-T', '--case-ranges',
-            ])
+        print("Generating parsers.")
+        target = "debug" if debug else "codec"
+        check_output(["make", "-C", CODEC_DIR, target])
 
         install.run(self)
 
@@ -83,26 +80,46 @@ class LSUPInstallCmd(install):
 setup(
     name="lsup_rdf",
     version="1.0a2",
-    description='Ultra-compact RDF library.',
-    author='Stefano Cossu <https://notabug.org/scossu>',
-    url='https://notabug.org/scossu/lsup_rdf',
-    license='https://notabug.org/scossu/lsup_rdf/src/master/LICENSE',
-    package_dir={'lsup_rdf': path.join(MOD_DIR, 'lsup_rdf')},
-    packages=['lsup_rdf'],
-    cmdclasss={'install': LSUPInstallCmd},
+    description="Ultra-compact RDF library.",
+    # long_description=long_description,
+    # long_description_content_type="text/markdown",
+    author="Stefano Cossu <https://git.knowledgetx.com/scossu>",
+    url="https://git.knowledgetx.com/scossu/lsup_rdf",
+    license="https://git.knowledgetx.com/scossu/lsup_rdf/src/master/LICENSE",
+    package_dir={"": "."},
+    packages=find_packages(where="."),
+    include_package_data=True,
+    cmdclass={"install": LSUPInstallCmd},
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "Intended Audience :: Science/Research",
+        "Intended Audience :: Telecommunications Industry",
+        "License :: Public Domain",
+        "Natural Language :: English",
+        "Operating System :: POSIX :: BSD",
+        "Operating System :: POSIX :: Linux",
+        "Operating System :: POSIX :: Other",
+        "Programming Language :: C",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Topic :: Database :: Database Engines/Servers",
+        "Topic :: Internet",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Software Development :: Embedded Systems",
+        "Topic :: Software Development :: Libraries",
+    ],
     ext_modules=[
         Extension(
             "_lsup_rdf",
             sources,
-            include_dirs=[
-                ROOT_DIR,
-                INCL_DIR,
-                path.join(EXT_DIR, 'hashmap'),
-                path.join(EXT_DIR, 'tpl', 'src'),
-                path.join(EXT_DIR, 'log', 'src'),
-            ],
-            libraries=['uuid'],
+            language="c",
+            include_dirs=include_dirs,
+            libraries=["xxhash", "lmdb", "uuid"],
             extra_compile_args=compile_args,
         ),
     ],
+    python_requires=">=3.8",
 )

src/codec/Makefile

@@ -18,10 +18,10 @@ DBG_OBJ = $(OBJ:%.o=%_dbg.o)
 INCLUDE ::= -I$(INCLUDE_DIR) -I../../ext/openldap/libraries/liblmdb \
 	-I../../ext/xxHash -I../../ext/tpl/src -I../../ext/hashmap \
 	-I../../ext/log/src
-CFLAGS = -Wall -fPIC -MMD -DLOG_USE_COLOR $(INCLUDE)
+CFLAGS = -std=gnu11 -Wall -fPIC -MMD $(INCLUDE)
 DBG_CFLAGS = -I../../test -O0 -g3 -DDEBUG
 
-.DEFAULT_GOAL := all
+.DEFAULT_GOAL := codec
 
 .PHONY: codec
 codec: $(OBJ)

src/codec/lexer_nt.re

@@ -67,7 +67,9 @@ static void parse_init(ParseIterator *it, FILE *fh)
 void *NTParseAlloc();
 void NTParse();
 void NTParseFree();
-void NTParseTrace();
+#ifdef DEBUG
+void NTParseTrace(FILE *TraceFILE, char *zTracePrompt);
+#endif
 
 // Lexer.
 

src/codec/lexer_ttl.re

@@ -86,7 +86,9 @@ static inline void newline (ParseIterator *it) {
 void *TTLParseAlloc();
 void TTLParse();
 void TTLParseFree();
-void TTLParseTrace();
+#ifdef DEBUG
+void TTLParseTrace(FILE *TraceFILE, char *zTracePrompt);
+#endif
 
 // Lexer.
 

src/graph.c

@@ -575,8 +575,8 @@ LSUP_graph_term_set (
     LSUP_Term *spo_l[3] = {NULL};
     spo_l[t1_pos] = t1;
     spo_l[t2_pos] = t2;
-    LSUP_TriplePos rpos;  // Position of term to be added to results.
-    for (int i = 0; i < 3; i++)
+    LSUP_TriplePos rpos = 0;  // Position of term to be added to results.
+    for (unsigned i = 0; i < 3; i++)
         if (t1_pos != i && t2_pos != i) rpos = i;
 
     LSUP_GraphIterator *it = LSUP_graph_lookup (

test/test_codec_ttl.c

@@ -120,10 +120,10 @@ int codec_ttl_tests()
     codec = ttl_codec;
 
     RUN (test_encode_ttl_graph);
-    //RUN (test_decode_nt_graph);
-    //RUN (test_decode_nt_bad_graph);
-    //RUN (test_w3c_pos);
-    //RUN (test_w3c_neg);
+    RUN (test_decode_nt_graph);
+    RUN (test_decode_nt_bad_graph);
+    RUN (test_w3c_pos);
+    RUN (test_w3c_neg);
 
     free_terms(terms);