Use separate Makefile for codecs.

.gitignore

@@ -113,6 +113,7 @@ sandbox/
 src/codec/grammar_*.c
 src/codec/grammar_*.h
 src/codec/parser_*.c
+include/codec/tokens_*.h
 
 # IDE
 .syntastic*

Makefile

@@ -4,44 +4,37 @@
 
 CC = gcc
 AR = ar
-LEXER = re2c
-PARSER = lemon
 
 
 ## Paths.
 
 PREFIX ?= /usr/local
-bindir := $(PREFIX)/bin
-libdir := $(PREFIX)/lib
+bindir ::= $(PREFIX)/bin
+libdir ::= $(PREFIX)/lib
 includedir = $(PREFIX)/include/lsup
 MDB_DIR = ext/openldap/libraries/liblmdb
 XXHASH_DIR = ext/xxHash
 VALGRIND_DUMP = /tmp/lsup_valgrind.log
 CALLGRIND_DUMP = /tmp/lsup_callgrind.out
 
-INCLUDE_BASE = . -Iinclude -I$(MDB_DIR) -I$(XXHASH_DIR) \
+INCLUDE_BASE ::= . -Iinclude -I$(MDB_DIR) -I$(XXHASH_DIR) \
 	-Iext/tpl/src -Iext/hashmap -Iext/log/src
-INCLUDE = -I$(INCLUDE_BASE)
-CFLAGS += -Wall -fPIC -MMD -DLOG_USE_COLOR $(INCLUDE)
+INCLUDE ::= -I$(INCLUDE_BASE)
+CFLAGS ::= -Wall -fPIC -MMD -DLOG_USE_COLOR $(INCLUDE)
 DBG_CFLAGS = -Itest -O0 -g3 -DDEBUG
 # NOTE: -luuid is a Linux system library. Other OS's might need a different
 # link or a non-system library built.
-LDFLAGS = -L. -L$(libdir) -llmdb -lxxhash -luuid
+LDFLAGS ::= -L. -L$(libdir) -llmdb -lxxhash -luuid
 
 CODEC_DIR = src/codec
-CODEC_SRC = $(wildcard src/codec/{codec,grammar,lexer}*.c)
-CODEC_OBJ = $(CODEC_SRC:.c=.o)
-CODEC_DBG_OBJ = $(CODEC_SRC:.c=_dbg.o)
-
-
 
 # External sources compiled in core object.
-EXT_SRC = $(wildcard ext/log/src/*.c) \
+EXT_SRC ::= $(wildcard ext/log/src/*.c) \
 	  	  $(wildcard ext/hashmap/*.c) \
 	  	  $(wildcard ext/tpl/src/*.c)
 
 # External headers of libraries compiled in core.
-EXT_H = $(wildcard ext/log/src/*.h) \
+EXT_H ::= $(wildcard ext/log/src/*.h) \
 	  	$(wildcard ext/tpl/src/*.h) \
 	  	$(wildcard ext/hashmap/*.h)
 
@@ -49,16 +42,29 @@ LSUP_SRC = $(wildcard src/*.c)
 SRC = $(EXT_SRC) $(LSUP_SRC)
 TEST_SRC = $(wildcard test/*.c) test.c
 
-EXT_OBJ = $(EXT_SRC:.c=.o)
-OBJ = $(EXT_OBJ) $(CODEC_OBJ) $(LSUP_SRC:.c=.o)
-DBG_OBJ = $(EXT_OBJ) $(CODEC_DBG_OBJ) $(LSUP_SRC:.c=_dbg.o)
+EXT_OBJ ::= $(EXT_SRC:.c=.o)
+# TODO This is extremely convoluted, simplify if possible.
+CODEC_SRC ::= $(wildcard $(CODEC_DIR)/codec_*.c)
+CODEC_REL_SRC ::= $(CODEC_SRC:$(CODEC_DIR)/%=%)
+ALL_CODEC_REL_SRC ::= $(CODEC_REL_SRC) $(CODEC_REL_SRC:codec_%=parser_%) \
+			$(CODEC_REL_SRC:codec_%=grammar_%)
+CODEC_SRC = $(ALL_CODEC_REL_SRC:%=$(CODEC_DIR)/%)
+CODEC_OBJ = $(CODEC_SRC:.c=.o)
+CODEC_DBG_OBJ = $(CODEC_SRC:.c=_dbg.o)
+OBJ = $(EXT_OBJ) $(LSUP_SRC:.c=.o)
+DBG_OBJ = $(EXT_OBJ) $(LSUP_SRC:.c=_dbg.o)
+
+$(warning Codec rel source: $(CODEC_REL_SRC))
+$(warning Codec source: $(CODEC_SRC))
+$(warning Codec Objects: $(CODEC_OBJ))
+$(warning Codec Debug objects: $(CODEC_DBG_OBJ))
 
 DEPLIBS = libxxhash liblmdb
 LIBS = liblsuprdf.a liblsuprdf.so
 DBG_LIBS = liblsuprdf_dbg.a liblsuprdf_dbg.so
 
 # For visual dep graph.
-DEPS := $(shell echo "${INCLUDE_BASE}" | sed -e 's/ -I/,/g')
+DEPS := $(shell echo "${INCLUDE_BASE}" | sed -e 's/ -I/,/g'),include/codec
 DOCS = docs
 
 
@@ -81,52 +87,50 @@ help:
 		| column -t  -s '|'
 	
 
-lib: $(DEPLIBS) $(LIBS) ## Compile main library (static and dynamic linking).
+lib: $(DEPLIBS) codec $(LIBS) ## Compile main library (static and dynamic linking).
 
 
-debug: $(DEPLIBS) $(DBG_LIBS) ## Compile main library with debug symbols.
+debug: $(DEPLIBS) codec_dbg $(DBG_LIBS) ## Compile main library with debug symbols.
 
 
 # Static library.
 liblsuprdf.a: $(OBJ)
-	$(AR) rs $@ $^
+	$(AR) rs $@ $^ $(CODEC_OBJ)
 
 
 # Dynamic library.
 liblsuprdf.so: $(OBJ)
-	$(CC) -shared $(LDFLAGS) -o $@ $^
+	$(CC) -shared $(LDFLAGS) -o $@ $^ $(CODEC_OBJ)
 
 
 # Static debug library.
 liblsuprdf_dbg.a: $(DBG_OBJ)
-	$(AR) rs $@ $^
+	$(AR) rs $@ $^ $(CODEC_DBG_OBJ)
 
 
 # Dynamic debug library.
 liblsuprdf_dbg.so: $(DBG_OBJ)
-	$(warning Making $(DBG_OBJ))
-	$(CC) -shared $(LDFLAGS) -o $@ $^
+	$(CC) -shared $(LDFLAGS) -o $@ $^ $(CODEC_DBG_OBJ)
 
 
 # Debug objects.
 %_dbg.o: %.c
-	$(CC) $(CFLAGS) $(DBG_CFLAGS) $(LDFLAGS) -c $^ -o $@
+	$(CC) $(CFLAGS) $(DBG_CFLAGS) -c $^ -o $@
 
 
-.PHONY: parsers
-parsers: $(PARSER_SRC) ## Make intermediate parser sources for development.
+# Codecs in a subfolder.
 
-# Codecs.
-# Parser C sources.
-$(CODEC_DIR)/parser_%.c: $(CODEC_DIR)/lexer_%.re $(CODEC_DIR)/grammar_%.c include/codec/tokens_%.h src/codec.c
-	$(LEXER) $< -o $@ -T --case-ranges
+.PHONY: parsers
+parsers: ## Make intermediate parser sources for development.
+	$(MAKE) -C $(CODEC_DIR) parsers
 
+.PHONY: codec
+codec:
+	$(MAKE) -C $(CODEC_DIR) codec
 
-.PRECIOUS: $(CODEC_DIR)/grammar_%.c include/codec/tokens_%.h
-# Parser generators.
-$(CODEC_DIR)/grammar_%.c include/codec/grammar_%.h: $(CODEC_DIR)/grammar_%.y
-	$(PARSER) $< -q -T$(CODEC_DIR)/lempar.c -d$(CODEC_DIR)
-	mv $(CODEC_DIR)/grammar_*.h include/codec
+.PHONY: codec_dbg
+codec_dbg:
+	$(MAKE) -C $(CODEC_DIR) debug
 
 
 # Ext libraries.
@@ -245,7 +249,7 @@ pytest: ## Run a test suite for the Python package.
 
 
 # Requires cinclude2dot (https://www.flourish.org/cinclude2dot) and Graphviz.
-depgraph: src/* include/* ## Build a visual dependency graph of the code.
+depgraph: $(LSUP_SRC) $(CODEC_SRC) include/* include/codec/* ## Build a visual dependency graph of the code.
 	cinclude2dot --merge=module --include=$(DEPS) \
 		--exclude='test|ext' >| $(DOCS)/dev/deps.dot
 	dot $(DOCS)/dev/deps.dot -Tpdf >| $(DOCS)/dev/deps.pdf

cpython/py_graph.h

@@ -7,7 +7,8 @@
 #include <structmember.h>
 
 #include "graph.h"
-#include "codec_nt.h"
+#include "codec/codec_nt.h"
+#include "codec/codec_ttl.h"
 #include "py_triple.h"
 
 

cpython/py_lsup_rdf.c

@@ -12,7 +12,6 @@
 #include <Python.h>
 
 #include "py_graph.h"
-#include "py_namespace.h"
 
 
 static PyObject *

docs/dev/deps.dot

@@ -5,44 +5,44 @@ digraph "source tree" {
     fontsize="16";
     fontname="Helvetica";
 	clusterrank="local";
-	"py_term" -> "py_namespace"
-	"namespace" -> "core"
-	"py_lsup_rdf" -> "py_namespace"
-	"store_htable" -> "store_base"
-	"store_mdb" -> "buffer"
-	"py_term" -> "term"
-	"graph" -> "environment"
-	"py_graph" -> "py_triple"
 	"store_mdb" -> "lmdb"
-	"store_base" -> "environment"
-	"core" -> "lmdb"
-	"store_htable" -> "buffer"
+	"buffer" -> "core"
 	"store_htable" -> "hashmap"
 	"py_triple" -> "py_term"
-	"py_namespace" -> "namespace"
-	"term" -> "tpl"
-	"nt_parser" -> "graph"
-	"environment" -> "bootstrap"
-	"codec_nt" -> "codec_base"
-	"py_graph" -> "codec_nt"
-	"nt_grammar" -> "graph"
-	"graph" -> "store"
-	"environment" -> "term"
-	"store" -> "store_htable"
-	"graph" -> "term"
-	"codec_nt" -> "nt_parser"
-	"namespace" -> "hashmap"
+	"graph" -> "environment"
 	"py_lsup_rdf" -> "py_graph"
+	"store_mdb" -> "store_interface"
+	"py_graph" -> "py_triple"
+	"py_term" -> "term"
 	"lsup_rdf" -> "codec_nt"
+	"store_interface" -> "environment"
+	"lsup_rdf" -> "codec_ttl"
+	"py_graph" -> "codec_nt"
+	"parser_nt" -> "codec"
+	"store_htable" -> "buffer"
 	"core" -> "log"
-	"buffer" -> "core"
-	"term" -> "buffer"
+	"codec" -> "graph"
+	"namespace" -> "hashmap"
+	"graph" -> "term"
+	"codec_ttl" -> "parser_ttl"
+	"term" -> "namespace"
 	"py_graph" -> "graph"
-	"store_mdb" -> "store_base"
-	"profile" -> "lsup_rdf"
+	"environment" -> "bootstrap"
+	"store_mdb" -> "buffer"
+	"store_htable" -> "store_interface"
+	"core" -> "lmdb"
+	"parser_ttl" -> "codec"
 	"core" -> "xxhash"
-	"term" -> "namespace"
-	"codec_base" -> "graph"
+	"term" -> "buffer"
 	"store" -> "store_mdb"
-	"nt_parser" -> "nt_grammar"
+	"graph" -> "store"
+	"codec_nt" -> "parser_nt"
+	"profile" -> "lsup_rdf"
+	"py_lsup_rdf" -> "py_namespace"
+	"py_term" -> "py_namespace"
+	"term" -> "tpl"
+	"py_namespace" -> "namespace"
+	"environment" -> "term"
+	"store" -> "store_htable"
+	"namespace" -> "core"
 }

src/codec/Makefile

@@ -0,0 +1,45 @@
+CC = gcc
+LEXER = re2c
+PARSER = lemon
+
+INCLUDE_DIR = ../../include
+CODEC_INCLUDE_DIR = $(INCLUDE_DIR)/codec
+
+CODEC_SRC ::= $(wildcard codec_*.c)
+PARSER_SRC ::= $(CODEC_SRC:codec_%=parser_%)
+CODEC_OBJ ::= $(CODEC_SRC:.c=.o)
+PARSER_OBJ ::= $(CODEC_OBJ:codec_%=parser_%)
+GRAMMAR_OBJ ::= $(CODEC_OBJ:codec_%=grammar_%)
+OBJ = $(GRAMMAR_OBJ) $(PARSER_OBJ) $(CODEC_OBJ)
+DBG_OBJ = $(OBJ:%.o=%_dbg.o)
+
+INCLUDE ::= -I$(INCLUDE_DIR) -I../../ext/openldap/libraries/liblmdb \
+	-I../../ext/xxHash -I../../ext/tpl/src -I../../ext/hashmap \
+	-I../../ext/log/src
+CFLAGS = -Wall -fPIC -MMD -DLOG_USE_COLOR $(INCLUDE)
+DBG_CFLAGS = -I../../test -O0 -g3 -DDEBUG
+
+.DEFAULT_GOAL := all
+
+.PHONY: codec
+codec: $(OBJ)
+
+.PHONY: debug
+debug: $(DBG_OBJ)
+
+.PHONY: parsers
+parsers: $(PARSER_SRC)
+
+%_dbg.o: %.c
+	$(CC) $(CFLAGS) $(DBG_CFLAGS) -c $^ -o $@
+
+# Parser C sources.
+parser_%.c: lexer_%.re grammar_%.c ../codec.c
+	$(LEXER) $< -o $@ -T --case-ranges
+
+
+.PRECIOUS: grammar_%.c $(CODEC_INCLUDE_DIR)/tokens_%.h
+# Parser generators.
+grammar_%.c $(CODEC_INCLUDE_DIR)/tokens_%.h: grammar_%.y
+	$(PARSER) $< -q -Tlempar.c -d.
+	mv grammar_$*.h $(CODEC_INCLUDE_DIR)/tokens_$*.h

src/codec/grammar_ttl.y

@@ -11,15 +11,12 @@
  */
 
 #include "codec.h"
-
-
-DEFINE XSD_PFX "http://www.w3.org/2001/XMLSchema#" // FIXME
 }
 
 %name TTLParse
 
 %stack_overflow {
-    log_error ("Stack oveflow in TTL parsing. Please jettison the parser.";
+    log_error ("Stack oveflow in TTL parsing. Please jettison the parser.");
 }
 
 %parse_failure {
@@ -30,12 +27,12 @@ DEFINE XSD_PFX "http://www.w3.org/2001/XMLSchema#" // FIXME
 
 %syntax_error {
     //UNUSED_PARAMETER (yymajor);  /* Silence some compiler warnings */
-    if (TOKEN.z[0]) log_error ("near \"%T\": syntax error", &TOKEN);
+    if (TOKEN[0]) log_error ("near \"%T\": syntax error", &TOKEN);
     else log_error ("incomplete input");
 }
 
 %token_prefix "T_"
-%token_type { uint8_t * }
+%token_type { char * }
 %token_destructor { free ($$); }
 
 /* NULL-terminated array of object term handles. */
@@ -46,7 +43,7 @@ DEFINE XSD_PFX "http://www.w3.org/2001/XMLSchema#" // FIXME
     }
 }
 
-%default_type           { uint8_t * }
+%default_type           { char * }
 
 %extra_argument         { LSUP_TTLParserState *state }
 
@@ -83,13 +80,13 @@ triples 	::= subject(S) predObjList(L) . {
                 LSUP_pred_obj_list_free (L);
             }
 
-%type predObjList       { PredObjList * }
+%type predObjList       { LSUP_PredObjList * }
 %destructor predObjList { LSUP_pred_obj_list_free ($$); }
 predObjList(A) ::= predObjList(A) SEMICOLON predicate(P) objectList(O) . {
                 return LSUP_pred_obj_list_add (A, P, O);
             }
 predObjList(A) ::= predicate(P) objectList(O) . {
-                A = LSUP_pred_ob_list_new();
+                A = LSUP_pred_obj_list_new();
                 return LSUP_pred_obj_list_add (A, P, O);
             }
 predObjList ::= predObjList SEMICOLON .
@@ -99,23 +96,25 @@ objectList(A) ::= objectList(L) COMMA object(O) . {
                 A = LSUP_obj_list_add (L, O);
             }
 objectList(A) ::= object(O) . {
-                A = calloc (sizeof (*A) * 2);
-                if (UNLIKELY (!A)) return LSUP_MEM_ERR; // TODO error handling
+                A = calloc (sizeof (*A), 2);
+                //if (UNLIKELY (!A)) return LSUP_MEM_ERR; // TODO error handling
                 A[0] = O;
             }
 
+%type subject { LSUP_Term * }
 subject 	::= resource .
 subject 	::= blank .
 
+%type predicate { LSUP_Term * }
 predicate   ::= resource .
 predicate(A)::= RDF_TYPE . { A = LSUP_iriref_new ("rdf:type", state->nsm); }
 
+%type object { LSUP_Term * }
 object 	    ::= resource .
 object 	    ::= blank .
 object 	    ::= literal .
 
 %type literal { LSUP_Term * }
-//%destructor literal { LSUP_term_free ($$); } // Destroyed with PO list.
 literal(A)  ::= STRING(D) . {
                 A = LSUP_term_new (LSUP_TERM_LITERAL, D, NULL);
             }
@@ -146,6 +145,7 @@ literal(A)  ::= BOOLEAN(D) . {
                     LSUP_iriref_new ("xsd:boolean", state->nsm));
             }
 
+%type blank { LSUP_Term * }
 blank(A)    ::= nodeID(D) . {
                 A = LSUP_term_new (LSUP_TERM_BNODE, D, NULL);
             }
@@ -154,7 +154,7 @@ blank(A)    ::= LBRACKET ows RBRACKET . {
             }
 blank(A)    ::= LBRACKET predObjList(L) RBRACKET . {
                 A = LSUP_term_new (LSUP_TERM_BNODE, NULL, NULL);
-                LSUP_spo_list_add_triples (A, L);
+                LSUP_spo_list_add_triples (state->it, A, L);
 
                 LSUP_pred_obj_list_free (L);
             }
@@ -175,15 +175,14 @@ collection(A) ::= LPAREN ows itemList(L) ows RPAREN . {
 %destructor itemList {}
 itemList(A) ::= itemList(L) WS object(O) . { A = LSUP_obj_list_add (L, O); }
 itemList(A) ::= object(O) . {
-                A = calloc (sizeof (*A) * 2);
-                if (UNLIKELY (!A)) return LSUP_MEM_ERR; // TODO error handling
+                A = calloc (sizeof (*A), 2);
+                //if (UNLIKELY (!A)) return LSUP_MEM_ERR; // TODO error handling
                 A[0] = O;
             }
 
 %type resource { LSUP_Term * }
-%destructor resource { LSUP_term_free ($$); }
 resource(A) ::= IRIREF(D) . {
-                LSUP_Term rel_iri = LSUP_iriref_new (D, NULL);
+                LSUP_Term *rel_iri = LSUP_iriref_new (D, NULL);
                 free (D);
                 if (state->base) {
                     A = LSUP_iriref_absolute (rel_iri, state->base);

src/codec/lexer_nt.re

@@ -1,5 +1,5 @@
-#include "codec/grammar_base_nt.h"
-#include "codec/grammar_nt.h"
+#include "codec/parser_nt.h"
+#include "codec/tokens_nt.h"
 
 
 #define YYCTYPE     uint8_t
@@ -63,6 +63,13 @@ static void parse_init(ParseIterator *it, FILE *fh)
 }
 
 
+// Parser interface. Required because Lemon doesn't export these in the header
+// automatically.
+void *NTParseAlloc();
+void NTParse();
+void NTParseFree();
+
+
 // Lexer.
 
 static int lex (ParseIterator *it, LSUP_Term **term)

src/codec/lexer_ttl.re

@@ -1,5 +1,5 @@
 #include "codec/parser_ttl.h"
-#include "codec/grammar_ttl.h"
+#include "codec/tokens_ttl.h"
 
 
 /** @brief TTL is UTF-8 encoded.

test/test_codec_nt.c

@@ -1,5 +1,5 @@
 #include "test.h"
-#include "codec_nt.h"
+#include "codec/codec_nt.h"
 
 #define TERM_CT 10