volksdata/Makefile

# NOTE: only tested in GNU/Linux.

SHELL := /bin/bash

## Binary dependencies.

CC = gcc
AR = ar

# Global switches.
# Compile with debug symbols. Libraries are suffixed with _dbg and can coexist
# with non-debug libraries.
DEBUG := $(filter 1,$(DEBUG))

# Install in a user-owned directory provided as LOCAL_PREFIX or a default
# (~/.local), instead of a system directory.
LOCAL ?= 0

## Paths.

LOCAL_PREFIX ?= $(HOME)/.local
ifneq ($(LOCAL), 0)
PREFIX ?= $(LOCAL_PREFIX)
else
PREFIX ?= /usr/local
endif

BINDIR := $(PREFIX)/bin
LIBDIR := $(PREFIX)/lib
INCLUDEDIR := $(PREFIX)/include
BUILDDIR := ./build
TMPDIR ?= /tmp
VALGRIND_DUMP := $(TMPDIR)/volksdata_valgrind.log
CALLGRIND_DUMP := $(TMPDIR)/volksdata_callgrind.out
MASSIF_DUMP := $(TMPDIR)/volksdata_massif.out

INCLUDE_BASE := . -Iinclude -Iext/hashmap -Iext/log/src
INCLUDE := -I$(INCLUDE_BASE)
_CFLAGS = -std=gnu11 -Wall -Wextra -fPIC $(INCLUDE)
CFLAGS = $(if $(DEBUG),\
		 $(_CFLAGS) -Itest -O0 -ggdb -DDEBUG,\
		 $(_CFLAGS) -O3 -g0)

#$(info CFLAGS: $(CFLAGS))
# NOTE: -luuid is a Linux system library. Other OS's might need a different
# link or a non-system library built.
LDFLAGS := -L$(BUILDDIR) -L$(LIBDIR) -llmdb -lxxhash -luuid

PARSER = bin/lemon
LEMON_SRC = ext/sqlite/tool/lemon.c
CODEC_DIR = src/codec

# External sources compiled in core object.
EXT_SRC := $(wildcard ext/log/src/*.c) \
	  	  $(wildcard ext/hashmap/*.c)

# External headers of libraries compiled in core.
EXT_H := $(wildcard ext/log/src/*.h) \
	  	$(wildcard ext/hashmap/*.h)

VOLK_SRC = $(wildcard src/*.c)
SRC = $(EXT_SRC) $(VOLK_SRC)
TEST_SRC = $(wildcard test/*.c) test.c

EXT_OBJ := $(EXT_SRC:%.c=$(BUILDDIR)/%.o)
# TODO This is extremely convoluted, simplify if possible.
CODEC_SRC := $(wildcard $(CODEC_DIR)/codec_*.c)
CODEC_REL_SRC := $(CODEC_SRC:$(CODEC_DIR)/%=%)
ALL_CODEC_REL_SRC := $(CODEC_REL_SRC) $(CODEC_REL_SRC:codec_%=parser_%) \
			$(CODEC_REL_SRC:codec_%=grammar_%)
CODEC_SRC = $(ALL_CODEC_REL_SRC:%=$(CODEC_DIR)/%)

CODEC_OBJ = $(if $(DEBUG),\
			$(patsubst $(CODEC_DIR)/%.c, $(BUILDDIR)/%_dbg.o, $(CODEC_SRC)),\
			$(patsubst $(CODEC_DIR)/%.c, $(BUILDDIR)/%.o, $(CODEC_SRC)))

STATIC_LIB = $(if $(DEBUG),\
			$(BUILDDIR)/libvolksdata_dbg.a,\
			$(BUILDDIR)/libvolksdata.a)

LOCAL_OBJ = $(if $(DEBUG),\
			$(VOLK_SRC:src/%.c=$(BUILDDIR)/%_dbg.o),\
			$(VOLK_SRC:src/%.c=$(BUILDDIR)/%.o))

OBJ = $(EXT_OBJ) $(LOCAL_OBJ)
DYN_LIB = $(STATIC_LIB:.a=.so)

LIBS = $(STATIC_LIB) $(DYN_LIB)

#$(info EXT_SRC: $(EXT_SRC))
#$(info EXT_OBJ: $(EXT_OBJ))
#$(info OBJ: $(OBJ))
#$(info LIBS: $(LIBS))

# LDD for Linux, otool -L for OSX.
ifeq (, $(shell which ldd))
LDD := otool -L
else
LDD := ldd
endif

# For visual dep graph.
DEPS := $(shell echo "${INCLUDE_BASE}" | sed -e 's/ -I/,/g'),include/codec
DOCS = docs

## Environment.

# Tests need the freshly compiled libs.
export LD_LIBRARY_PATH = $(LIBDIR)


## Rules.

.DEFAULT_GOAL := lib

# Extract all rule comments into a help message.
.PHONY: help
help:
	@echo "Command overview:"; echo
	grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
	| sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1|\3/p' \
	| column -t  -s '|'


.PHONY: lib
lib: codec $(LIBS) ## Compile main library (static and dynamic linking).


# Static library.
$(STATIC_LIB): $(OBJ)
	$(AR) rs $(STATIC_LIB) $^ $(CODEC_OBJ)


# Dynamic library.
$(DYN_LIB): $(OBJ)
	$(CC) -shared $(LDFLAGS) -o $(DYN_LIB) $^ $(CODEC_OBJ)


# External libraries.
$(BUILDDIR)/ext/%.o: ext/%.c
	mkdir -p $(dir $@)
	$(CC) $(CFLAGS) -c $^ -o $@


# Standard objects.
$(BUILDDIR)/%.o: src/%.c
	$(CC) $(CFLAGS) -c $^ -o $@


# Debug objects.
$(BUILDDIR)/%_dbg.o: src/%.c
	$(CC) $(CFLAGS) -c $^ -o $@


# Codecs in a subfolder.

.PHONY: codec
codec: $(PARSER)
	mkdir -p $(BUILDDIR)
	$(MAKE) -C $(CODEC_DIR) codec DEBUG=$(DEBUG)


# Build the parser executable.
$(PARSER): $(LEMON_SRC)
	mkdir -p $(dir $@)
	$(CC) $^ -o $@


install: lib ## Install library and dependencies to $PREFIX (or, if LOCAL = 1, to $LOCAL_PREFIX). May require sudo.
	@echo "Installing library files in $(PREFIX)."
	mkdir -p $(LIBDIR)
	mkdir -p $(INCLUDEDIR)/volksdata
	cp $(LIBS) $(LIBDIR)
	cp -r include/volksdata/* $(EXT_H) $(INCLUDEDIR)/volksdata
	cp include/*.h $(INCLUDEDIR)


.PHONY: clean
clean: ## Clean up artifacts.
	rm -rf $(BUILDDIR)


.PHONY: deepclean
deepclean: clean ## Clean up artifacts, including language parsers that normally are kept.
	rm -f include/codec/grammar_*.h
	rm -f src/codec/grammar_*.c src/codec/parser_*.c


.PHONY: uninstall ## Uninstall library.
uninstall:
	rm -f $(LIBDIR)/libvolksdata*
	rm -rf $(INCLUDEDIR)/volksdata*
	rm -f bin/test*


bin/test: lib $(TEST_SRC)
	$(CC) $(CFLAGS) $(LDFLAGS) -lvolksdata_dbg test.c -o bin/test


.PHONY: test
test: DEBUG = 1
test: bin/test
	LD_LIBRARY_PATH=$(BUILDDIR) exec bin/test


.PHONY: gdb_test
gdb_test: DEBUG = 1
gdb_test: bin/test  ## Run a test suite within gdb.
	LD_LIBRARY_PATH=$(BUILDDIR) exec gdb bin/test


.PHONY: memtest
memtest: DEBUG = 1
memtest: bin/test  ## Run test suite within Valgrind and report memory errors.
	LD_LIBRARY_PATH=$(BUILDDIR) valgrind \
	--leak-check=full --show-leak-kinds=all --track-origins=yes \
	--log-file=$(VALGRIND_DUMP) \
	./bin/test || true
	@echo "Memcheck complete. Valgrind log is at $(VALGRIND_DUMP)"


lint:
	splint \
		$(INCLUDE) -Itest \
		-DUINT_MAX=0xFFFFFFFFUL \
		-nullpass \
		-posix-lib \
		test.c


# Profiling application.
bin/profile: lib profile.c
	$(CC) $(CFLAGS) -g -DTESTING $(LDFLAGS) -lvolksdata_dbg \
		profile.c -o bin/profile


# Performance test application. Essentially the profiling code without debug.
bin/perftest: lib profile.c
	$(CC) $(CFLAGS) -g $(LDFLAGS) -lvolksdata profile.c -o bin/perftest


.PHONY: perftest
perftest: bin/perftest ## Run a performance test by creating, inserting and looking up triples.
	bin/perftest


.PHONY: profile
profile: DEBUG = 1
profile: bin/profile  ## Run a profiling session on a limited set of high-volume commands.
	LD_LIBRARY_PATH=$(BUILDDIR) VOLK_MDB_MAPSIZE=800000 exec valgrind --tool=callgrind \
		--callgrind-out-file="$(CALLGRIND_DUMP)" bin/profile 1000
	@echo "Profile dump written at $(CALLGRIND_DUMP) . Open it with "\
		"qcachegrind, kcachegrind, etc."


.PHONY: test_profile
test_profile: DEBUG = 1
test_profile: bin/profile  ## Run profiling on the standard test suite.
	LD_LIBRARY_PATH=$(BUILDDIR) VOLK_MDB_MAPSIZE=800000 exec valgrind --tool=callgrind \
		--callgrind-out-file="$(CALLGRIND_DUMP)" bin/test
	@echo "Profile dump written at $(CALLGRIND_DUMP) . Open it with "\
		"qcachegrind, kcachegrind, etc."


.PHONY: footprint
footprint: bin/perftest ## Measure memory footprint by generating and storing 100K triples.
	LD_LIBRARY_PATH=$(BUILDDIR) VOLK_MDB_MAPSIZE=80000000 exec valgrind --tool=massif \
		--massif-out-file=$(MASSIF_DUMP) bin/perftest 100000
	@echo "Memory stats file written at $(MASSIF_DUMP). Open it with "\
		"massif-visualizer or similar."


# Requires cinclude2dot (https://www.flourish.org/cinclude2dot) and Graphviz.
depgraph: $(VOLK_SRC) include/* include/volksdata/* include/volksdata/codec/* ## Build a visual dependency graph of the code.
	cinclude2dot --merge=module --include=$(DEPS) \
		--exclude='test|ext' >| $(DOCS)/dev/deps.dot
	dot $(DOCS)/dev/deps.dot -Tpdf >| $(DOCS)/dev/deps.pdf