volksdata/src/codec/lexer_nt.re

#include "volksdata/codec/parser_nt.h"
#include "volksdata/codec/tokens_nt.h"


#define YYCTYPE     uint8_t
#define YYCURSOR    it->cur
#define YYMARKER    it->mar
#define YYLIMIT     it->lim
#define YYFILL      fill(it) == 0


typedef struct {
    FILE *          fh;                 // Input file handle.
    YYCTYPE         buf[CHUNK_SIZE],    // Start of buffer.
            *       lim,                // Position after the last available
                                        //   input character (YYLIMIT).
            *       cur,                // Next input character to be read
                                        //   (YYCURSOR)
            *       mar,                // Most recent match (YYMARKER)
            *       tok,                // Start of current token.
            *       bol;                // Address of the beginning of the
                                        //   current line (for debugging).
    unsigned        line;               // Current line no. (for debugging).
    unsigned        ct;                 // Number of parsed triples.
    bool            eof;                // if we have reached EOF.
    /*!stags:re2c format = "YYCTYPE *@@;"; */
} ParseIterator;


static int fill(ParseIterator *it)
{
    if (it->eof) {
        return 1;
    }
    const size_t shift = it->tok - it->buf;
    if (shift < 1) {
        return 2;
    }
    LOG_DEBUG("Shifting bytes: %lu", shift);
    memmove(it->buf, it->tok, it->lim - it->tok);
    it->lim -= shift;
    it->cur -= shift;
    it->mar -= shift;
    it->tok -= shift;
    it->lim += fread(it->lim, 1, shift, it->fh);
    /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */
    it->lim[0] = 0;
    it->eof |= it->lim < it->buf + CHUNK_SIZE - 1;
    return 0;
}


static void parse_init(ParseIterator *it, FILE *fh)
{
    it->fh = fh;
    it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE - 1;
    it->line = 1;
    it->bol = it->buf;
    it->ct = 0;
    it->eof = 0;
    /*!stags:re2c format = "it->@@ = NULL; "; */
    fill (it);
}


// Parser interface. Required here to silence linters.
void *NTParseAlloc();
void NTParse();
void NTParseFree();
#ifdef DEBUG
void NTParseTrace();
#endif

// Lexer.

static int lex (ParseIterator *it, VOLK_Term **term)
{
    const YYCTYPE *lit_data_e, *dtype_s, *lang_s;

loop:

    it->tok = it->cur;

    *term = NULL;

    /*!re2c
    re2c:eof = 0;
    re2c:flags:8 = 1;
    re2c:flags:tags = 1;
    re2c:tags:expression = "it->@@";
    re2c:api:style = functions;
    re2c:define:YYFILL:naked = 1;


    // For unresolved and partially resolved inconsistencies of the spec, see
    // https://lists.w3.org/Archives/Public/public-rdf-comments/2017Jun/0000.html
    _WS                 = [\x09\x20];
    WS                  = _WS+;
    EOL                 = [\x0D\x0A] (_WS | [\x0D\x0A])*;
    DOT                 = ".";
    HEX                 = [0-9A-Fa-f];
    ECHAR               = [\\] [tbnrf"'\\];
    UCHAR               = "\\u" HEX{4} | "\\U" HEX{8};
    PN_CHARS_BASE       = [A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF];
    PN_CHARS_U          = PN_CHARS_BASE | '_' | ':';
    PN_CHARS            = PN_CHARS_U | '-' | [0-9\u00B7\u0300-\u036F\u203F-\u2040];
    IRI_CHARS           = ([^\x00-\x20<>"{}|^`\\] | UCHAR)*;
    LITERAL_QUOTE       = ["] ([^\x22\x5C\x0A\x0D] | ECHAR|UCHAR)* ["];
    LANGTAG             = [@] [a-zA-Z]+ ("-" [a-zA-Z0-9]+)*;

    IRIREF              = [<] IRI_CHARS [>];
    LITERAL             = LITERAL_QUOTE @lit_data_e _WS* ("^^" _WS* @dtype_s IRIREF | @lang_s LANGTAG)?;
    BNODE               = "_:" ((PN_CHARS_U | [0-9]) ((PN_CHARS | ".")* PN_CHARS)?);
    COMMENT             = "#" .*;


    EOL {
        it->line ++;
        it->bol = YYCURSOR;
        LOG_DEBUG("New line: #%u.", it->line);
        return T_EOL;
    }

    $ {
        LOG_DEBUG("End of buffer.");
        return T_EOF;
    }

    IRIREF {
        YYCTYPE *data = unescape_unicode (it->tok + 1, YYCURSOR - it->tok - 2);

        LOG_DEBUG("URI data: %s", data);

        *term = VOLK_iriref_new ((char*)data);
        free (data);

        if (!UNLIKELY (term)) return -1;
        return T_IRIREF;
    }

    LITERAL {
        // Only unescape Unicode from data.
        size_t size = lit_data_e - it->tok - 2;
        YYCTYPE *data = unescape_unicode (it->tok + 1, size);
        LOG_TRACE("Literal data: %s", data);

        char *metadata = NULL;
        const YYCTYPE *md_marker;
        VOLK_TermType type = VOLK_TERM_LITERAL;

        if (dtype_s) {
            md_marker = dtype_s;
            size = YYCURSOR - md_marker - 1;
        } else if (lang_s) {
            type = VOLK_TERM_LT_LITERAL;
            md_marker = lang_s;
            size = YYCURSOR - md_marker;
        } else md_marker = NULL;

        if (md_marker) {
            metadata = malloc (size);
            memcpy (metadata, md_marker + 1, size);
            metadata [size - 1] = '\0';
            LOG_TRACE("metadata: %s", metadata);
        }

        if (type == VOLK_TERM_LITERAL) {
            VOLK_Term *dtype;
            dtype = (metadata ? VOLK_iriref_new ((char *) metadata) : NULL);

            *term = VOLK_literal_new ((char *) data, dtype);

        } else *term = VOLK_lt_literal_new ((char *) data, (char *) metadata);

        free (data);
        free (metadata);

        if (!UNLIKELY (term)) return -1;
        return T_LITERAL;
    }

    BNODE {
        YYCTYPE *data = unescape_unicode (it->tok + 2, YYCURSOR - it->tok - 2);

        LOG_DEBUG("BNode data: %s", data);

        *term = VOLK_term_new (VOLK_TERM_BNODE, (char*)data, NULL);
        free (data);

        if (!UNLIKELY (term)) return -1;
        return T_BNODE;
    }

    DOT {
        LOG_DEBUG("End of triple.");
        it->ct ++;

        return T_DOT;
    }

    WS {
        LOG_DEBUG("Separator.");

        return T_WS;
    }

    COMMENT {
        size_t size = YYCURSOR - it->tok + 1;
        YYCTYPE *data = malloc (size);
        memcpy (data, it->tok, size);
        data [size - 1] = '\0';
        LOG_DEBUG("Comment: `%s`", data);
        free (data);

        goto loop;
    }

    * {
        LOG_DEBUG(
            "Invalid token @ %lu: %s (\\x%x)",
            YYCURSOR - it->buf - 1, it->tok, *it->tok);

        return -1;
    }

    */
}


VOLK_rc
VOLK_nt_parse_term (const char *rep, VOLK_Term **term)
{
    FILE *fh = fmemopen ((void *)rep, strlen (rep), "r");

    ParseIterator it;
    parse_init (&it, fh);

    int ttype = lex (&it, term);

    fclose (fh);

    switch (ttype) {
        case T_IRIREF:
        case T_LITERAL:
        case T_BNODE:
            return VOLK_OK;
        default:
            return VOLK_PARSE_ERR;
    }
}

VOLK_rc
VOLK_nt_parse_doc (FILE *fh, VOLK_Graph **gr_p, size_t *ct, char **err_p)
{
    *err_p = NULL;
    *gr_p = NULL;

    ParseIterator parse_it;
    parse_init (&parse_it, fh);

#ifdef DEBUG
    NTParseTrace (stdout, "NT Parser > ");
#endif

    void *parser = NTParseAlloc (malloc);

    VOLK_rc rc;

    VOLK_Graph *gr = VOLK_graph_new (NULL, NULL);
    if (UNLIKELY (!gr)) return VOLK_MEM_ERR;

    VOLK_GraphIterator *it = VOLK_graph_add_init (gr);
    if (UNLIKELY (!it)) {
        VOLK_graph_free (gr);
        return VOLK_MEM_ERR;
    }

    VOLK_Term *term = NULL;

    for (;;) {
        int ttype = lex (&parse_it, &term);

        if (ttype == -1) {
            char token[16] = {};
            strncpy (token, (const char *)parse_it.tok, 15);

            char *err_start = "Parse error near token `";

            char err_info [64];
            sprintf(
                    err_info, "[...]' at line %u, character %ld.\n",
                    parse_it.line, parse_it.cur - parse_it.bol);

            size_t err_size = strlen (err_start) + 16 + strlen(err_info);
            char *err_str = malloc (err_size);
            sprintf (err_str, "%s%s%s", err_start, token, err_info);

            rc = VOLK_PARSE_ERR;
            *err_p = err_str;
            log_error (err_str);

            goto finally;
        }

        NTParse (parser, ttype, term, it);

        if (ttype == T_EOF) break;
    };

    if (ct) *ct = parse_it.ct;

    log_info ("Parsed %u triples.", parse_it.ct);
    LOG_DEBUG("Graph size: %lu", VOLK_graph_size (gr));

    rc = parse_it.ct > 0 ? VOLK_OK : VOLK_NORESULT;
    *gr_p = gr;

finally: ;
    NTParse (parser, 0, NULL, it);
    NTParseFree (parser, free);

    VOLK_graph_add_done (it);
    VOLK_term_free (term);

    if (rc < 0) VOLK_graph_free (gr);

    return rc;
}