lsup_rdf/src/codec/lexer_ttl.re

#include "codec/parser_ttl.h"
#include "codec/tokens_ttl.h"


/** @brief TTL is UTF-8 encoded.
 *
 * @sa https://www.w3.org/TeamSubmission/turtle/#sec-grammar
 *
 * `char` should be considered to be UTF-8 throughout this library, however,
 * setting YYCTYPE to char generates case labels outside of the char range.
 */
#define YYCTYPE     uint8_t
#define YYCURSOR    it->cur
#define YYMARKER    it->mar
#define YYLIMIT     it->lim
#define YYFILL      fill(it) == 0


typedef struct {
    FILE *          fh;                 // Input file handle.
    YYCTYPE         buf[CHUNK_SIZE],    // Start of buffer.
            *       lim,                // Position after the last available
                                        //   input character (YYLIMIT).
            *       cur,                // Next input character to be read
                                        //   (YYCURSOR)
            *       mar,                // Most recent match (YYMARKER)
            *       tok,                // Start of current token.
            *       bol;                // Address of the beginning of the
                                        //   current line (for debugging).
    unsigned        line;               // Current line no. (for debugging).
    unsigned        stmt;               // Current statement.
    unsigned        ct;                 // Number of parsed triples.
    bool            eof;                // if we have reached EOF.
    /*!stags:re2c format = "YYCTYPE *@@;"; */
} ParseIterator;

typedef struct {
    YYCTYPE *       data;
    size_t          size;
} ParserToken;


static int fill(ParseIterator *it)
{
    if (it->eof) {
        return 1;
    }
    const size_t shift = it->tok - it->buf;
    if (shift < 1) {
        return 2;
    }
    log_debug ("Shifting bytes: %lu", shift);
    memmove(it->buf, it->tok, it->lim - it->tok);
    it->lim -= shift;
    it->cur -= shift;
    it->mar -= shift;
    it->tok -= shift;
    it->lim += fread(it->lim, 1, shift, it->fh);
    /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */
    it->lim[0] = 0;
    it->eof |= it->lim < it->buf + CHUNK_SIZE - 1;
    return 0;
}


static void parse_init (ParseIterator *it, FILE *fh)
{
    it->fh = fh;
    it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE - 1;
    it->line = 1;
    it->stmt = 1;
    it->bol = it->buf;
    it->ct = 0;
    it->eof = 0;
    /*!stags:re2c format = "it->@@ = NULL; "; */
    fill (it);
}


// Parser interface. Required because Lemon doesn't export these in the header
// automatically.
/*
void *TTLParseAlloc( void*(*malloc)(size_t));
void TTLParseFree(void *pParser, void(*free)(void*) );
void TTLParse(void *pParser, int tokenCode, YYCTYPE *token, ...);
void TTLParseTrace(FILE *stream, char *zPrefix);
*/
void *TTLParseAlloc();
void TTLParse();
void TTLParseFree();

// Lexer.

static int lex (ParseIterator *it, YYCTYPE **token_p)
{

loop:

    it->tok = it->cur;

    *token_p = NULL;

    /*!re2c
    re2c:eof = 0;
    re2c:flags:8 = 1;
    re2c:flags:tags = 1;
    re2c:tags:expression = "it->@@";
    re2c:api:style = functions;
    re2c:define:YYFILL:naked = 1;


    // Character classes.
    EOL             = [\x0A\x0D];
    NCWS            = [\x09\x20] | EOL;
    HEX             = [\x30-\x39\x41-\x46];
    CHAR_BASE       = "\\u" HEX{4} | "\\U" HEX{8} | '\\'
                    | [\U0000005D-\U0010FFFF];
    CHARACTER       = CHAR_BASE | [\x20-\x5B];
    NSTART_CHAR     = [a-zA-Z_] | [\u00C0-\u00D6] | [\u00D8-\u00F6]
                    | [\u00F8-\u02FF] | [\u0370-\u037D] | [\u037F-\u1FFF]
                    | [\u200C-\u200D] | [\u2070-\u218F] | [\u2C00-\u2FEF]
                    | [\u3001-\uD7FF] | [\uF900-\uFDCF] | [\uFDF0-\uFFFD]
                    | [\U00010000-\U000EFFFF];
    NAME_CHAR       = NSTART_CHAR | '-'
                    | [0-9\u00B7\u0300-\u036F\u203F-\u2040];
    ECHAR           = CHARACTER | [\t\n\r];
    UCHAR           = (CHAR_BASE | ([\x20-\x5B] \ [\x3E])) | '>';
    SCHAR           = (CHAR_BASE | ([\x20-\x5B] \ [\x22])) | '"';
    LCHAR           = ECHAR | ["\x09\x0A\x0D];

    // Constructs.
    COMMENT         = '#' ( [^\x0A\x0D] )*;
    WS              = NCWS+ | COMMENT;
    INTEGER         = ('-' | '+')? [0-9]+;
    EXPONENT        = [eE] INTEGER;
    LANGUAGE        = [a-z]+ ('-' [a-z0-9]+)*;
    REL_IRI         = UCHAR*;

    // Token aliases.
    IRIREF          = '<' REL_IRI '>';
    PFX_NAME        = (NSTART_CHAR \ [_]) NAME_CHAR*;
    NAME            = NSTART_CHAR NAME_CHAR*;
    LSTRING         = [\x22]{3} LCHAR [\x22]{3};
    STRING          = [\x22] SCHAR [\x22];
    LANGTAG         = '@' LANGUAGE;
    DOUBLE          = ('-' | '+') ? ([0-9]+ '.' [0-9]* EXPONENT
                    | '.' ([0-9])+ EXPONENT | ([0-9])+ EXPONENT);
    DECIMAL         = ('-' | '+')?
                    ( [0-9]+ '.' [0-9]* | '.' ([0-9])+ | ([0-9])+ );
    BOOLEAN         = 'true' | 'false';
    //RDF_TYPE        = NCWS 'a' / WS;


    $ {
        log_debug ("End of document.");
        return T_EOF;
    }

    '.' {
        log_debug ("End of statement #%u.", it->stmt);
        it->stmt++;
        return T_EOS;
    }

    EOL {
        it->line ++;
        it->bol = YYCURSOR;
        log_debug ("New line: #%u.", it->line);
        goto loop;
    }

    IRIREF {
        *token_p = unescape_unicode (it->tok + 1);
        log_debug ("URI data: %s", *token_p);

        return T_IRIREF;
    }

    PFX_NAME {
        *token_p = unescape_unicode (it->tok);
        log_debug ("Prefix name: %s", *token_p);

        return T_PFX_NAME;
    }

    NAME {
        *token_p = unescape_unicode (it->tok);
        log_debug ("name: %s", *token_p);

        return T_IDNAME;
    }

    LSTRING {
        *token_p = unescape_unicode (it->tok + 3);
        log_debug ("Long string: %s", *token_p);

        return T_STRING;
    }

    STRING {
        *token_p = unescape_unicode (it->tok + 1);
        log_debug ("Long string: %s", *token_p);

        return T_STRING;
    }

    LANGTAG {
        *token_p = uint8_ndup (it->tok + 1, YYCURSOR - it->tok - 1);
        log_debug ("Lang tag: %s", *token_p);

        return T_LANGTAG;
    }

    INTEGER {
        // Normalize sign.
        size_t offset = *it->tok == '+' ? 1 : 0;

        *token_p = uint8_ndup (it->tok + offset, YYCURSOR - it->tok - 1);
        log_debug ("Integer: %s", *token_p);

        return T_INTEGER;
    }

    DOUBLE {
        // Normalize sign.
        size_t offset = *it->tok == '+' ? 1 : 0;

        *token_p = uint8_ndup (it->tok + offset, YYCURSOR - it->tok - 1);
        log_debug ("Integer: %s", *token_p);

        return T_DOUBLE;
    }

    DECIMAL {
        // Normalize sign.
        YYCTYPE offset = *it->tok == '+' ? 1 : 0;

        // Normalize trailing zeros in fractional part.
        size_t size = YYCURSOR - it->tok - 1;
        if (strchr ((char *)it->tok, '.'))
            for (YYCTYPE *i = YYCURSOR; *i == '0'; i--) *i = '\0';

        *token_p = uint8_ndup (it->tok + offset, size);
        log_debug ("Integer: %s", *token_p);

        return T_DECIMAL;
    }

    BOOLEAN {
        *token_p = uint8_ndup (it->tok, YYCURSOR - it->tok - 1);
        log_debug ("Boolean: %s", *token_p);

        return T_BOOLEAN;
    }

    '(' { return T_LPAREN; }
    ')' { return T_RPAREN; }
    '[' { return T_LBRACKET; }
    ']' { return T_RBRACKET; }
    ';' { return T_SEMICOLON; }
    ',' { return T_COMMA; }
    ':' { return T_COLON; }
    'a' { return T_RDF_TYPE; }
    '_:' { return T_BNODE_PFX; }
    '^^' { return T_DTYPE_MARKER; }
    '@base' {return T_BASE; }
    '@prefix' {return T_PREFIX; }

    COMMENT {
        size_t size = YYCURSOR - it->tok + 1;
        YYCTYPE *data = malloc (size);
        memcpy (data, it->tok, size);
        data [size - 1] = '\0';
        log_debug ("Comment: `%s`", data);
        free (data);

        goto loop;
    }

    * {
        log_debug (
            "Invalid token @ %lu: %s (\\x%x)",
            YYCURSOR - it->buf - 1, it->tok, *it->tok);

        return -1;
    }

    */
}


LSUP_rc
LSUP_ttl_parse_doc (FILE *fh, LSUP_Graph **gr_p, size_t *ct, char **err_p)
{
    *err_p = NULL;
    *gr_p = NULL;

    ParseIterator parse_it;
    parse_init (&parse_it, fh);

    void *parser = TTLParseAlloc (malloc);

    LSUP_rc rc;

    LSUP_TTLParserState *state = malloc (sizeof (*state));
    if (UNLIKELY (!state)) {
        rc = LSUP_MEM_ERR;
        goto finally;
    }
    state->base = NULL;

    state->nsm = LSUP_nsmap_new();
    // TODO add basic NS, critically xsd: and rdf:
    LSUP_Graph *gr = LSUP_graph_new (
            LSUP_iriref_new (NULL, NULL), LSUP_STORE_HTABLE, NULL, state->nsm, 0);
    if (UNLIKELY (!gr)) return LSUP_MEM_ERR;

    state->it = LSUP_graph_add_init (gr);
    if (UNLIKELY (!state->it)) {
        LSUP_graph_free (gr);
        return LSUP_MEM_ERR;
    }

    YYCTYPE *token;

    for (;;) {
        int ttype = lex (&parse_it, &token);

        if (ttype == -1) {
            char token[16] = {'\0'};
            strncpy (token, (const char *)parse_it.tok, 15);

            char *err_start = "Parse error near token `";

            char err_info [64];
            sprintf(
                    err_info, "[...]' at line %u, character %ld.\n",
                    parse_it.line, parse_it.cur - parse_it.bol);

            size_t err_size = strlen (err_start) + 16 + strlen(err_info);
            char *err_str = malloc (err_size);
            sprintf (err_str, "%s%s%s", err_start, token, err_info);

            rc = LSUP_VALUE_ERR;
            *err_p = err_str;

            goto finally;
        }

        TTLParse (parser, ttype, token, state);

        if (ttype == T_EOF) break;
    };

    if (ct) *ct = parse_it.ct;

    log_info ("Parsed %u triples.", parse_it.ct);
    log_debug ("Graph size: %lu", LSUP_graph_size (gr));

    rc = parse_it.ct > 0 ? LSUP_OK : LSUP_NORESULT;
    *gr_p = gr;

finally:
    TTLParse (parser, 0, NULL, state);
    TTLParseFree (parser, free);

    LSUP_graph_add_done (state->it);
    free (state);

    if (rc < 0) LSUP_graph_free (gr);

    return rc;
}