lsup_rdf/src/codec/nt_lexer.re

#include "graph.h"
#include "src/codec/nt_grammar.h"
#include "nt_parser.h"


#define YYCTYPE     unsigned char
#define YYCURSOR    it->cur
#define YYMARKER    it->mar
#define YYLIMIT     it->lim
#define YYFILL      fill(it) == 0

/**
 * Max chunk size passed to scanner at each iteration.
 */
#ifdef LSUP_RDF_STREAM_CHUNK_SIZE
#define CHUNK_SIZE LSUP_RDF_STREAM_CHUNK_SIZE
#else
#define CHUNK_SIZE 8192
#endif


typedef struct {
    FILE *          fh;                 // Input file handle.
    YYCTYPE         buf[CHUNK_SIZE + 1],// Start of buffer.
            *       lim,                // Position after the last available
                                        //   input character (YYLIMIT).
            *       cur,                // Next input character to be read
                                        //   (YYCURSOR)
            *       mar,                // Most recent match (YYMARKER)
            *       tok,                // Start of current token.
            *       bol;                // Address of the beginning of the
                                        //   current line (for debugging).
    unsigned        line;               // Current line no. (for debugging).
    unsigned        ct;                 // Number of parsed triples.
    bool            eof;                // if we have reached EOF.
    /*!stags:re2c format = "YYCTYPE *@@;"; */
} ParseIterator;


// TODO The opposite of this is in codec_nt.c. Find a better place for both.
static inline char unescape_char(const char c) {
    switch (c) {
        case 't': return '\t';
        case 'b': return '\b';
        case 'n': return '\n';
        case 'r': return '\r';
        case 'f': return '\f';
        default: return c;
    }
}


static int fill(ParseIterator *it)
{
    if (it->eof) {
        return 1;
    }
    const size_t shift = it->tok - it->buf;
    if (shift < 1) {
        return 2;
    }
    log_debug ("Shifting bytes: %lu", shift);
    memmove(it->buf, it->tok, it->lim - it->tok);
    it->lim -= shift;
    it->cur -= shift;
    it->mar -= shift;
    it->tok -= shift;
    it->lim += fread(it->lim, 1, shift, it->fh);
    /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */
    it->lim[0] = 0;
    it->eof |= it->lim < it->buf + CHUNK_SIZE;
    return 0;
}


static void parse_init(ParseIterator *it, FILE *fh)
{
    it->fh = fh;
    it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE;
    it->line = 1;
    it->bol = it->buf;
    it->ct = 0;
    it->eof = 0;
    /*!stags:re2c format = "it->@@ = NULL; "; */
    fill (it);
}


/** @brief Replace \uxxxx and \Uxxxxxxxx with Unicode bytes.
 */
static YYCTYPE *unescape_unicode (const YYCTYPE *esc_str, size_t size)
{
    YYCTYPE *uc_str = malloc (size + 1);

    size_t j = 0;
    YYCTYPE tmp_chr[5];
    for (size_t i = 0; i < size;) {
        if (esc_str[i] == '\\') {
            i++; // Skip over '\\'

            // 4-hex sequence.
            if (esc_str[i] == 'u') {
                i ++; // Skip over 'u'

                // Use tmp_chr to hold the hex string for the code point.
                memcpy(tmp_chr, esc_str + i, sizeof (tmp_chr) - 1);
                tmp_chr[4] = '\0';

                uint32_t tmp_val = strtol ((char*)tmp_chr, NULL, 16);
                log_debug ("tmp_val: %d", tmp_val);

                // Reuse tmp_chr to hold the byte values for the code point.
                int nbytes = utf8_encode (tmp_val, tmp_chr);

                // Copy bytes into destination.
                memcpy (uc_str + j, tmp_chr, nbytes);
                log_debug ("UC byte value: %x %x", uc_str[j], uc_str[j + 1]);

                j += nbytes;
                i += 4;

            // 8-hex sequence.
            } else if (esc_str[i] == 'U') {
                i ++; // Skip over 'U'
                log_error ("UTF-16 sequence unescaping not yet implemented.");
                return NULL; // TODO encode UTF-16

            // Unescape other escaped characters.
            } else uc_str[j++] = unescape_char(esc_str[i++]);
        } else {
            // Copy ASCII char verbatim.
            uc_str[j++] = esc_str[i++];
        }
    }

    YYCTYPE *tmp = realloc (uc_str, j + 1);
    if (UNLIKELY (!tmp)) return NULL;
    uc_str = tmp;
    uc_str[j] = '\0';

    return uc_str;
}


// Parser interface.

void *ParseAlloc();
void Parse();
void ParseFree();


// Lexer.

static int lex (ParseIterator *it, LSUP_Term **term)
{
    const YYCTYPE *lit_data_e, *dtype_s, *lang_s;

loop:

    it->tok = it->cur;

    *term = NULL;

    /*!re2c
    re2c:eof = 0;
    re2c:flags:8 = 1;
    re2c:flags:tags = 1;
    re2c:tags:expression = "it->@@";
    re2c:api:style = functions;
    re2c:define:YYFILL:naked = 1;


    // For unresolved and partially resolved inconsistencies of the spec, see
    // https://lists.w3.org/Archives/Public/public-rdf-comments/2017Jun/0000.html
    _WS                 = [\x09\x20];
    WS                  = _WS+;
    EOL                 = [\x0D\x0A] (_WS | [\x0D\x0A])*;
    DOT                 = [.];
    HEX                 = [0-9A-Fa-f];
    ECHAR               = [\\] [tbnrf"'\\];
    UCHAR               = "\\u" HEX{4} | "\\U" HEX{8};
    PN_CHARS_BASE       = [A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF];
    PN_CHARS_U          = PN_CHARS_BASE | '_' | ':';
    PN_CHARS            = PN_CHARS_U | '-' | [0-9\u00B7\u0300-\u036F\u203F-\u2040];
    IRI_CHARS           = ([^\x00-\x20<>"{}|^`\\] | UCHAR)*;
    LITERAL_QUOTE       = ["] ([^\x22\x5C\x0A\x0D] | ECHAR|UCHAR)* ["];
    LANGTAG             = [@] [a-zA-Z]+ ("-" [a-zA-Z0-9]+)*;

    IRIREF              = [<] IRI_CHARS [>];
    LITERAL             = LITERAL_QUOTE @lit_data_e _WS* ("^^" _WS* @dtype_s IRIREF | @lang_s LANGTAG)?;
    BNODE               = "_:" ((PN_CHARS_U | [0-9]) ((PN_CHARS | ".")* PN_CHARS)?);
    COMMENT             = "#" .*;


    EOL {
        it->line ++;
        it->bol = YYCURSOR;
        log_debug ("New line: #%u.", it->line);
        return T_EOL;
    }

    $ {
        log_debug ("End of buffer.");
        return T_EOF;
    }

    IRIREF {
        YYCTYPE *data = unescape_unicode (it->tok + 1, YYCURSOR - it->tok - 2);

        log_debug ("URI data: %s", data);

        *term = LSUP_uri_new ((char*)data);
        free (data);

        return T_IRIREF;
    }

    LITERAL {
        // Only unescape Unicode from data.
        size_t size = lit_data_e - it->tok - 2;
        YYCTYPE *data = unescape_unicode (it->tok + 1, size);
        log_debug ("Literal data: %s", data);

        YYCTYPE *datatype = NULL, *lang = NULL;

        if (dtype_s) {
            size = YYCURSOR - dtype_s - 1;
            datatype = malloc (size);
            memcpy (datatype, dtype_s + 1, size);
            datatype [size - 1] = '\0';
            log_debug ("datatype: %s", datatype);
        }

        if (lang_s) {
            size = YYCURSOR - lang_s;
            lang = malloc (size);
            memcpy (lang, lang_s + 1, size);
            lang [size - 1] = '\0';
            log_debug ("lang: %s", lang);
        }

        *term = LSUP_term_new (LSUP_TERM_LITERAL, (char*)data, (char*)datatype, (char*)lang);

        free (data);
        free (datatype);
        free (lang);

        return T_LITERAL;
    }

    BNODE {
        YYCTYPE *data = unescape_unicode (it->tok + 2, YYCURSOR - it->tok - 2);

        log_debug ("BNode data: %s", data);

        *term = LSUP_term_new (LSUP_TERM_BNODE, (char*)data, NULL, NULL);
        free (data);

        return T_BNODE;
    }

    DOT {
        log_debug ("End of triple.");
        it->ct ++;

        return T_DOT;
    }

    WS {
        log_debug ("Separator.");

        return T_WS;
    }

    COMMENT {
        size_t size = YYCURSOR - it->tok + 1;
        YYCTYPE *data = malloc (size);
        memcpy (data, it->tok, size);
        data [size - 1] = '\0';
        log_debug ("Comment: `%s`", data);
        free (data);

        goto loop;
    }

    * {
        log_debug (
            "Invalid token @ %lu: %s (\\x%x)",
            YYCURSOR - it->buf - 1, it->tok, *it->tok);

        return -1;
    }

    */
}


LSUP_rc
LSUP_nt_parse_term (const char *rep, const LSUP_NSMap *map, LSUP_Term **term)
{
    FILE *fh = fmemopen ((void *)rep, strlen (rep), "r");

    ParseIterator it;
    parse_init (&it, fh);

    int ttype = lex (&it, term);

    fclose (fh);

    switch (ttype) {
        case T_IRIREF:
        case T_LITERAL:
        case T_BNODE:
            return LSUP_OK;
        default:
            return LSUP_VALUE_ERR;
    }
}

LSUP_rc
LSUP_nt_parse_doc (FILE *fh, LSUP_Graph **gr_p, size_t *ct, char **err_p)
{
    *err_p = NULL;
    *gr_p = NULL;

    ParseIterator parse_it;
    parse_init (&parse_it, fh);

    void *parser = ParseAlloc (malloc);

    LSUP_rc rc;

    LSUP_Graph *gr = LSUP_graph_new (LSUP_STORE_MEM);
    if (UNLIKELY (!gr)) return LSUP_MEM_ERR;

    LSUP_GraphIterator *it = LSUP_graph_add_init (gr);
    if (UNLIKELY (!it)) {
        LSUP_graph_free (gr);
        return LSUP_MEM_ERR;
    }

    LSUP_Term *term = NULL;

    for (;;) {
        int ttype = lex (&parse_it, &term);

        if (ttype == -1) {
            char token[16] = {'\0'};
            strncpy (token, (const char *)parse_it.tok, 15);

            char *err_start = "Parse error near token `";

            char err_info [64];
            sprintf(
                    err_info, "[...]' at line %u, character %ld.\n",
                    parse_it.line, parse_it.cur - parse_it.bol);

            size_t err_size = strlen (err_start) + 16 + strlen(err_info);
            char *err_str = malloc (err_size);
            sprintf (err_str, "%s%s%s", err_start, token, err_info);

            rc = LSUP_VALUE_ERR;
            *err_p = err_str;

            goto finally;
        }

        Parse (parser, ttype, term, it);

        if (ttype == T_EOF) break;
    };

    if (ct) *ct = parse_it.ct;

    log_info ("Parsed %u triples.", parse_it.ct);
    log_debug ("Graph size: %lu", LSUP_graph_size (gr));

    rc = parse_it.ct > 0 ? LSUP_OK : LSUP_NORESULT;
    *gr_p = gr;

finally:
    Parse (parser, 0, NULL, it);
    ParseFree (parser, free);

    LSUP_graph_add_done (it);
    LSUP_term_free (term);

    if (rc < 0) LSUP_graph_free (gr);

    return rc;
}