#include "graph.h" #include "src/codec/nt_grammar.h" #include "nt_parser.h" #define YYCTYPE unsigned char #define YYCURSOR it->cur #define YYMARKER it->mar #define YYLIMIT it->lim #define YYFILL fill(it) == 0 /** * Max chunk size passed to scanner at each iteration. */ #ifdef LSUP_RDF_STREAM_CHUNK_SIZE #define CHUNK_SIZE LSUP_RDF_STREAM_CHUNK_SIZE #else #define CHUNK_SIZE 8192 #endif typedef struct { FILE * fh; // Input file handle. YYCTYPE buf[CHUNK_SIZE + 1],// Start of buffer. * lim, // Position after the last available // input character (YYLIMIT). * cur, // Next input character to be read // (YYCURSOR) * mar, // Most recent match (YYMARKER) * tok, // Start of current token. * bol; // Address of the beginning of the // current line (for debugging). unsigned line; // Current line no. (for debugging). unsigned ct; // Number of parsed triples. bool eof; // if we have reached EOF. /*!stags:re2c format = "YYCTYPE *@@;"; */ } ParseIterator; // TODO The opposite of this is in codec_nt.c. Find a better place for both. static inline char unescape_char(const char c) { switch (c) { case 't': return '\t'; case 'b': return '\b'; case 'n': return '\n'; case 'r': return '\r'; case 'f': return '\f'; default: return c; } } static int fill(ParseIterator *it) { if (it->eof) { return 1; } const size_t shift = it->tok - it->buf; if (shift < 1) { return 2; } log_debug ("Shifting bytes: %lu", shift); memmove(it->buf, it->tok, it->lim - it->tok); it->lim -= shift; it->cur -= shift; it->mar -= shift; it->tok -= shift; it->lim += fread(it->lim, 1, shift, it->fh); /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */ it->lim[0] = 0; it->eof |= it->lim < it->buf + CHUNK_SIZE; return 0; } static void parse_init(ParseIterator *it, FILE *fh) { it->fh = fh; it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE; it->line = 1; it->bol = it->buf; it->ct = 0; it->eof = 0; /*!stags:re2c format = "it->@@ = NULL; "; */ fill (it); } /** @brief Replace \uxxxx and \Uxxxxxxxx with Unicode bytes. */ static YYCTYPE *unescape_unicode (const YYCTYPE *esc_str, size_t size) { YYCTYPE *uc_str = malloc (size + 1); size_t j = 0; YYCTYPE tmp_chr[5]; for (size_t i = 0; i < size;) { if (esc_str[i] == '\\') { i++; // Skip over '\\' // 4-hex sequence. if (esc_str[i] == 'u') { i ++; // Skip over 'u' // Use tmp_chr to hold the hex string for the code point. memcpy(tmp_chr, esc_str + i, sizeof (tmp_chr) - 1); tmp_chr[4] = '\0'; uint32_t tmp_val = strtol ((char*)tmp_chr, NULL, 16); log_debug ("tmp_val: %d", tmp_val); // Reuse tmp_chr to hold the byte values for the code point. int nbytes = utf8_encode (tmp_val, tmp_chr); // Copy bytes into destination. memcpy (uc_str + j, tmp_chr, nbytes); log_debug ("UC byte value: %x %x", uc_str[j], uc_str[j + 1]); j += nbytes; i += 4; // 8-hex sequence. } else if (esc_str[i] == 'U') { i ++; // Skip over 'U' log_error ("UTF-16 sequence unescaping not yet implemented."); return NULL; // TODO encode UTF-16 // Unescape other escaped characters. } else uc_str[j++] = unescape_char(esc_str[i++]); } else { // Copy ASCII char verbatim. uc_str[j++] = esc_str[i++]; } } YYCTYPE *tmp = realloc (uc_str, j + 1); if (UNLIKELY (!tmp)) return NULL; uc_str = tmp; uc_str[j] = '\0'; return uc_str; } // Parser interface. void *ParseAlloc(); void Parse(); void ParseFree(); // Lexer. static int lex (ParseIterator *it, LSUP_Term **term) { const YYCTYPE *lit_data_e, *dtype_s, *lang_s; loop: it->tok = it->cur; *term = NULL; /*!re2c re2c:eof = 0; re2c:flags:8 = 1; re2c:flags:tags = 1; re2c:tags:expression = "it->@@"; re2c:api:style = functions; re2c:define:YYFILL:naked = 1; // For unresolved and partially resolved inconsistencies of the spec, see // https://lists.w3.org/Archives/Public/public-rdf-comments/2017Jun/0000.html _WS = [\x09\x20]; WS = _WS+; EOL = [\x0D\x0A] (_WS | [\x0D\x0A])*; DOT = [.]; HEX = [0-9A-Fa-f]; ECHAR = [\\] [tbnrf"'\\]; UCHAR = "\\u" HEX{4} | "\\U" HEX{8}; PN_CHARS_BASE = [A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF]; PN_CHARS_U = PN_CHARS_BASE | '_' | ':'; PN_CHARS = PN_CHARS_U | '-' | [0-9\u00B7\u0300-\u036F\u203F-\u2040]; IRI_CHARS = ([^\x00-\x20<>"{}|^`\\] | UCHAR)*; LITERAL_QUOTE = ["] ([^\x22\x5C\x0A\x0D] | ECHAR|UCHAR)* ["]; LANGTAG = [@] [a-zA-Z]+ ("-" [a-zA-Z0-9]+)*; IRIREF = [<] IRI_CHARS [>]; LITERAL = LITERAL_QUOTE @lit_data_e _WS* ("^^" _WS* @dtype_s IRIREF | @lang_s LANGTAG)?; BNODE = "_:" ((PN_CHARS_U | [0-9]) ((PN_CHARS | ".")* PN_CHARS)?); COMMENT = "#" .*; EOL { it->line ++; it->bol = YYCURSOR; log_debug ("New line: #%u.", it->line); return T_EOL; } $ { log_debug ("End of buffer."); return T_EOF; } IRIREF { YYCTYPE *data = unescape_unicode (it->tok + 1, YYCURSOR - it->tok - 2); log_debug ("URI data: %s", data); *term = LSUP_uri_new ((char*)data); free (data); return T_IRIREF; } LITERAL { // Only unescape Unicode from data. size_t size = lit_data_e - it->tok - 2; YYCTYPE *data = unescape_unicode (it->tok + 1, size); log_debug ("Literal data: %s", data); YYCTYPE *datatype = NULL, *lang = NULL; if (dtype_s) { size = YYCURSOR - dtype_s - 1; datatype = malloc (size); memcpy (datatype, dtype_s + 1, size); datatype [size - 1] = '\0'; log_debug ("datatype: %s", datatype); } if (lang_s) { size = YYCURSOR - lang_s; lang = malloc (size); memcpy (lang, lang_s + 1, size); lang [size - 1] = '\0'; log_debug ("lang: %s", lang); } *term = LSUP_term_new (LSUP_TERM_LITERAL, (char*)data, (char*)datatype, (char*)lang); free (data); free (datatype); free (lang); return T_LITERAL; } BNODE { YYCTYPE *data = unescape_unicode (it->tok + 2, YYCURSOR - it->tok - 2); log_debug ("BNode data: %s", data); *term = LSUP_term_new (LSUP_TERM_BNODE, (char*)data, NULL, NULL); free (data); return T_BNODE; } DOT { log_debug ("End of triple."); it->ct ++; return T_DOT; } WS { log_debug ("Separator."); return T_WS; } COMMENT { size_t size = YYCURSOR - it->tok + 1; YYCTYPE *data = malloc (size); memcpy (data, it->tok, size); data [size - 1] = '\0'; log_debug ("Comment: `%s`", data); free (data); goto loop; } * { log_debug ( "Invalid token @ %lu: %s (\\x%x)", YYCURSOR - it->buf - 1, it->tok, *it->tok); return -1; } */ } LSUP_rc LSUP_nt_parse_term (const char *rep, const LSUP_NSMap *map, LSUP_Term **term) { FILE *fh = fmemopen ((void *)rep, strlen (rep), "r"); ParseIterator it; parse_init (&it, fh); int ttype = lex (&it, term); fclose (fh); switch (ttype) { case T_IRIREF: case T_LITERAL: case T_BNODE: return LSUP_OK; default: return LSUP_VALUE_ERR; } } LSUP_rc LSUP_nt_parse_doc (FILE *fh, LSUP_Graph **gr_p, size_t *ct, char **err_p) { *err_p = NULL; *gr_p = NULL; ParseIterator parse_it; parse_init (&parse_it, fh); void *parser = ParseAlloc (malloc); LSUP_rc rc; LSUP_Graph *gr = LSUP_graph_new (LSUP_STORE_MEM); if (UNLIKELY (!gr)) return LSUP_MEM_ERR; LSUP_GraphIterator *it = LSUP_graph_add_init (gr); if (UNLIKELY (!it)) { LSUP_graph_free (gr); return LSUP_MEM_ERR; } LSUP_Term *term = NULL; for (;;) { int ttype = lex (&parse_it, &term); if (ttype == -1) { char token[16] = {'\0'}; strncpy (token, (const char *)parse_it.tok, 15); char *err_start = "Parse error near token `"; char err_info [64]; sprintf( err_info, "[...]' at line %u, character %ld.\n", parse_it.line, parse_it.cur - parse_it.bol); size_t err_size = strlen (err_start) + 16 + strlen(err_info); char *err_str = malloc (err_size); sprintf (err_str, "%s%s%s", err_start, token, err_info); rc = LSUP_VALUE_ERR; *err_p = err_str; goto finally; } Parse (parser, ttype, term, it); if (ttype == T_EOF) break; }; if (ct) *ct = parse_it.ct; log_info ("Parsed %u triples.", parse_it.ct); log_debug ("Graph size: %lu", LSUP_graph_size (gr)); rc = parse_it.ct > 0 ? LSUP_OK : LSUP_NORESULT; *gr_p = gr; finally: Parse (parser, 0, NULL, it); ParseFree (parser, free); LSUP_graph_add_done (it); LSUP_term_free (term); if (rc < 0) LSUP_graph_free (gr); return rc; }