|
@@ -0,0 +1,400 @@
|
|
|
|
+#include "nt_grammar.h"
|
|
|
|
+#include "nt_parser.h"
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+#define YYCTYPE unsigned char
|
|
|
|
+#define YYCURSOR it->cur
|
|
|
|
+#define YYMARKER it->mar
|
|
|
|
+#define YYLIMIT it->lim
|
|
|
|
+#define YYFILL fill(it) == 0
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * Max chunk size passed to scanner at each iteration.
|
|
|
|
+ */
|
|
|
|
+#ifdef LSUP_RDF_STREAM_CHUNK_SIZE
|
|
|
|
+#define CHUNK_SIZE LSUP_RDF_STREAM_CHUNK_SIZE
|
|
|
|
+#else
|
|
|
|
+#define CHUNK_SIZE 8192
|
|
|
|
+#endif
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+typedef struct {
|
|
|
|
+ FILE * fh; // Input file handle.
|
|
|
|
+ YYCTYPE buf[CHUNK_SIZE + 1],// Start of buffer.
|
|
|
|
+ * lim, // Position after the last available
|
|
|
|
+ // input character (YYLIMIT).
|
|
|
|
+ * cur, // Next input character to be read
|
|
|
|
+ // (YYCURSOR)
|
|
|
|
+ * mar, // Most recent match (YYMARKER)
|
|
|
|
+ * tok, // Start of current token.
|
|
|
|
+ * bol; // Address of the beginning of the
|
|
|
|
+ // current line (for debugging).
|
|
|
|
+ unsigned line; // Current line no. (for debugging).
|
|
|
|
+ unsigned ct; // Number of parsed triples.
|
|
|
|
+ bool eof; // if we have reached EOF.
|
|
|
|
+ /*!stags:re2c format = "YYCTYPE *@@;"; */
|
|
|
|
+} ParseIterator;
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+// TODO The opposite of this is in codec_nt.c. Find a better place for both.
|
|
|
|
+static inline char unescape_char(const char c) {
|
|
|
|
+ switch (c) {
|
|
|
|
+ case 't': return '\t';
|
|
|
|
+ case 'b': return '\b';
|
|
|
|
+ case 'n': return '\n';
|
|
|
|
+ case 'r': return '\r';
|
|
|
|
+ case 'f': return '\f';
|
|
|
|
+ default: return c;
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+static int fill(ParseIterator *it)
|
|
|
|
+{
|
|
|
|
+ if (it->eof) {
|
|
|
|
+ return 1;
|
|
|
|
+ }
|
|
|
|
+ const size_t shift = it->tok - it->buf;
|
|
|
|
+ if (shift < 1) {
|
|
|
|
+ return 2;
|
|
|
|
+ }
|
|
|
|
+ log_debug ("Shifting bytes: %lu", shift);
|
|
|
|
+ memmove(it->buf, it->tok, it->lim - it->tok);
|
|
|
|
+ it->lim -= shift;
|
|
|
|
+ it->cur -= shift;
|
|
|
|
+ it->mar -= shift;
|
|
|
|
+ it->tok -= shift;
|
|
|
|
+ it->lim += fread(it->lim, 1, shift, it->fh);
|
|
|
|
+ /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */
|
|
|
|
+ it->lim[0] = 0;
|
|
|
|
+ it->eof |= it->lim < it->buf + CHUNK_SIZE;
|
|
|
|
+ return 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+static void parse_init(ParseIterator *it, FILE *fh)
|
|
|
|
+{
|
|
|
|
+ it->fh = fh;
|
|
|
|
+ it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE;
|
|
|
|
+ it->line = 1;
|
|
|
|
+ it->bol = it->buf;
|
|
|
|
+ it->ct = 0;
|
|
|
|
+ it->eof = 0;
|
|
|
|
+ /*!stags:re2c format = "it->@@ = NULL; "; */
|
|
|
|
+ fill (it);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+/** @brief Replace \uxxxx and \Uxxxxxxxx with Unicode bytes.
|
|
|
|
+ */
|
|
|
|
+static YYCTYPE *unescape_unicode (const YYCTYPE *esc_str, size_t size)
|
|
|
|
+{
|
|
|
|
+ YYCTYPE *uc_str = malloc (size + 1);
|
|
|
|
+
|
|
|
|
+ size_t j = 0;
|
|
|
|
+ YYCTYPE tmp_chr[5];
|
|
|
|
+ for (size_t i = 0; i < size;) {
|
|
|
|
+ if (esc_str[i] == '\\') {
|
|
|
|
+ i++; // Skip over '\\'
|
|
|
|
+
|
|
|
|
+ // 4-hex sequence.
|
|
|
|
+ if (esc_str[i] == 'u') {
|
|
|
|
+ i ++; // Skip over 'u'
|
|
|
|
+
|
|
|
|
+ // Use tmp_chr to hold the hex string for the code point.
|
|
|
|
+ memcpy(tmp_chr, esc_str + i, sizeof (tmp_chr) - 1);
|
|
|
|
+ tmp_chr[4] = '\0';
|
|
|
|
+
|
|
|
|
+ uint32_t tmp_val = strtol ((char*)tmp_chr, NULL, 16);
|
|
|
|
+ log_debug ("tmp_val: %d", tmp_val);
|
|
|
|
+
|
|
|
|
+ // Reuse tmp_chr to hold the byte values for the code point.
|
|
|
|
+ int nbytes = utf8_encode (tmp_val, tmp_chr);
|
|
|
|
+
|
|
|
|
+ // Copy bytes into destination.
|
|
|
|
+ memcpy (uc_str + j, tmp_chr, nbytes);
|
|
|
|
+ log_debug ("UC byte value: %x %x", uc_str[j], uc_str[j + 1]);
|
|
|
|
+
|
|
|
|
+ j += nbytes;
|
|
|
|
+ i += 4;
|
|
|
|
+
|
|
|
|
+ // 8-hex sequence.
|
|
|
|
+ } else if (esc_str[i] == 'U') {
|
|
|
|
+ i ++; // Skip over 'U'
|
|
|
|
+ log_error ("UTF-16 sequence unescaping not yet implemented.");
|
|
|
|
+ return NULL; // TODO encode UTF-16
|
|
|
|
+
|
|
|
|
+ // Unescape other escaped characters.
|
|
|
|
+ } else uc_str[j++] = unescape_char(esc_str[i++]);
|
|
|
|
+ } else {
|
|
|
|
+ // Copy ASCII char verbatim.
|
|
|
|
+ uc_str[j++] = esc_str[i++];
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ YYCTYPE *tmp = realloc (uc_str, j + 1);
|
|
|
|
+ if (UNLIKELY (!tmp)) return NULL;
|
|
|
|
+ uc_str = tmp;
|
|
|
|
+ uc_str[j] = '\0';
|
|
|
|
+
|
|
|
|
+ return uc_str;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+// Parser interface.
|
|
|
|
+
|
|
|
|
+void *ParseAlloc();
|
|
|
|
+void Parse();
|
|
|
|
+void ParseFree();
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+// Lexer.
|
|
|
|
+
|
|
|
|
+static int lex (ParseIterator *it, LSUP_Term **term)
|
|
|
|
+{
|
|
|
|
+ const YYCTYPE *lit_data_e, *dtype_s, *lang_s;
|
|
|
|
+
|
|
|
|
+loop:
|
|
|
|
+
|
|
|
|
+ it->tok = it->cur;
|
|
|
|
+
|
|
|
|
+ *term = NULL;
|
|
|
|
+
|
|
|
|
+ /*!re2c
|
|
|
|
+ re2c:eof = 0;
|
|
|
|
+ re2c:flags:8 = 1;
|
|
|
|
+ re2c:flags:tags = 1;
|
|
|
|
+ re2c:tags:expression = "it->@@";
|
|
|
|
+ re2c:api:style = functions;
|
|
|
|
+ re2c:define:YYFILL:naked = 1;
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ // For unresolved and partially resolved inconsistencies of the spec, see
|
|
|
|
+ // https://lists.w3.org/Archives/Public/public-rdf-comments/2017Jun/0000.html
|
|
|
|
+ _WS = [\x09\x20];
|
|
|
|
+ WS = _WS+;
|
|
|
|
+ EOL = [\x0D\x0A] (_WS | [\x0D\x0A])*;
|
|
|
|
+ DOT = [.];
|
|
|
|
+ HEX = [0-9A-Fa-f];
|
|
|
|
+ ECHAR = [\\] [tbnrf"'\\];
|
|
|
|
+ UCHAR = "\\u" HEX{4} | "\\U" HEX{8};
|
|
|
|
+ PN_CHARS_BASE = [A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF];
|
|
|
|
+ PN_CHARS_U = PN_CHARS_BASE | '_' | ':';
|
|
|
|
+ PN_CHARS = PN_CHARS_U | '-' | [0-9\u00B7\u0300-\u036F\u203F-\u2040];
|
|
|
|
+ IRI_CHARS = ([^\x00-\x20<>"{}|^`\\] | UCHAR)*;
|
|
|
|
+ LITERAL_QUOTE = ["] ([^\x22\x5C\x0A\x0D] | ECHAR|UCHAR)* ["];
|
|
|
|
+ LANGTAG = [@] [a-zA-Z]+ ("-" [a-zA-Z0-9]+)*;
|
|
|
|
+
|
|
|
|
+ IRIREF = [<] IRI_CHARS [>];
|
|
|
|
+ LITERAL = LITERAL_QUOTE @lit_data_e _WS* ("^^" _WS* @dtype_s IRIREF | @lang_s LANGTAG)?;
|
|
|
|
+ BNODE = "_:" ((PN_CHARS_U | [0-9]) ((PN_CHARS | ".")* PN_CHARS)?);
|
|
|
|
+ COMMENT = "#" .*;
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ EOL {
|
|
|
|
+ it->line ++;
|
|
|
|
+ it->bol = YYCURSOR;
|
|
|
|
+ log_debug ("New line: #%u.", it->line);
|
|
|
|
+ return T_EOL;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ $ {
|
|
|
|
+ log_debug ("End of buffer.");
|
|
|
|
+ return T_EOF;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ IRIREF {
|
|
|
|
+ YYCTYPE *data = unescape_unicode (it->tok + 1, YYCURSOR - it->tok - 2);
|
|
|
|
+
|
|
|
|
+ log_debug ("URI data: %s", data);
|
|
|
|
+
|
|
|
|
+ *term = LSUP_iriref_new ((char*)data, NULL);
|
|
|
|
+ free (data);
|
|
|
|
+
|
|
|
|
+ return T_IRIREF;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ LITERAL {
|
|
|
|
+ // Only unescape Unicode from data.
|
|
|
|
+ size_t size = lit_data_e - it->tok - 2;
|
|
|
|
+ YYCTYPE *data = unescape_unicode (it->tok + 1, size);
|
|
|
|
+ log_trace ("Literal data: %s", data);
|
|
|
|
+
|
|
|
|
+ char *metadata = NULL;
|
|
|
|
+ const YYCTYPE *md_marker;
|
|
|
|
+ LSUP_TermType type = LSUP_TERM_LITERAL;
|
|
|
|
+
|
|
|
|
+ if (dtype_s) {
|
|
|
|
+ md_marker = dtype_s;
|
|
|
|
+ size = YYCURSOR - md_marker - 1;
|
|
|
|
+ } else if (lang_s) {
|
|
|
|
+ type = LSUP_TERM_LT_LITERAL;
|
|
|
|
+ md_marker = lang_s;
|
|
|
|
+ size = YYCURSOR - md_marker;
|
|
|
|
+ } else md_marker = NULL;
|
|
|
|
+
|
|
|
|
+ if (md_marker) {
|
|
|
|
+ metadata = malloc (size);
|
|
|
|
+ memcpy (metadata, md_marker + 1, size);
|
|
|
|
+ metadata [size - 1] = '\0';
|
|
|
|
+ log_trace ("metadata: %s", metadata);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (type == LSUP_TERM_LITERAL) {
|
|
|
|
+ LSUP_Term *dtype;
|
|
|
|
+ dtype = (
|
|
|
|
+ metadata ? LSUP_iriref_new ((char *) metadata, NULL) : NULL);
|
|
|
|
+
|
|
|
|
+ *term = LSUP_literal_new ((char *) data, dtype);
|
|
|
|
+
|
|
|
|
+ } else *term = LSUP_lt_literal_new ((char *) data, (char *) metadata);
|
|
|
|
+
|
|
|
|
+ free (data);
|
|
|
|
+ free (metadata);
|
|
|
|
+
|
|
|
|
+ return T_LITERAL;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ BNODE {
|
|
|
|
+ YYCTYPE *data = unescape_unicode (it->tok + 2, YYCURSOR - it->tok - 2);
|
|
|
|
+
|
|
|
|
+ log_debug ("BNode data: %s", data);
|
|
|
|
+
|
|
|
|
+ *term = LSUP_term_new (LSUP_TERM_BNODE, (char*)data, NULL);
|
|
|
|
+ free (data);
|
|
|
|
+
|
|
|
|
+ return T_BNODE;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ DOT {
|
|
|
|
+ log_debug ("End of triple.");
|
|
|
|
+ it->ct ++;
|
|
|
|
+
|
|
|
|
+ return T_DOT;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ WS {
|
|
|
|
+ log_debug ("Separator.");
|
|
|
|
+
|
|
|
|
+ return T_WS;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ COMMENT {
|
|
|
|
+ size_t size = YYCURSOR - it->tok + 1;
|
|
|
|
+ YYCTYPE *data = malloc (size);
|
|
|
|
+ memcpy (data, it->tok, size);
|
|
|
|
+ data [size - 1] = '\0';
|
|
|
|
+ log_debug ("Comment: `%s`", data);
|
|
|
|
+ free (data);
|
|
|
|
+
|
|
|
|
+ goto loop;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ * {
|
|
|
|
+ log_debug (
|
|
|
|
+ "Invalid token @ %lu: %s (\\x%x)",
|
|
|
|
+ YYCURSOR - it->buf - 1, it->tok, *it->tok);
|
|
|
|
+
|
|
|
|
+ return -1;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ */
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+LSUP_rc
|
|
|
|
+LSUP_nt_parse_term (const char *rep, const LSUP_NSMap *map, LSUP_Term **term)
|
|
|
|
+{
|
|
|
|
+ FILE *fh = fmemopen ((void *)rep, strlen (rep), "r");
|
|
|
|
+
|
|
|
|
+ ParseIterator it;
|
|
|
|
+ parse_init (&it, fh);
|
|
|
|
+
|
|
|
|
+ int ttype = lex (&it, term);
|
|
|
|
+
|
|
|
|
+ fclose (fh);
|
|
|
|
+
|
|
|
|
+ switch (ttype) {
|
|
|
|
+ case T_IRIREF:
|
|
|
|
+ case T_LITERAL:
|
|
|
|
+ case T_BNODE:
|
|
|
|
+ return LSUP_OK;
|
|
|
|
+ default:
|
|
|
|
+ return LSUP_VALUE_ERR;
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+LSUP_rc
|
|
|
|
+LSUP_nt_parse_doc (FILE *fh, LSUP_Graph **gr_p, size_t *ct, char **err_p)
|
|
|
|
+{
|
|
|
|
+ *err_p = NULL;
|
|
|
|
+ *gr_p = NULL;
|
|
|
|
+
|
|
|
|
+ ParseIterator parse_it;
|
|
|
|
+ parse_init (&parse_it, fh);
|
|
|
|
+
|
|
|
|
+ void *parser = ParseAlloc (malloc);
|
|
|
|
+
|
|
|
|
+ LSUP_rc rc;
|
|
|
|
+
|
|
|
|
+ LSUP_Graph *gr = LSUP_graph_new (
|
|
|
|
+ LSUP_iriref_new (NULL, NULL), LSUP_STORE_HTABLE, NULL, NULL, 0);
|
|
|
|
+ if (UNLIKELY (!gr)) return LSUP_MEM_ERR;
|
|
|
|
+
|
|
|
|
+ LSUP_GraphIterator *it = LSUP_graph_add_init (gr);
|
|
|
|
+ if (UNLIKELY (!it)) {
|
|
|
|
+ LSUP_graph_free (gr);
|
|
|
|
+ return LSUP_MEM_ERR;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ LSUP_Term *term = NULL;
|
|
|
|
+
|
|
|
|
+ for (;;) {
|
|
|
|
+ int ttype = lex (&parse_it, &term);
|
|
|
|
+
|
|
|
|
+ if (ttype == -1) {
|
|
|
|
+ char token[16] = {'\0'};
|
|
|
|
+ strncpy (token, (const char *)parse_it.tok, 15);
|
|
|
|
+
|
|
|
|
+ char *err_start = "Parse error near token `";
|
|
|
|
+
|
|
|
|
+ char err_info [64];
|
|
|
|
+ sprintf(
|
|
|
|
+ err_info, "[...]' at line %u, character %ld.\n",
|
|
|
|
+ parse_it.line, parse_it.cur - parse_it.bol);
|
|
|
|
+
|
|
|
|
+ size_t err_size = strlen (err_start) + 16 + strlen(err_info);
|
|
|
|
+ char *err_str = malloc (err_size);
|
|
|
|
+ sprintf (err_str, "%s%s%s", err_start, token, err_info);
|
|
|
|
+
|
|
|
|
+ rc = LSUP_VALUE_ERR;
|
|
|
|
+ *err_p = err_str;
|
|
|
|
+
|
|
|
|
+ goto finally;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ Parse (parser, ttype, term, it);
|
|
|
|
+
|
|
|
|
+ if (ttype == T_EOF) break;
|
|
|
|
+ };
|
|
|
|
+
|
|
|
|
+ if (ct) *ct = parse_it.ct;
|
|
|
|
+
|
|
|
|
+ log_info ("Parsed %u triples.", parse_it.ct);
|
|
|
|
+ log_debug ("Graph size: %lu", LSUP_graph_size (gr));
|
|
|
|
+
|
|
|
|
+ rc = parse_it.ct > 0 ? LSUP_OK : LSUP_NORESULT;
|
|
|
|
+ *gr_p = gr;
|
|
|
|
+
|
|
|
|
+finally:
|
|
|
|
+ Parse (parser, 0, NULL, it);
|
|
|
|
+ ParseFree (parser, free);
|
|
|
|
+
|
|
|
|
+ LSUP_graph_add_done (it);
|
|
|
|
+ LSUP_term_free (term);
|
|
|
|
+
|
|
|
|
+ if (rc < 0) LSUP_graph_free (gr);
|
|
|
|
+
|
|
|
|
+ return rc;
|
|
|
|
+}
|
|
|
|
+
|