lexer_ttl.re 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. #include "codec/parser_ttl.h"
  2. #include "codec/tokens_ttl.h"
  3. /** @brief TTL is UTF-8 encoded.
  4. *
  5. * @sa https://www.w3.org/TeamSubmission/turtle/#sec-grammar
  6. *
  7. * `char` should be considered to be UTF-8 throughout this library, however,
  8. * setting YYCTYPE to char generates case labels outside of the char range.
  9. */
  10. #define YYCTYPE uint8_t
  11. #define YYCURSOR it->cur
  12. #define YYMARKER it->mar
  13. #define YYLIMIT it->lim
  14. #define YYFILL fill(it) == 0
  15. typedef struct {
  16. FILE * fh; // Input file handle.
  17. YYCTYPE buf[CHUNK_SIZE], // Start of buffer.
  18. * lim, // Position after the last available
  19. // input character (YYLIMIT).
  20. * cur, // Next input character to be read
  21. // (YYCURSOR)
  22. * mar, // Most recent match (YYMARKER)
  23. * tok, // Start of current token.
  24. * bol; // Address of the beginning of the
  25. // current line (for debugging).
  26. unsigned line; // Current line no. (for debugging).
  27. unsigned stmt; // Current statement.
  28. bool eof; // if we have reached EOF.
  29. /*!stags:re2c format = "YYCTYPE *@@;"; */
  30. } ParseIterator;
  31. typedef struct {
  32. YYCTYPE * data;
  33. size_t size;
  34. } ParserToken;
  35. static int fill (ParseIterator *it)
  36. {
  37. if (it->eof) {
  38. return 1;
  39. }
  40. const size_t shift = it->tok - it->buf;
  41. if (shift < 1) {
  42. return 2;
  43. }
  44. log_trace ("Shifting bytes: %lu", shift);
  45. memmove(it->buf, it->tok, it->lim - it->tok);
  46. it->lim -= shift;
  47. it->cur -= shift;
  48. it->mar -= shift;
  49. it->tok -= shift;
  50. it->lim += fread(it->lim, 1, shift, it->fh);
  51. /*!stags:re2c format = "if (it->@@) it->@@ -= shift; "; */
  52. it->lim[0] = 0;
  53. it->eof |= it->lim < it->buf + CHUNK_SIZE - 1;
  54. return 0;
  55. }
  56. static void parse_init (ParseIterator *it, FILE *fh)
  57. {
  58. it->fh = fh;
  59. it->cur = it->mar = it->tok = it->lim = it->buf + CHUNK_SIZE - 1;
  60. it->line = 1;
  61. it->stmt = 1;
  62. it->bol = it->buf;
  63. it->eof = 0;
  64. /*!stags:re2c format = "it->@@ = NULL; "; */
  65. fill (it);
  66. }
  67. // Parser interface. Required here to silence linters.
  68. void *TTLParseAlloc();
  69. void TTLParse();
  70. void TTLParseFree();
  71. void TTLParseTrace();
  72. // Lexer.
  73. static int lex (ParseIterator *it, YYCTYPE **token_p)
  74. {
  75. const YYCTYPE *pfx;
  76. /*!re2c
  77. re2c:eof = 0;
  78. re2c:flags:8 = 1;
  79. re2c:flags:tags = 1;
  80. re2c:tags:expression = "it->@@";
  81. re2c:api:style = functions;
  82. re2c:define:YYFILL:naked = 1;
  83. // Character classes.
  84. EOL = [\n\r];
  85. NCWS = [\t\x20] | EOL;
  86. HEX = [\x30-\x39\x41-\x46];
  87. CHAR_BASE = "\\u" HEX{4} | "\\U" HEX{8} | '\\'
  88. | [\U0000005D-\U0010FFFF];
  89. CHARACTER = CHAR_BASE | [\x20-\x5B];
  90. // Prefix start character.
  91. PSTART_CHAR = [a-zA-Z] | [\u00C0-\u00D6] | [\u00D8-\u00F6]
  92. | [\u00F8-\u02FF] | [\u0370-\u037D] | [\u037F-\u1FFF]
  93. | [\u200C-\u200D] | [\u2070-\u218F] | [\u2C00-\u2FEF]
  94. | [\u3001-\uD7FF] | [\uF900-\uFDCF] | [\uFDF0-\uFFFD]
  95. | [\U00010000-\U000EFFFF];
  96. // Name start character.
  97. NSTART_CHAR = PSTART_CHAR | '_';
  98. NAME_CHAR = NSTART_CHAR | [0-9\-\u00B7\u0300-\u036F\u203F-\u2040];
  99. ECHAR = CHARACTER | ([\\] [tnr]);
  100. UCHAR = (CHAR_BASE | ([\x20-\x5B] \ [>])) | ([\\] [>]);
  101. SCHAR = (CHAR_BASE | ([\x20-\x5B] \ ["])) | ([\\] ["]);
  102. LCHAR = ECHAR | ([\\] ["]) | [\t\n\r];
  103. // Constructs.
  104. COMMENT = '#' ( [^\n\r] )*;
  105. WS = NCWS+ | COMMENT;
  106. INTEGER = ('-' | '+')? [0-9]+;
  107. EXPONENT = [eE] INTEGER;
  108. DOUBLE = ('-' | '+') ? ([0-9]+ '.' [0-9]* EXPONENT
  109. | '.' ([0-9])+ EXPONENT | ([0-9])+ EXPONENT);
  110. DECIMAL = ('-' | '+')?
  111. ( [0-9]+ '.' [0-9]* | '.' ([0-9])+ | ([0-9])+ );
  112. */
  113. loop: // Start new token.
  114. it->tok = it->cur;
  115. *token_p = NULL;
  116. cont: // Continue token parsing. Do not move token start pointer.
  117. /*!re2c
  118. * {
  119. log_warn (
  120. "Invalid token @ %lu: %s (\\x%x)",
  121. YYCURSOR - it->buf - 1, it->tok, *it->tok);
  122. return -1;
  123. }
  124. $ {
  125. log_trace ("End of document.");
  126. return T_EOF;
  127. }
  128. EOL {
  129. it->line ++;
  130. it->bol = YYCURSOR;
  131. log_trace ("New line: #%u.", it->line);
  132. goto loop;
  133. }
  134. [\x22]{3} { goto lchar; }
  135. [\x22] { goto schar; }
  136. "true" | "false" {
  137. *token_p = uint8_ndup (it->tok, YYCURSOR - it->tok);
  138. log_trace ("Boolean: %s", *token_p);
  139. return T_BOOLEAN;
  140. }
  141. '<' UCHAR* '>' {
  142. *token_p = uint8_ndup (it->tok + 1, YYCURSOR - it->tok - 2);
  143. log_trace ("URI data: %s", *token_p);
  144. return T_IRIREF;
  145. }
  146. '@prefix' WS @pfx (PSTART_CHAR NAME_CHAR*)? ":" {
  147. *token_p = uint8_ndup (pfx, YYCURSOR - pfx - 1);
  148. log_trace ("Prefix declaration: '%s'", *token_p);
  149. return T_PREFIX;
  150. }
  151. '@base' {
  152. log_trace ("'@base' keyword.");
  153. return T_BASE;
  154. }
  155. (PSTART_CHAR NAME_CHAR*)? ":" (NSTART_CHAR NAME_CHAR*)? {
  156. *token_p = uint8_ndup (it->tok, YYCURSOR - it->tok);
  157. log_trace ("ID name: %s", *token_p);
  158. return T_QNAME;
  159. }
  160. '_:' NSTART_CHAR NAME_CHAR* {
  161. *token_p = uint8_ndup (it->tok + 2, YYCURSOR - it->tok - 2);
  162. log_trace ("BNode name: %s", *token_p);
  163. return T_BNODE_ID;
  164. }
  165. WS {
  166. log_trace ("Whitespace.");
  167. return T_WS;
  168. }
  169. '@' [a-z]+ ('-' [a-z0-9]+)* {
  170. *token_p = uint8_ndup (it->tok + 1, YYCURSOR - it->tok);
  171. log_trace ("Lang tag: %s", *token_p);
  172. return T_LANGTAG;
  173. }
  174. INTEGER {
  175. // Normalize sign.
  176. size_t offset = *it->tok == '+' ? 1 : 0;
  177. *token_p = uint8_ndup (it->tok + offset, YYCURSOR - it->tok - offset);
  178. log_trace ("Integer: %s", *token_p);
  179. return T_INTEGER;
  180. }
  181. DOUBLE {
  182. // Normalize sign.
  183. size_t offset = *it->tok == '+' ? 1 : 0;
  184. *token_p = uint8_ndup (it->tok + offset, YYCURSOR - it->tok - offset);
  185. log_trace ("Integer: %s", *token_p);
  186. return T_DOUBLE;
  187. }
  188. DECIMAL {
  189. // Normalize sign.
  190. YYCTYPE offset = *it->tok == '+' ? 1 : 0;
  191. // Normalize trailing zeros in fractional part.
  192. size_t size = YYCURSOR - it->tok - offset;
  193. if (strchr ((char *)it->tok, '.'))
  194. for (YYCTYPE *i = YYCURSOR; *i == '0'; i--) size--;
  195. *token_p = uint8_ndup (it->tok + offset, size);
  196. log_trace ("Integer: %s", *token_p);
  197. return T_DECIMAL;
  198. }
  199. '(' { return T_LPAREN; }
  200. ')' { return T_RPAREN; }
  201. '[' { return T_LBRACKET; }
  202. ']' { return T_RBRACKET; }
  203. ':' { return T_COLON; }
  204. ';' { return T_SEMICOLON; }
  205. ',' { return T_COMMA; }
  206. '.' {
  207. log_trace ("End of statement #%u.", it->stmt);
  208. it->stmt++;
  209. return T_PERIOD;
  210. }
  211. '^^' { return T_DTYPE_MARKER; }
  212. COMMENT {
  213. log_trace ("Comment: `%s`", it->tok);
  214. goto loop;
  215. }
  216. "a" / WS {
  217. log_trace ("RDF type shorthand 'a'.");
  218. return T_RDF_TYPE;
  219. }
  220. */
  221. schar:
  222. /*!re2c
  223. * {
  224. log_warn (
  225. "Invalid token @ %lu: %s (\\x%x)",
  226. YYCURSOR - it->buf - 1, it->tok, *it->tok);
  227. return -1;
  228. }
  229. $ {
  230. log_warn ("Unterminated string!");
  231. return -1;
  232. }
  233. SCHAR { goto schar; }
  234. [\x22] {
  235. *token_p = unescape_unicode (it->tok + 1, YYCURSOR - it->tok - 2);
  236. log_trace ("String: %s", *token_p);
  237. return T_STRING;
  238. }
  239. */
  240. lchar:
  241. /*!re2c
  242. $ {
  243. log_warn ("Unterminated long string!");
  244. return -1;
  245. }
  246. LCHAR { goto lchar; }
  247. [\x22]{3} {
  248. *token_p = unescape_unicode (it->tok + 3, YYCURSOR - it->tok - 6);
  249. log_trace ("Long string: %s", it->tok);
  250. return T_STRING;
  251. }
  252. * {
  253. log_warn (
  254. "Invalid token @ %lu: %s (\\x%x)",
  255. YYCURSOR - it->buf - 1, it->tok, *it->tok);
  256. return -1;
  257. }
  258. */
  259. bnode_id:
  260. /*!re2c
  261. NSTART_CHAR NAME_CHAR* {
  262. *token_p = unescape_unicode (it->tok, YYCURSOR - it->tok);
  263. log_trace ("ID name: %s", *token_p);
  264. return T_BNODE_ID;
  265. }
  266. * {
  267. log_warn (
  268. "Invalid token @ %lu: %s (\\x%x)",
  269. YYCURSOR - it->buf - 1, it->tok, *it->tok);
  270. return -1;
  271. }
  272. $ {
  273. log_trace ("End of document.");
  274. return T_EOF;
  275. }
  276. */
  277. }
  278. LSUP_rc
  279. LSUP_ttl_parse_doc (FILE *fh, LSUP_Graph **gr_p, size_t *ct, char **err_p)
  280. {
  281. *err_p = NULL;
  282. *gr_p = NULL;
  283. ParseIterator parse_it;
  284. parse_init (&parse_it, fh);
  285. void *parser = TTLParseAlloc (malloc);
  286. LSUP_rc rc;
  287. LSUP_TTLParserState *state = malloc (sizeof (*state));
  288. if (UNLIKELY (!state)) {
  289. rc = LSUP_MEM_ERR;
  290. goto finally;
  291. }
  292. state->base = NULL;
  293. state->ct = 0;
  294. state->rc = LSUP_NORESULT;
  295. state->nsm = LSUP_nsmap_new();
  296. // TODO add basic NS, critically xsd: and rdf:
  297. LSUP_Graph *gr = LSUP_graph_new (
  298. LSUP_iriref_new (NULL, NULL), LSUP_STORE_HTABLE, NULL, state->nsm, 0);
  299. if (UNLIKELY (!gr)) return LSUP_MEM_ERR;
  300. state->it = LSUP_graph_add_init (gr);
  301. if (UNLIKELY (!state->it)) {
  302. LSUP_graph_free (gr);
  303. return LSUP_MEM_ERR;
  304. }
  305. YYCTYPE *token;
  306. #ifdef DEBUG
  307. TTLParseTrace (stdout, "TTL Parser > ");
  308. #endif
  309. for (;;) {
  310. int ttype = lex (&parse_it, &token);
  311. if (ttype == -1) {
  312. char err_token[16] = {'\0'};
  313. strncpy (err_token, (const char *)parse_it.tok, 15);
  314. char *err_start = "Lexer error near token `";
  315. char err_info [64];
  316. sprintf(
  317. err_info, "[...]' at line %u, character %ld.\n",
  318. parse_it.line, parse_it.cur - parse_it.bol);
  319. size_t err_size = strlen (err_start) + 16 + strlen(err_info);
  320. char *err_str = malloc (err_size);
  321. sprintf (err_str, "%s%s%s", err_start, err_token, err_info);
  322. rc = LSUP_PARSE_ERR;
  323. *err_p = err_str;
  324. goto finally;
  325. }
  326. TTLParse (parser, ttype, token, state);
  327. if (ttype == T_EOF) break;
  328. };
  329. if (ct) *ct = state->ct;
  330. log_info ("Parsed %u triples.", state->ct);
  331. log_debug ("Graph size: %lu", LSUP_graph_size (gr));
  332. *gr_p = gr;
  333. finally:
  334. TTLParse (parser, 0, NULL, state);
  335. TTLParseFree (parser, free);
  336. rc = state->rc;
  337. LSUP_graph_add_done (state->it);
  338. free (state);
  339. if (rc < 0) LSUP_graph_free (gr);
  340. return rc;
  341. }