@ -272,6 +272,10 @@ match_symbol(source_t source,
return false;
}
i++;
while ((isspace(source[i]) ||
(source[i] == '\'')) && i < length) { /* consume leading whitespace and quotes */
while (!isspace(source[i]) && i < length) {
@ -47,6 +47,6 @@ def tokenize(source):
tokenizer.pop_token(tp)
tokenizer.release_tokens(tp)
line = " '34 34"
line = " '''' a b"
xs = list(tokenize(line))
print xs