Browse Source

more quoting stuff

master
nisstyre56 10 years ago
parent
commit
c9855f3473
  1. 4
      tokenize.c
  2. 2
      tokenize.py

4
tokenize.c

@ -272,6 +272,10 @@ match_symbol(source_t source,
return false;
}
i++;
while ((isspace(source[i]) ||
(source[i] == '\'')) && i < length) { /* consume leading whitespace and quotes */
i++;
}
while (!isspace(source[i]) && i < length) {
i++;
}

2
tokenize.py

@ -47,6 +47,6 @@ def tokenize(source):
tokenizer.pop_token(tp)
tokenizer.release_tokens(tp)
line = " '34 34"
line = " '''' a b"
xs = list(tokenize(line))
print xs