diff --git a/tokenize.c b/tokenize.c index daa1815..9133076 100644 --- a/tokenize.c +++ b/tokenize.c @@ -272,6 +272,10 @@ match_symbol(source_t source, return false; } i++; + while ((isspace(source[i]) || + (source[i] == '\'')) && i < length) { /* consume leading whitespace and quotes */ + i++; + } while (!isspace(source[i]) && i < length) { i++; } diff --git a/tokenize.py b/tokenize.py index 049e0cb..e7612c3 100644 --- a/tokenize.py +++ b/tokenize.py @@ -47,6 +47,6 @@ def tokenize(source): tokenizer.pop_token(tp) tokenizer.release_tokens(tp) -line = " '34 34" +line = " '''' a b" xs = list(tokenize(line)) print xs