From c9855f34732bf1114b50b757962580ccf6d6d078 Mon Sep 17 00:00:00 2001
From: nisstyre56 <wjak56@gmail.com>
Date: Wed, 9 Jul 2014 02:46:18 -0400
Subject: [PATCH] more quoting stuff

---
 tokenize.c  | 4 ++++
 tokenize.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tokenize.c b/tokenize.c
index daa1815..9133076 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -272,6 +272,10 @@ match_symbol(source_t source,
     return false;
   }
   i++;
+  while ((isspace(source[i]) ||
+         (source[i] == '\'')) && i < length) { /* consume leading whitespace and quotes */
+    i++;
+  }
   while (!isspace(source[i]) && i < length) {
     i++;
   }
diff --git a/tokenize.py b/tokenize.py
index 049e0cb..e7612c3 100644
--- a/tokenize.py
+++ b/tokenize.py
@@ -47,6 +47,6 @@ def tokenize(source):
         tokenizer.pop_token(tp)
     tokenizer.release_tokens(tp)
 
-line = " '34 34"
+line = " '''' a b"
 xs = list(tokenize(line))
 print xs