From f260d5dcb1fea9cfa5796f62624717dd4a29a7b4 Mon Sep 17 00:00:00 2001
From: Wesley Kerfoot <wjak56@gmail.com>
Date: Mon, 19 Aug 2013 14:58:00 -0400
Subject: [PATCH] added t tokenizer

---
 tokenize.js | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tokenize.js b/tokenize.js
index 8306c26..2b8dd3b 100755
--- a/tokenize.js
+++ b/tokenize.js
@@ -88,6 +88,18 @@ function tokenizeStr(tokstream) {
 
 }
 
+function tokenizeT(tokstream) {
+  if (tokstream.length < 4)
+    return false;
+  var next4 = tokstream.substr(0,4);
+  if (next4 === "then")
+    return ["then-exp", "then"];
+  else if (next4 === "true")
+    return ["bool", "true"];
+  else
+    return false;
+}
+
 function tokenize(tokstream) {
   var tokens = [];
 
@@ -161,6 +173,14 @@ function tokenize(tokstream) {
           tokens.push(num);
         tokstream = tokstream.substr(i);
         break;
+      case 116: // 't'
+        var result = tokenizeT(tokstream);
+        if (result) {
+          var token = result[1];
+          tokens.push(token);
+          tokstream = tokstream.substr(4); // 4 = length of either token
+          break;
+        }
       default:
         if (isDigit(tokstream[0])) {
           var result = tokenizeNum(tokstream);