fixed bug with defop parsing, wrong token type in the pattern

closure_conversion.js

@@ -152,7 +152,6 @@ function test(src) {
 }
 
 //console.log(test(pprint.pprint(parser.parse(pprint.pprint(parser.parse("if something then if a then if b then c else d else rtrrt else some_other_thing")[0]))[0])));
-console.log(pprint.pprint(parser.parse("defop 1 Right (a $# b) (a - b) def main (4 $# b $# c)")[1]));
 module.export = {
   test : test,
   closureConvert : closure_convert_all

parse.js

@@ -636,7 +636,8 @@ function parseFull(tokenized) {
 
 module.exports = { parse : function(str) {
                               return parseFull(tokenizer.tokenize(str));
-                            }
+                            },
+                  tokenize : tokenizer.tokenize
                  };
-/*var istr = fs.readFileSync('/dev/stdin').toString();
-console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint));*/
+var istr = fs.readFileSync('/dev/stdin').toString();
+console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint));

tokenize.js

@@ -25,7 +25,7 @@ function isWhitespace(c) {
 
   var code = c.charCodeAt();
   if (isNaN(code)) {
-    return false;
+    return true;
   }
   return (code === 9 ||
           code === 32 ||
@@ -388,12 +388,12 @@ function tokenizeHelp(input, matchop, strip_whitespace) {
   }
 }
 
-var defop_pattern = ["defop", "integer", "identifier",
+var defop_pattern = ["defop", "integer", "constructor",
                      "left_paren", "identifier",
                      "identifier", "identifier", "right_paren"];
 
 function tokenizeFull(input) {
-  var matchop = $.opMatch(operators);
+  var matchop;
   var initialPass = tokenizeHelp(input, _.constant(false), true).reverse();
   for (var i = 0; i < initialPass.length; i++) {
     if (initialPass.slice(i, i+8).map(_.first).every(