Browse Source

fixed bug, was using old index for result in t tokenizer

pull/21/head
Wesley Kerfoot 12 years ago
parent
commit
229842cc85
  1. 7
      tokenize.js

7
tokenize.js

@ -176,8 +176,7 @@ function tokenize(tokstream) {
case 116: // 't' case 116: // 't'
var result = tokenizeT(tokstream); var result = tokenizeT(tokstream);
if (result) { if (result) {
var token = result[1]; tokens.push(result);
tokens.push(token);
tokstream = tokstream.substr(4); // 4 = length of either token tokstream = tokstream.substr(4); // 4 = length of either token
break; break;
} }
@ -203,6 +202,6 @@ function tokenize(tokstream) {
var tokstream = fs.readFileSync("/dev/stdin").toString(); var tokstream = fs.readFileSync("/dev/stdin").toString();
//console.log(tokenize(tokstream)); console.log(tokenize(tokstream));
tokenize(tokstream); //tokenize(tokstream);

Loading…
Cancel
Save