Browse Source

tweaked the tokenizer

pull/21/head
Wesley Kerfoot 12 years ago
parent
commit
6729a11df4
  1. 2
      desugar.js
  2. 4
      parse.js
  3. 4
      pprint.js
  4. 3
      representation.js
  5. 14
      tokenize.js

2
desugar.js

@ -40,7 +40,7 @@ function desugar(stx) {
return stx;
case "Application":
if (stx.p)
return new typ.App(stx.func, desugar(stx.p));
return new typ.App(desugar(stx.func), desugar(stx.p));
return new typ.App(stx.func);
case "Function":
return new typ.FuncT(stx.p, desugar(stx.body));

4
parse.js

@ -339,6 +339,8 @@ function parseFull(tokenized) {
}
return ast;
}
console.log(parseFull(tokenizer.tokenize(istr))[0].func.p.p.val);
console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n"));
//console.log(tokenizer.tokenize(istr));
//module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) };

4
pprint.js

@ -36,9 +36,9 @@ function pprint(expr) {
else
return "False";
else if (expr.exprType === "Integer")
return expr.val;
return "("+expr.val+")";
else if (expr.exprType === "Float")
return expr.val;
return "("+expr.val+")";
else if (expr.exprType === "String")
return '"'+expr.val+'"';
else if (expr.exprType === "Name")

3
representation.js

@ -168,7 +168,8 @@ OPInfo = {"+" : [3, "Left"],
"$" : [1, "Left"],
">>" : [1, "Left"],
">>=" : [1, "Left"],
"<$>" : [1, "Left"]}
"<$>" : [1, "Left"],
"." : [1, "Left"]}
module.exports =
{ IntT : IntT,

14
tokenize.js

@ -121,8 +121,9 @@ function peek(tokstream, toktype, word) {
if (tokstream.length < n)
return false;
var nextN = tokstream.substr(0, n);
if (nextN == word)
if (nextN == word) {
return [toktype, word];
}
return false;
}
@ -179,7 +180,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i);
break;
case 43: // '+'
/* case 43: // '+'
if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream);
var num = result[1];
@ -189,6 +190,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i);
break;
}
*/
case 45: // '-'
var lambda = peek(tokstream, "arrow", "->");
if (lambda) {
@ -196,7 +198,12 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(2);
break;
}
if (isDigit(tokstream[1])) {
else {
tokens.push(["identifier", "-"]);
tokstream = tokstream.substr(1);
break;
}
/* if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream);
var num = result[1];
var i = result[0];
@ -205,6 +212,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i);
break;
}
*/
case 46: // '.'
if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream);

Loading…
Cancel
Save