Browse Source

tweaked the tokenizer

pull/21/head
Wesley Kerfoot 12 years ago
parent
commit
6729a11df4
  1. 2
      desugar.js
  2. 4
      parse.js
  3. 4
      pprint.js
  4. 3
      representation.js
  5. 14
      tokenize.js

2
desugar.js

@ -40,7 +40,7 @@ function desugar(stx) {
return stx; return stx;
case "Application": case "Application":
if (stx.p) if (stx.p)
return new typ.App(stx.func, desugar(stx.p)); return new typ.App(desugar(stx.func), desugar(stx.p));
return new typ.App(stx.func); return new typ.App(stx.func);
case "Function": case "Function":
return new typ.FuncT(stx.p, desugar(stx.body)); return new typ.FuncT(stx.p, desugar(stx.body));

4
parse.js

@ -339,6 +339,8 @@ function parseFull(tokenized) {
} }
return ast; return ast;
} }
console.log(parseFull(tokenizer.tokenize(istr))[0].func.p.p.val); console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n"));
//console.log(tokenizer.tokenize(istr));
//module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) }; //module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) };

4
pprint.js

@ -36,9 +36,9 @@ function pprint(expr) {
else else
return "False"; return "False";
else if (expr.exprType === "Integer") else if (expr.exprType === "Integer")
return expr.val; return "("+expr.val+")";
else if (expr.exprType === "Float") else if (expr.exprType === "Float")
return expr.val; return "("+expr.val+")";
else if (expr.exprType === "String") else if (expr.exprType === "String")
return '"'+expr.val+'"'; return '"'+expr.val+'"';
else if (expr.exprType === "Name") else if (expr.exprType === "Name")

3
representation.js

@ -168,7 +168,8 @@ OPInfo = {"+" : [3, "Left"],
"$" : [1, "Left"], "$" : [1, "Left"],
">>" : [1, "Left"], ">>" : [1, "Left"],
">>=" : [1, "Left"], ">>=" : [1, "Left"],
"<$>" : [1, "Left"]} "<$>" : [1, "Left"],
"." : [1, "Left"]}
module.exports = module.exports =
{ IntT : IntT, { IntT : IntT,

14
tokenize.js

@ -121,8 +121,9 @@ function peek(tokstream, toktype, word) {
if (tokstream.length < n) if (tokstream.length < n)
return false; return false;
var nextN = tokstream.substr(0, n); var nextN = tokstream.substr(0, n);
if (nextN == word) if (nextN == word) {
return [toktype, word]; return [toktype, word];
}
return false; return false;
} }
@ -179,7 +180,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i); tokstream = tokstream.substr(i);
break; break;
case 43: // '+' /* case 43: // '+'
if (isDigit(tokstream[1])) { if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream); var result = tokenizeNum(tokstream);
var num = result[1]; var num = result[1];
@ -189,6 +190,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i); tokstream = tokstream.substr(i);
break; break;
} }
*/
case 45: // '-' case 45: // '-'
var lambda = peek(tokstream, "arrow", "->"); var lambda = peek(tokstream, "arrow", "->");
if (lambda) { if (lambda) {
@ -196,7 +198,12 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(2); tokstream = tokstream.substr(2);
break; break;
} }
if (isDigit(tokstream[1])) { else {
tokens.push(["identifier", "-"]);
tokstream = tokstream.substr(1);
break;
}
/* if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream); var result = tokenizeNum(tokstream);
var num = result[1]; var num = result[1];
var i = result[0]; var i = result[0];
@ -205,6 +212,7 @@ function tokenize(tokstream) {
tokstream = tokstream.substr(i); tokstream = tokstream.substr(i);
break; break;
} }
*/
case 46: // '.' case 46: // '.'
if (isDigit(tokstream[1])) { if (isDigit(tokstream[1])) {
var result = tokenizeNum(tokstream); var result = tokenizeNum(tokstream);

Loading…
Cancel
Save