From c566aeb6749fd6f764a3f1c23d0d18e917f44366 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Wed, 11 Dec 2013 21:25:26 -0500 Subject: [PATCH 01/43] expanded test program --- test.jl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test.jl b/test.jl index bb84621..eb9dca5 100644 --- a/test.jl +++ b/test.jl @@ -27,9 +27,25 @@ def getFile def (testUnary n) ((-n) + n) +def (splitHelp acc xs ys) + if (null xs) + then ((reverse acc), ys) + else if (null (tail xs)) + then ((reverse acc), ys) + else + (splitHelp ((head ys) : acc) + (tail (tail xs)) + (tail ys)) + +def (splitxs xs) + (splitHelp [] xs xs) + def main ((print (testUnary 6)) >> if False then undefined - else getFile) + else (getFile >>= print) + >> + (print + (splitxs "abcdefghijk"))) -- 2.30.2 From 2458430357337140e2f427153e5fa68593bde575 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Wed, 11 Dec 2013 21:25:43 -0500 Subject: [PATCH 02/43] added tuple operator --- parse.js | 2 +- representation.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parse.js b/parse.js index abeaff1..612023e 100755 --- a/parse.js +++ b/parse.js @@ -342,6 +342,6 @@ function parseFull(tokenized) { console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); //console.log(tokenizer.tokenize(istr)); -//console.log(parseFull(tokenizer.tokenize(istr))); +//console.log(parseFull(tokenizer.tokenize(istr))[8].val.body); //module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) }; diff --git a/representation.js b/representation.js index 0676bc7..0afe9d5 100644 --- a/representation.js +++ b/representation.js @@ -175,7 +175,8 @@ OPInfo = {"+" : [3, "Left"], ">>" : [1, "Left"], ">>=" : [1, "Left"], "<$>" : [1, "Left"], - "." : [1, "Left"]} + "." : [1, "Left"], + "," : [1, "Left"]} module.exports = { IntT : IntT, -- 2.30.2 From fc6c61994db5fcf416e1e5f46d74ee5399005756 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Thu, 12 Dec 2013 01:51:59 -0500 Subject: [PATCH 03/43] almost done adding let forms to the language, made sure to throw exceptions in most places --- parse.js | 72 ++++++++++++++++------------------------------- representation.js | 18 +++++++++++- tokenize.js | 27 ++++++++++++------ tools.js | 11 +++++++- typecheck.js | 0 5 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 typecheck.js diff --git a/parse.js b/parse.js index 612023e..709bde9 100755 --- a/parse.js +++ b/parse.js @@ -43,12 +43,9 @@ function parseMany(exprType, valid, tokens) { if (valid(fst(tokens)[0])) { parsed = parse(tokens); - //console.log(parsed.exprType); } else { - console.log("Error: unexpected token "+fst(tokens)); - console.log("in parseMany," + ", " + tokens); - return; + throw "Error: unexpected token "+fst(tokens)[0]+" in parseMany"; } results.push(parsed); @@ -61,13 +58,14 @@ function parseMany(exprType, valid, tokens) { results.push(parse(tokens)); if (!exprType(fst(results).exprType)) break; - //console.log(results); current = fst(tokens)[0] if (tokens.length <= 1) break; } } //do the same validity check as before and in the loop + if (!fst(tokens)) + throw "Error: unexpected end of source"; if (valid(fst(tokens)[0])) results.push(parse(tokens)); return results; @@ -80,8 +78,7 @@ function parseMany(exprType, valid, tokens) { function parseBetween(exprType, between, tokens) { var first = parse(tokens); if (!exprType(first)) { - console.log("Error, unexpected token:"+fst(tokens)[0]); - return; + throw "Error, unexpected token:"+fst(tokens)[0]; } var items = [first]; var parsed; @@ -108,8 +105,7 @@ function parseList(tokens) { var xs = parseBetween(function (x) { return true; }, "comma", tokens); } if (fst(tokens)[0] !== "right_square") { - console.log("Error, list must be terminated by ]"); - return undefined; + throw "Error, list must be terminated by ]"; } tokens.pop(); return new typ.ListT(xs); @@ -119,8 +115,7 @@ function parseList(tokens) { function parseDefFunction(tokens) { var fname = parse(tokens); if (!fname.exprType === "identifier") { - console.log("Error, expected an identifier in function definition"); - return undefined; + throw "Error, expected an identifier in function definition"; } if (fst(tokens)[0] === "right_paren") { var parameters = []; @@ -129,8 +124,7 @@ function parseDefFunction(tokens) { var parameters = parseMany(validName, validFormPar, tokens); } if ((fst(tokens)[0]) !== "right_paren") { - console.log("Error, formal parameters must be followed by )"); - return undefined; + throw "Error, formal parameters must be followed by )"; } tokens.pop(); var body = parse(tokens); @@ -146,14 +140,12 @@ function parseDef(tokens) { return parseDefFunction(tokens); } if (notFollowedBy(tokens, ["identifier"])) { - console.log("Error: def must be followed by identifier, not "+fst(tokens)[0]); - return undefined; + throw "Error: def must be followed by identifier, not "+fst(tokens)[0]; } else { var identifier = parse(tokens); if (!notFollowedBy(tokens, ["def", "comma", "arrow", "right_brace", "right_square"])) { - console.log("Error: def " + identifier.val + " must not be followed by " + fst(tokens)[0]); - return; + throw "Error: def " + identifier.val + " must not be followed by " + fst(tokens)[0]; } return new typ.Def(identifier, parse(tokens)); } @@ -162,13 +154,12 @@ function parseDef(tokens) { function parseIf(tokens) { if (!notFollowedBy(tokens, ["def","comma","lambda"])) { - console.log("Error: ``if'' cannot be followed by "+fst(tokens)[0]) - return; + throw "Error: ``if'' cannot be followed by "+fst(tokens)[0]; } else { var ifC = parse(tokens); if (!fst(tokens) || fst(tokens)[0] !== "thenexp") - console.log("Error: if must be folowed by exp, not "+snd(tokens)[0]); + throw "Error: if must be folowed by exp, not "+snd(tokens)[0]; else { tokens.pop(); var thenC = parse(tokens); @@ -194,8 +185,7 @@ function parseLambda(tokens) { var parameters = parseMany(validName,validFormPar, tokens); if (fst(tokens)[0] !== "arrow") { - console.log("Error: arrow must follow parameters in lambda, not "+fst(tokens)[0]) - return; + throw "Error: arrow must follow parameters in lambda, not "+fst(tokens)[0]; } tokens.pop() var body = parse(tokens); @@ -207,31 +197,19 @@ var validArgument = tool.compose(tool.not, makeChecker(invalidArguments)); var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); var validOperator = makeChecker(["identifier"]); -function checkParse(p) { - if (p === undefined) { - console.log("Quitting, could not finish parsing!"); - process.exit(code=1); - } - else - return p; -} - //Parses function application (either infix or prefix) function computeApp(tokens) { var lhs = parse(tokens); - //console.log(lhs); if (fst(tokens)) var next = fst(tokens); else { - console.log("Unexpected end of source"); - process.exit(code=1); + throw "Error: Unexpected end of source"; } if (typ.OPInfo[next[1]]) { //it's an infix expression var result = parseInfix(tokens, 1, lhs); if (fst(tokens)[0] !== "right_paren") { - console.log("Error: mismatched parentheses"); - process.exit(code=1); + throw "Error: mismatched parentheses"; } else { //return the result @@ -243,10 +221,8 @@ function computeApp(tokens) { //it's a prefix application var parameters = parseMany(validArgTypes, validArgument, tokens); - //console.log(parameters); if (fst(tokens)[0] !== "right_paren") { - console.log("Error: mismatched parentheses"); - process.exit(code=1); + throw "Error: mismatched parentheses"; } else { //return the result @@ -267,8 +243,7 @@ function parseInfix(tokens, minPrec, lhs) { while (true) { var cur = fst(tokens); if (!cur) { - console.log("Unexpected end of source") - process.exit(code=1); + throw "Error: Unexpected end of source"; } var opinfo = typ.OPInfo[cur[1]]; @@ -291,7 +266,6 @@ function parse(tokens) { if (fst(tokens)) var toktype = fst(tokens)[0]; else { - //console.log("Unexpected end of source") process.exit(code=1); } var token = fst(tokens)[1]; @@ -301,7 +275,7 @@ function parse(tokens) { else if (toktype === "left_square") return parseList(tokens); else if (toktype === "lambda") - return checkParse(parseLambda(tokens)); + return parseLambda(tokens); else if (toktype === "integer") return new typ.IntT(token); else if (toktype === "float") @@ -311,22 +285,24 @@ function parse(tokens) { else if (toktype === "truelit" || toktype === "falselit") return new typ.BoolT(token); else if (toktype === "def") - return checkParse(parseDef(tokens)); + return parseDef(tokens); else if (toktype === "ifexp") - return checkParse(parseIf(tokens)); + return parseIf(tokens); else if (toktype === "left_paren") { if (fst(tokens)[0] === "lambda") { tokens.pop(); - var parsed = checkParse(parseLambda(tokens)); + var parsed = parseLambda(tokens); tokens.pop(); return parsed; } else return computeApp(tokens); } + /*else if (toktype === "let") { + return parseLet(tokens); + }*/ else { - console.log("Unexpected token: " + toktype); - process.exit(code=1); + throw "Error: Unexpected token: " + toktype; } } diff --git a/representation.js b/representation.js index 0afe9d5..7151338 100644 --- a/representation.js +++ b/representation.js @@ -1,3 +1,5 @@ +var tool = require("./tools.js"); + var Expression = { display : function() { @@ -18,6 +20,17 @@ var Expression = { } }; +function LetExp(pairs) { + if (!pairs.every(function(x) { + return x.exprType === "Name"; + })) { + throw "let can only be used to bind things to names"; + } + this.exprType = "Let"; + this.val = tool.dict(pairs); + return this; +} + function UnaryOp(op, v) { this.exprType = "Unary"; this.val = v; @@ -170,6 +183,8 @@ OPInfo = {"+" : [3, "Left"], ">=" : [2, "Left"], "<" : [2, "Left"], "<=" : [2, "Left"], + "&&" : [2, "Left"], + "||" : [2, "Left"], ":" : [2, "Left"], "$" : [1, "Left"], ">>" : [1, "Left"], @@ -194,4 +209,5 @@ module.exports = If : If, DefFunc : DefFunc, UnaryOp : UnaryOp, - Nil : Nil } + Nil : Nil, + LetExp : LetExp} diff --git a/tokenize.js b/tokenize.js index 28e4e01..28f96e9 100755 --- a/tokenize.js +++ b/tokenize.js @@ -96,8 +96,7 @@ function tokenizeStr(tokstream) { tokstream = tokstream.substr(1); n++; if (tokstream.length < 1) { - console.log("Error: missing quotation mark"); - process.exit(code=1); + throw "Error: missing quotation mark"; } } n++; @@ -232,9 +231,15 @@ function tokenize(tokstream) { } case 105: // 'i' - var result = peek(tokstream, "ifexp", "if"); - if (result) { - tokens.push(result); + var ifexp = peek(tokstream, "ifexp", "if"); + if (ifexp) { + tokens.push(ifexp); + tokstream = tokstream.substr(2); + break; + } + var inkeyword = peek(tokstream, "in", "in"); + if (inkeyword) { + tokens.push(inkeyword); tokstream = tokstream.substr(2); break; } @@ -261,12 +266,18 @@ function tokenize(tokstream) { break; } case 108: // l - var result = peek(tokstream, "lambda", "lambda"); - if (result) { - tokens.push(result); + var lambda = peek(tokstream, "lambda", "lambda"); + if (lambda) { + tokens.push(lambda); tokstream = tokstream.substr(6); break; } + var letexp = peek(tokstream, "let", "let"); + if (letexp) { + tokens.push(letexp); + tokstream = tokstream.substr(3); + break; + } default: if (isDigit(tokstream[0])) { diff --git a/tools.js b/tools.js index 4da833f..564fec2 100644 --- a/tools.js +++ b/tools.js @@ -114,6 +114,14 @@ function find(f, haystack) { return false; } +function dict(pairs) { + var o = new Object(); + pairs.map(function(p) { + o[p[0]] = p[1]; + }); + return o; +} + /* * Problem: @@ -154,4 +162,5 @@ module.exports = {compose : compose, maxBy : maxBy, len : len, groupOps : groupOps, - opMatch : operatorMatch} + opMatch : operatorMatch, + dict: dict} diff --git a/typecheck.js b/typecheck.js new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 37c79418d4aca6795f627acb5caa9d3583f12780 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Thu, 12 Dec 2013 15:01:37 -0500 Subject: [PATCH 04/43] started to add proper errors --- errors.js | 35 +++++++++++++++++++++++++++++++++++ parse.js | 23 ++++++++++++++++------- representation.js | 2 ++ tokenize.js | 12 +++++++++--- 4 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 errors.js diff --git a/errors.js b/errors.js new file mode 100644 index 0000000..1a9b08c --- /dev/null +++ b/errors.js @@ -0,0 +1,35 @@ +/* + * This file defines common error objects + * for reporting on syntax errors, type errors, + * and perhaps runtime exceptions although I have + * not thought about how that will work much + */ + +var JLException = { + stxerror : + function () { + console.log("There was an error\n", + "Line #",this.linenum,"\n", + "Character #", this.charnum,"\n", + this.errormessage); + }, + type_error : + function () { + return; + } +} + +function SyntaxError(linenum, charnum, message) { + this.linenum = linenum; + this.charnum = charnum; + this.errormessage = message; + return this; +} + +function TypeError(linenum, charnum, token, message) { + this.linenum = linenum; + this.charnum = charnum; + this.errormessage = message; + this.token = token; + return this; +} diff --git a/parse.js b/parse.js index 709bde9..6b10954 100755 --- a/parse.js +++ b/parse.js @@ -192,6 +192,9 @@ function parseLambda(tokens) { return new typ.FuncT(parameters, body); } +//function parseLet(tokens) { + + var invalidArguments = ["def", "comma", "right_paren", "right_square", "right_brace", "left_brace", "right_brace"]; var validArgument = tool.compose(tool.not, makeChecker(invalidArguments)); var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); @@ -298,9 +301,9 @@ function parse(tokens) { else return computeApp(tokens); } - /*else if (toktype === "let") { - return parseLet(tokens); - }*/ +// else if (toktype === "let") { +// return parseLet(tokens); +// } else { throw "Error: Unexpected token: " + toktype; } @@ -309,11 +312,17 @@ function parse(tokens) { var istr = fs.readFileSync('/dev/stdin').toString(); function parseFull(tokenized) { var ast = new Array(); - while (tokenized.length > 0) { - var parsed = desugarer.desugar(parse(tokenized)); - ast.push(parsed); + try { + while (tokenized.length > 0) { + var parsed = desugarer.desugar(parse(tokenized)); + ast.push(parsed); + } + return ast; + } catch (e) { + print("An exception occured, could not finish parsing"); + print(e); + process.exit(1); } - return ast; } console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); diff --git a/representation.js b/representation.js index 7151338..0521183 100644 --- a/representation.js +++ b/representation.js @@ -30,6 +30,7 @@ function LetExp(pairs) { this.val = tool.dict(pairs); return this; } +LetExp.prototype = Expression; function UnaryOp(op, v) { this.exprType = "Unary"; @@ -37,6 +38,7 @@ function UnaryOp(op, v) { this.op = op; return this; } +UnaryOp.prototype = Expression; function IntT(v) { this.exprType = "Integer"; diff --git a/tokenize.js b/tokenize.js index 28f96e9..1a7c04f 100755 --- a/tokenize.js +++ b/tokenize.js @@ -308,9 +308,15 @@ function tokenize(tokstream) { } function tokenizeFull(input) { - return tokenize(input).reverse().filter(function(x) { - return x[0] !== "whitespace"; - }); + try { + return tokenize(input).reverse().filter(function(x) { + return x[0] !== "whitespace"; + }); + } catch (e) { + console.log("An error occured during tokenization"); + console.log(e); + process.exit(1); + } } -- 2.30.2 From 246faaad2aad6d4ef98ff4c9bc354e06854e3ae1 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Sun, 22 Dec 2013 15:50:53 -0500 Subject: [PATCH 05/43] added source character and line numbers to tokenizer --- tokenize.js | 110 ++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 59 deletions(-) diff --git a/tokenize.js b/tokenize.js index 1a7c04f..2587b2d 100755 --- a/tokenize.js +++ b/tokenize.js @@ -26,20 +26,21 @@ function isIdentifier(a) { return code !== 41 && code !== 40 && code && 125 && code && 123 && code !== 93 && code !== 91 && code !== 44; } -function tokenizeNum(tokstream) { +function tokenizeNum(tokstream, charnum, linenum) { var number = []; var code = tokstream[0].charCodeAt(); var isFloat = false; var n = 0; // + - - if (code === 43 || code === 45) { + if (code === 43 || code === 45) { // + or - number.push(tokstream[0]); tokstream = tokstream.substr(1); n++; } - else if (code === 46) { + else if (code === 46) { // . tokstream = tokstream.substr(1); n++; + charnum++; number.push('0'); number.push('.'); isFloat = true; @@ -48,24 +49,27 @@ function tokenizeNum(tokstream) { while (isDigit(tokstream[0]) && tokstream.length !== 0) { number.push(tokstream[0]); tokstream = tokstream.substr(1); + charnum++; n++; } if (tokstream[0] === '.' && isDigit(tokstream[1])) { number.push('.'); number.push(tokstream[1]); tokstream = tokstream.substr(2); + charnum++; charnum++; n++; n++; while (isDigit(tokstream[0]) && tokstream.length !== 0) { number.push(tokstream[0]); tokstream = tokstream.substr(1); n++; + charnum++; } - return [n, ["float", parseFloat(number.join(''), 10)]]; + return [n, ["float", parseFloat(number.join(''), 10), charnum, linenum]]; } if (!isFloat) - return [n, ["integer", parseInt(number.join(''), 10)]]; + return [n, ["integer", parseInt(number.join(''), 10), charnum, linenum]]; else - return [n, ["float", parseFloat(number.join(''), 10)]]; + return [n, ["float", parseFloat(number.join(''), 10), charnum, linenum]]; } /* Split up the tokenized identifier if an operator appears in it @@ -74,20 +78,21 @@ function tokenizeNum(tokstream) { * Everything after the operator goes back on to the token stream */ -function tokenizeIdent(tokstream) { +function tokenizeIdent(tokstream, charnum, linenum) { var identifier = []; var n = 0; while ((!isWhitespace(tokstream[0])) && isIdentifier(tokstream[0]) && !matchop(tokstream)) { identifier.push(tokstream[0]); tokstream = tokstream.substr(1); n++; + charnum++; } identifier = identifier.join(''); - return [[n, ["identifier", identifier]]]; + return [[n, ["identifier", identifier, charnum, linenum]]]; } -function tokenizeStr(tokstream) { +function tokenizeStr(tokstream, charnum, linenum) { var stringlit = []; var n = 1; tokstream = tokstream.substr(1); @@ -95,16 +100,17 @@ function tokenizeStr(tokstream) { stringlit.push(tokstream[0]); tokstream = tokstream.substr(1); n++; + charnum++; if (tokstream.length < 1) { throw "Error: missing quotation mark"; } } n++; - return [n, ["stringlit", stringlit.join('')]]; + return [n, ["stringlit", stringlit.join(''), charnum, linenum]]; } -function tokenizeT(tokstream) { +function tokenizeT(tokstreami, charnum, linenum) { if (tokstream.length < 4) return false; var next4 = tokstream.substr(0,4); @@ -115,7 +121,7 @@ function tokenizeT(tokstream) { return false; } -function peek(tokstream, toktype, word) { +function peek(tokstream, toktype, word, charnum, linenum) { var n = word.length; if (tokstream.length < n) return false; @@ -128,68 +134,70 @@ function peek(tokstream, toktype, word) { function tokenize(tokstream) { var tokens = []; + var charnum = 1; + var linenum = 1; while (tokstream) { switch (tokstream[0].charCodeAt()) { case 9: // '\t' - tokens.push(["whitespace", '\t']); + charnum++; + tokens.push(["whitespace", '\t', charnum, linenum]); tokstream = tokstream.substr(1); break; case 32: // ' ' - tokens.push(["whitespace", ' ']); + charnum++; + tokens.push(["whitespace", ' ', charnum, linenum]); tokstream = tokstream.substr(1); break; case 10: // '\n' - tokens.push(["whitespace", '\n']); + linenum++; + charnum = 1; + tokens.push(["whitespace", '\n', charnum, linenum]); tokstream = tokstream.substr(1); break; case 44: // ',' - tokens.push(["comma", ","]); + charnum++; + tokens.push(["comma", ",", charnum, linenum]); tokstream = tokstream.substr(1); break; case 40: // '(' - tokens.push(["left_paren", '(']); + charnum++; + tokens.push(["left_paren", '(', charnum, linenum]); tokstream = tokstream.substr(1); break; case 41: // ')' - tokens.push(["right_paren", ')']); + charnum++; + tokens.push(["right_paren", ')', charnum, linenum]); tokstream = tokstream.substr(1); break; case 123: // '{' - tokens.push(["left_brace", '{']); + charnum++; + tokens.push(["left_brace", '{', charnum, linenum]); tokstream = tokstream.substr(1); break; case 125: // '}' - tokens.push(["right_brace", '}']); + charnum++; + tokens.push(["right_brace", '}', charnum, linenum]); tokstream = tokstream.substr(1); break; case 91: // '[' - tokens.push(["left_square", '[']); + charnum++; + tokens.push(["left_square", '[', charnum, linenum]); tokstream = tokstream.substr(1); break; case 93: // ']' - tokens.push(["right_square", ']']); + charnum++; + tokens.push(["right_square", ']', charnum, linenum]); tokstream = tokstream.substr(1); break; case 34: // '"' - var result = tokenizeStr(tokstream); + var result = tokenizeStr(tokstream, charnum, linenum); var str = result[1]; var i = result[0]; tokens.push(str); tokstream = tokstream.substr(i); break; -/* case 43: // '+' - if (isDigit(tokstream[1])) { - var result = tokenizeNum(tokstream); - var num = result[1]; - var i = result[0]; - if (num[1] !== NaN) - tokens.push(num); - tokstream = tokstream.substr(i); - break; - } -*/ case 45: // '-' var lambda = peek(tokstream, "arrow", "->"); if (lambda) { @@ -198,23 +206,15 @@ function tokenize(tokstream) { break; } else { - tokens.push(["identifier", "-"]); + tokens.push(["identifier", "-", charnum, linenum]); + charnum++; tokstream = tokstream.substr(1); break; } -/* if (isDigit(tokstream[1])) { - var result = tokenizeNum(tokstream); - var num = result[1]; - var i = result[0]; - if (num[1] !== NaN) - tokens.push(num); - tokstream = tokstream.substr(i); - break; - } -*/ + case 46: // '.' if (isDigit(tokstream[1])) { - var result = tokenizeNum(tokstream); + var result = tokenizeNum(tokstream, charnum, linenum); var num = result[1]; var i = result[0]; if (num[1] !== NaN) @@ -281,7 +281,7 @@ function tokenize(tokstream) { default: if (isDigit(tokstream[0])) { - var result = tokenizeNum(tokstream); + var result = tokenizeNum(tokstream, charnum, linenum); var num = result[1]; var i = result[0]; if (num[1] !== NaN) @@ -292,12 +292,14 @@ function tokenize(tokstream) { var op = matchop(tokstream); if (op) { var l = op.length; + charnum = charnum + l; tokstream = tokstream.substr(l); - tokens.push(["identifier", op]); + tokens.push(["identifier", op, charnum, linenum]); } else { - var result = tokenizeIdent(tokstream); + var result = tokenizeIdent(tokstream, charnum, linenum); result.map(function(x) { + charnum++; tokens.push(x[1]); tokstream = tokstream.substr(x[0]); }); @@ -319,14 +321,4 @@ function tokenizeFull(input) { } } - module.exports = {tokenize : tokenizeFull}; - -//var tokstream = fs.readFileSync("/dev/stdin").toString(); -//console.log(tokenize(tokstream)); -//console.log(tools.buildTrie('', operators)[1][6]); -//console.log(isIdentifier(')')); -//console.log(tools.maxBy(tools.len, operators.filter(function (x) { return "#".indexOf(x) != -1;}))); -//console.log(tokenizeIdent("abc%%3")); - - -- 2.30.2 From 3103f2168cb798fe44294c33f32d58d0ea4b76af Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Sun, 22 Dec 2013 15:58:30 -0500 Subject: [PATCH 06/43] reverted bug --- tokenize.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tokenize.js b/tokenize.js index 2587b2d..bc9e045 100755 --- a/tokenize.js +++ b/tokenize.js @@ -32,6 +32,7 @@ function tokenizeNum(tokstream, charnum, linenum) { var isFloat = false; var n = 0; // + - + // might want to remove this since it probably won't ever get run? if (code === 43 || code === 45) { // + or - number.push(tokstream[0]); tokstream = tokstream.substr(1); -- 2.30.2 From 129d33939653fc709164ba26018b6fcc5ad750fb Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Sun, 22 Dec 2013 16:41:10 -0500 Subject: [PATCH 07/43] more error stuff --- errors.js | 15 +++++++++++++-- parse.js | 14 +++++++------- tokenize.js | 6 +++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/errors.js b/errors.js index 1a9b08c..e02eef9 100644 --- a/errors.js +++ b/errors.js @@ -19,17 +19,28 @@ var JLException = { } } -function SyntaxError(linenum, charnum, message) { +function JSyntaxError(linenum, charnum, message) { this.linenum = linenum; this.charnum = charnum; this.errormessage = message; + this.stxerror = function() { + console.log("Syntax Error\n", + "Line #", this.linenum,"\n", + "Character #", this.charnum, "\n", + this.errormessage); + }; return this; } -function TypeError(linenum, charnum, token, message) { +function JTypeError(linenum, charnum, token, message) { this.linenum = linenum; this.charnum = charnum; this.errormessage = message; this.token = token; return this; } +TypeError.prototype = JLException; + +module.exports = + {JSyntaxError : JSyntaxError, + JTypeError : JTypeError}; diff --git a/parse.js b/parse.js index 6b10954..7b2f1b1 100755 --- a/parse.js +++ b/parse.js @@ -6,6 +6,7 @@ var tool = require("./tools.js"); var tokenizer = require("./tokenize.js"); var desugarer = require("./desugar.js"); var pprint = require("./pprint.js"); +var error = require("./errors.js"); var print = console.log; @@ -45,7 +46,7 @@ function parseMany(exprType, valid, tokens) { parsed = parse(tokens); } else { - throw "Error: unexpected token "+fst(tokens)[0]+" in parseMany"; + throw error.JSyntaxError(fst(tokens)[2], fst(tokens)[3], "Error: unexpected token "+fst(tokens)[0]+" in parseMany"); } results.push(parsed); @@ -105,7 +106,7 @@ function parseList(tokens) { var xs = parseBetween(function (x) { return true; }, "comma", tokens); } if (fst(tokens)[0] !== "right_square") { - throw "Error, list must be terminated by ]"; + throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Error, list must be terminated by ]"); } tokens.pop(); return new typ.ListT(xs); @@ -114,8 +115,8 @@ function parseList(tokens) { function parseDefFunction(tokens) { var fname = parse(tokens); - if (!fname.exprType === "identifier") { - throw "Error, expected an identifier in function definition"; + if (fname.exprType != "Name") { + throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Error, expected an identifier in function definition"); } if (fst(tokens)[0] === "right_paren") { var parameters = []; @@ -319,9 +320,8 @@ function parseFull(tokenized) { } return ast; } catch (e) { - print("An exception occured, could not finish parsing"); - print(e); - process.exit(1); + e.stxerror(); + process.exit(1); } } console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); diff --git a/tokenize.js b/tokenize.js index bc9e045..8c42e77 100755 --- a/tokenize.js +++ b/tokenize.js @@ -2,6 +2,7 @@ var rep = require("./representation.js"); var tools = require("./tools.js"); +var error = require("./errors.js"); var operators = Object.keys(rep.OPInfo); var matchop = tools.opMatch(operators); @@ -103,7 +104,7 @@ function tokenizeStr(tokstream, charnum, linenum) { n++; charnum++; if (tokstream.length < 1) { - throw "Error: missing quotation mark"; + throw error.JSyntaxError(linenum, charnum, "Error: missing quotation mark"); } } n++; @@ -316,8 +317,7 @@ function tokenizeFull(input) { return x[0] !== "whitespace"; }); } catch (e) { - console.log("An error occured during tokenization"); - console.log(e); + console.log(e.stxerror()); process.exit(1); } } -- 2.30.2 From 31a5481d7c5a2238133d10856acfa7e6b48c0d8e Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Mon, 23 Dec 2013 17:39:31 -0500 Subject: [PATCH 08/43] changed error messages --- parse.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/parse.js b/parse.js index 7b2f1b1..680520b 100755 --- a/parse.js +++ b/parse.js @@ -116,7 +116,7 @@ function parseList(tokens) { function parseDefFunction(tokens) { var fname = parse(tokens); if (fname.exprType != "Name") { - throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Error, expected an identifier in function definition"); + throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Expected an identifier in function definition"); } if (fst(tokens)[0] === "right_paren") { var parameters = []; @@ -125,7 +125,7 @@ function parseDefFunction(tokens) { var parameters = parseMany(validName, validFormPar, tokens); } if ((fst(tokens)[0]) !== "right_paren") { - throw "Error, formal parameters must be followed by )"; + throw "Formal parameters must be followed by )"; } tokens.pop(); var body = parse(tokens); @@ -141,12 +141,12 @@ function parseDef(tokens) { return parseDefFunction(tokens); } if (notFollowedBy(tokens, ["identifier"])) { - throw "Error: def must be followed by identifier, not "+fst(tokens)[0]; + throw "def must be followed by identifier, not "+fst(tokens)[0]; } else { var identifier = parse(tokens); if (!notFollowedBy(tokens, ["def", "comma", "arrow", "right_brace", "right_square"])) { - throw "Error: def " + identifier.val + " must not be followed by " + fst(tokens)[0]; + throw "def " + identifier.val + " must not be followed by " + fst(tokens)[0]; } return new typ.Def(identifier, parse(tokens)); } @@ -155,12 +155,12 @@ function parseDef(tokens) { function parseIf(tokens) { if (!notFollowedBy(tokens, ["def","comma","lambda"])) { - throw "Error: ``if'' cannot be followed by "+fst(tokens)[0]; + throw "``if'' cannot be followed by "+fst(tokens)[0]; } else { var ifC = parse(tokens); if (!fst(tokens) || fst(tokens)[0] !== "thenexp") - throw "Error: if must be folowed by exp, not "+snd(tokens)[0]; + throw "if must be folowed by exp, not "+snd(tokens)[0]; else { tokens.pop(); var thenC = parse(tokens); @@ -186,7 +186,7 @@ function parseLambda(tokens) { var parameters = parseMany(validName,validFormPar, tokens); if (fst(tokens)[0] !== "arrow") { - throw "Error: arrow must follow parameters in lambda, not "+fst(tokens)[0]; + throw "arrow must follow parameters in lambda, not "+fst(tokens)[0]; } tokens.pop() var body = parse(tokens); @@ -207,13 +207,13 @@ function computeApp(tokens) { if (fst(tokens)) var next = fst(tokens); else { - throw "Error: Unexpected end of source"; + throw "Unexpected end of source"; } if (typ.OPInfo[next[1]]) { //it's an infix expression var result = parseInfix(tokens, 1, lhs); if (fst(tokens)[0] !== "right_paren") { - throw "Error: mismatched parentheses"; + throw "mismatched parentheses"; } else { //return the result @@ -226,7 +226,7 @@ function computeApp(tokens) { var parameters = parseMany(validArgTypes, validArgument, tokens); if (fst(tokens)[0] !== "right_paren") { - throw "Error: mismatched parentheses"; + throw "mismatched parentheses"; } else { //return the result @@ -247,7 +247,7 @@ function parseInfix(tokens, minPrec, lhs) { while (true) { var cur = fst(tokens); if (!cur) { - throw "Error: Unexpected end of source"; + throw "Unexpected end of source"; } var opinfo = typ.OPInfo[cur[1]]; @@ -306,7 +306,7 @@ function parse(tokens) { // return parseLet(tokens); // } else { - throw "Error: Unexpected token: " + toktype; + throw "Unexpected token: " + toktype; } } -- 2.30.2 From 6629ec62b83171855b1a54ee4dd9d5eb9905ecd4 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Mon, 23 Dec 2013 17:40:30 -0500 Subject: [PATCH 09/43] changed error messages --- parse.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parse.js b/parse.js index 680520b..9d52dd1 100755 --- a/parse.js +++ b/parse.js @@ -46,7 +46,7 @@ function parseMany(exprType, valid, tokens) { parsed = parse(tokens); } else { - throw error.JSyntaxError(fst(tokens)[2], fst(tokens)[3], "Error: unexpected token "+fst(tokens)[0]+" in parseMany"); + throw error.JSyntaxError(fst(tokens)[2], fst(tokens)[3], "unexpected token "+fst(tokens)[0]+" in parseMany"); } results.push(parsed); @@ -66,7 +66,7 @@ function parseMany(exprType, valid, tokens) { } //do the same validity check as before and in the loop if (!fst(tokens)) - throw "Error: unexpected end of source"; + throw "unexpected end of source"; if (valid(fst(tokens)[0])) results.push(parse(tokens)); return results; @@ -79,7 +79,7 @@ function parseMany(exprType, valid, tokens) { function parseBetween(exprType, between, tokens) { var first = parse(tokens); if (!exprType(first)) { - throw "Error, unexpected token:"+fst(tokens)[0]; + throw "unexpected token:"+fst(tokens)[0]; } var items = [first]; var parsed; @@ -106,7 +106,7 @@ function parseList(tokens) { var xs = parseBetween(function (x) { return true; }, "comma", tokens); } if (fst(tokens)[0] !== "right_square") { - throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Error, list must be terminated by ]"); + throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "list must be terminated by ]"); } tokens.pop(); return new typ.ListT(xs); -- 2.30.2 From 6eeebcb23c2670e69ad9dcf24fa8718f43e4d675 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Mon, 23 Dec 2013 17:47:23 -0500 Subject: [PATCH 10/43] added another new syntax error --- parse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parse.js b/parse.js index 9d52dd1..38d1deb 100755 --- a/parse.js +++ b/parse.js @@ -125,7 +125,7 @@ function parseDefFunction(tokens) { var parameters = parseMany(validName, validFormPar, tokens); } if ((fst(tokens)[0]) !== "right_paren") { - throw "Formal parameters must be followed by )"; + throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2],"Formal parameters must be followed by )"); } tokens.pop(); var body = parse(tokens); -- 2.30.2 From 3b4b0fd33cfc9b9a78e83e1a8ba3fb104b99991a Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Tue, 24 Dec 2013 18:51:18 -0500 Subject: [PATCH 11/43] added more error handling, fixed a typo in the tokenizer --- errors.js | 17 +------------- parse.js | 67 +++++++++++++++++++++++++++++++++++++---------------- tokenize.js | 4 ++-- 3 files changed, 50 insertions(+), 38 deletions(-) diff --git a/errors.js b/errors.js index e02eef9..05a1dab 100644 --- a/errors.js +++ b/errors.js @@ -5,20 +5,6 @@ * not thought about how that will work much */ -var JLException = { - stxerror : - function () { - console.log("There was an error\n", - "Line #",this.linenum,"\n", - "Character #", this.charnum,"\n", - this.errormessage); - }, - type_error : - function () { - return; - } -} - function JSyntaxError(linenum, charnum, message) { this.linenum = linenum; this.charnum = charnum; @@ -26,7 +12,7 @@ function JSyntaxError(linenum, charnum, message) { this.stxerror = function() { console.log("Syntax Error\n", "Line #", this.linenum,"\n", - "Character #", this.charnum, "\n", + "Near character #", this.charnum, "\n", this.errormessage); }; return this; @@ -39,7 +25,6 @@ function JTypeError(linenum, charnum, token, message) { this.token = token; return this; } -TypeError.prototype = JLException; module.exports = {JSyntaxError : JSyntaxError, diff --git a/parse.js b/parse.js index 38d1deb..5d2469a 100755 --- a/parse.js +++ b/parse.js @@ -37,7 +37,7 @@ function makeChecker(props) { /*Tries to parse until the prediction ``valid'' fails or the wrong type is parsed Collects the results into an array and returns it*/ -function parseMany(exprType, valid, tokens) { +function parseMany(exprType, valid, tokens, charnum, linenum) { var current = fst(tokens)[0]; var results = []; var parsed; @@ -46,7 +46,9 @@ function parseMany(exprType, valid, tokens) { parsed = parse(tokens); } else { - throw error.JSyntaxError(fst(tokens)[2], fst(tokens)[3], "unexpected token "+fst(tokens)[0]+" in parseMany"); + throw error.JSyntaxError(linenum, + charnum, + "unexpected token "+fst(tokens)[0]+" in parseMany"); } results.push(parsed); @@ -66,7 +68,9 @@ function parseMany(exprType, valid, tokens) { } //do the same validity check as before and in the loop if (!fst(tokens)) - throw "unexpected end of source"; + throw error.JSyntaxError(linenum, + charnum, + "unexpected end of source"); if (valid(fst(tokens)[0])) results.push(parse(tokens)); return results; @@ -76,7 +80,7 @@ function parseMany(exprType, valid, tokens) { /* Tries to parse exprType separated by the token between * e.g. ,,... */ -function parseBetween(exprType, between, tokens) { +function parseBetween(exprType, between, tokens, charnum, linenum) { var first = parse(tokens); if (!exprType(first)) { throw "unexpected token:"+fst(tokens)[0]; @@ -87,6 +91,10 @@ function parseBetween(exprType, between, tokens) { while (fst(tokens)[0] === between) { tokens.pop(); parsed = parse(tokens); + if (!fst(tokens)) + throw error.JSyntaxError(linenum, + charnum, + "Missing terminator: "+between); items.push(parsed); } return items; @@ -103,10 +111,12 @@ function parseList(tokens) { var xs = []; } else { - var xs = parseBetween(function (x) { return true; }, "comma", tokens); + var xs = parseBetween(function (x) { return true; }, "comma", tokens, fst(tokens)[3], fst(tokens)[2]); } if (fst(tokens)[0] !== "right_square") { - throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "list must be terminated by ]"); + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "list must be terminated by ]"); } tokens.pop(); return new typ.ListT(xs); @@ -116,16 +126,20 @@ function parseList(tokens) { function parseDefFunction(tokens) { var fname = parse(tokens); if (fname.exprType != "Name") { - throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Expected an identifier in function definition"); + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Expected an identifier in function definition"); } if (fst(tokens)[0] === "right_paren") { var parameters = []; } else { - var parameters = parseMany(validName, validFormPar, tokens); + var parameters = parseMany(validName, validFormPar, tokens, fst(tokens)[2], fst(tokens)[3]); } if ((fst(tokens)[0]) !== "right_paren") { - throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2],"Formal parameters must be followed by )"); + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Formal parameters must be followed by )"); } tokens.pop(); var body = parse(tokens); @@ -155,12 +169,16 @@ function parseDef(tokens) { function parseIf(tokens) { if (!notFollowedBy(tokens, ["def","comma","lambda"])) { - throw "``if'' cannot be followed by "+fst(tokens)[0]; + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "``if'' cannot be followed by "+fst(tokens)[0]) ; } else { var ifC = parse(tokens); if (!fst(tokens) || fst(tokens)[0] !== "thenexp") - throw "if must be folowed by exp, not "+snd(tokens)[0]; + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "if ``exp'' must be folowed by ``then'' exp, not "+snd(tokens)[0]); else { tokens.pop(); var thenC = parse(tokens); @@ -183,10 +201,12 @@ var validFormPar = makeChecker(["identifier"]); var validName = makeChecker(["Name"]); function parseLambda(tokens) { - var parameters = parseMany(validName,validFormPar, tokens); + var parameters = parseMany(validName,validFormPar, tokens, fst(tokens)[2], fst(tokens)[3]); if (fst(tokens)[0] !== "arrow") { - throw "arrow must follow parameters in lambda, not "+fst(tokens)[0]; + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "arrow must follow parameters in lambda, not "+fst(tokens)[0]); } tokens.pop() var body = parse(tokens); @@ -202,12 +222,14 @@ var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); var validOperator = makeChecker(["identifier"]); //Parses function application (either infix or prefix) -function computeApp(tokens) { +function computeApp(tokens, charnum, linenum) { var lhs = parse(tokens); if (fst(tokens)) var next = fst(tokens); else { - throw "Unexpected end of source"; + throw error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); } if (typ.OPInfo[next[1]]) { //it's an infix expression @@ -224,9 +246,11 @@ function computeApp(tokens) { else { //it's a prefix application - var parameters = parseMany(validArgTypes, validArgument, tokens); - if (fst(tokens)[0] !== "right_paren") { - throw "mismatched parentheses"; + var parameters = parseMany(validArgTypes, validArgument, tokens, fst(tokens)[2], fst(tokens)[3]); + if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { + throw error.JSyntaxError(linenum, + charnum, + "Mismatched parentheses or missing parenthesis on right-hand side"); } else { //return the result @@ -267,8 +291,11 @@ function parseInfix(tokens, minPrec, lhs) { } function parse(tokens) { - if (fst(tokens)) + var charnum = fst(tokens)[2]; + var linenum = fst(tokens)[3]; + if (fst(tokens)) { var toktype = fst(tokens)[0]; + } else { process.exit(code=1); } @@ -300,7 +327,7 @@ function parse(tokens) { return parsed; } else - return computeApp(tokens); + return computeApp(tokens, charnum, linenum); } // else if (toktype === "let") { // return parseLet(tokens); diff --git a/tokenize.js b/tokenize.js index 8c42e77..b80e58a 100755 --- a/tokenize.js +++ b/tokenize.js @@ -112,7 +112,7 @@ function tokenizeStr(tokstream, charnum, linenum) { } -function tokenizeT(tokstreami, charnum, linenum) { +function tokenizeT(tokstream, charnum, linenum) { if (tokstream.length < 4) return false; var next4 = tokstream.substr(0,4); @@ -249,7 +249,7 @@ function tokenize(tokstream) { case 100: // 'd' var result = peek(tokstream, "def", "def"); if (result) { - tokens.push(result); + tokens.push(["def", "def", charnum, linenum]); tokstream = tokstream.substr(3); break; } -- 2.30.2 From 863a7c58d7857c34bd11011775b4265aebb02283 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Wed, 25 Dec 2013 01:50:14 -0500 Subject: [PATCH 12/43] added more error checking --- parse.js | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/parse.js b/parse.js index 5d2469a..77603f6 100755 --- a/parse.js +++ b/parse.js @@ -20,7 +20,10 @@ function snd(ts) { //Checks if the next token is not followed by any of ``checks'' function notFollowedBy(tokens, checks) { - var nextT = fst(tokens)[0]; + if (!fst(tokens)) { + throw error.JSyntaxError(0,0,"unexpected end of source") + } + var nextT = fst(tokens)[0]; if (checks.some(function (x) {return x === nextT;})) return false; else @@ -38,6 +41,9 @@ function makeChecker(props) { /*Tries to parse until the prediction ``valid'' fails or the wrong type is parsed Collects the results into an array and returns it*/ function parseMany(exprType, valid, tokens, charnum, linenum) { + if (!fst(tokens)) { + throw error.JSyntaxError(charnum, linenum, "Unexpected end of source"); + } var current = fst(tokens)[0]; var results = []; var parsed; @@ -48,12 +54,12 @@ function parseMany(exprType, valid, tokens, charnum, linenum) { else { throw error.JSyntaxError(linenum, charnum, - "unexpected token "+fst(tokens)[0]+" in parseMany"); + "Unexpected token ``"+fst(tokens)[0]+"''"); } results.push(parsed); //make sure there are at least 2 tokens to parse - if (tokens.length > 1 && valid(fst(tokens)[0])) { + if (tokens.length > 1 && fst(tokens) && valid(fst(tokens)[0])) { while (valid(snd(tokens)[0])) { if (!(valid(fst(tokens)[0]))) break; @@ -61,7 +67,10 @@ function parseMany(exprType, valid, tokens, charnum, linenum) { results.push(parse(tokens)); if (!exprType(fst(results).exprType)) break; - current = fst(tokens)[0] + if (fst(tokens)) + current = fst(tokens)[0] + else + throw error.JSyntaxError(charnum, linenum, "Unexpected end of source"); if (tokens.length <= 1) break; } @@ -113,7 +122,7 @@ function parseList(tokens) { else { var xs = parseBetween(function (x) { return true; }, "comma", tokens, fst(tokens)[3], fst(tokens)[2]); } - if (fst(tokens)[0] !== "right_square") { + if (!fst(tokens) || fst(tokens)[0] !== "right_square") { throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "list must be terminated by ]"); @@ -146,8 +155,6 @@ function parseDefFunction(tokens) { return new typ.DefFunc(fname, parameters, body); } - - function parseDef(tokens) { if (fst(tokens)[0] === "left_paren") { // It's a function definition @@ -234,8 +241,10 @@ function computeApp(tokens, charnum, linenum) { if (typ.OPInfo[next[1]]) { //it's an infix expression var result = parseInfix(tokens, 1, lhs); - if (fst(tokens)[0] !== "right_paren") { - throw "mismatched parentheses"; + if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { + throw error.JSyntaxError(linenum, + charnum, + "Mismatched parentheses or missing parenthesis on right-hand side"); } else { //return the result @@ -246,8 +255,8 @@ function computeApp(tokens, charnum, linenum) { else { //it's a prefix application - var parameters = parseMany(validArgTypes, validArgument, tokens, fst(tokens)[2], fst(tokens)[3]); - if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { + var parameters = parseMany(validArgTypes, validArgument, tokens, charnum, linenum); + if ((!fst(tokens)) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, charnum, "Mismatched parentheses or missing parenthesis on right-hand side"); @@ -333,7 +342,7 @@ function parse(tokens) { // return parseLet(tokens); // } else { - throw "Unexpected token: " + toktype; + throw error.JSyntaxError(linenum, charnum, "Unexpected token: " + toktype); } } -- 2.30.2 From c9e43c8a40336a38399021211b4e5f4efb6920ba Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Wed, 25 Dec 2013 01:50:28 -0500 Subject: [PATCH 13/43] expanded test file --- test.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test.jl b/test.jl index eb9dca5..84ef9be 100644 --- a/test.jl +++ b/test.jl @@ -24,6 +24,10 @@ def empty [] def getFile (readFile "./parse.js") +def fileLines + (getFile >>= + ((mapM_ putStrLn) . lines)) + def (testUnary n) ((-n) + n) @@ -45,7 +49,7 @@ def main if False then undefined - else (getFile >>= print) + else fileLines >> (print - (splitxs "abcdefghijk"))) + (splitxs [12,3,4,56]))) -- 2.30.2 From 258e278f4e22ea9c7d07c219164bb66314829d23 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Wed, 25 Dec 2013 12:39:02 -0500 Subject: [PATCH 14/43] more error handling shit --- parse.js | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/parse.js b/parse.js index 77603f6..f2f2a47 100755 --- a/parse.js +++ b/parse.js @@ -42,7 +42,9 @@ function makeChecker(props) { Collects the results into an array and returns it*/ function parseMany(exprType, valid, tokens, charnum, linenum) { if (!fst(tokens)) { - throw error.JSyntaxError(charnum, linenum, "Unexpected end of source"); + throw error.JSyntaxError(charnum, + linenum, + "Unexpected end of source"); } var current = fst(tokens)[0]; var results = []; @@ -54,7 +56,7 @@ function parseMany(exprType, valid, tokens, charnum, linenum) { else { throw error.JSyntaxError(linenum, charnum, - "Unexpected token ``"+fst(tokens)[0]+"''"); + "Unexpected token: ``"+fst(tokens)[0]+"''"); } results.push(parsed); @@ -92,7 +94,7 @@ function parseMany(exprType, valid, tokens, charnum, linenum) { function parseBetween(exprType, between, tokens, charnum, linenum) { var first = parse(tokens); if (!exprType(first)) { - throw "unexpected token:"+fst(tokens)[0]; + throw error.JSyntaxError(charnum, linenum, "Unexpected token: ``"+fst(tokens)[0]+"''"); } var items = [first]; var parsed; @@ -156,18 +158,26 @@ function parseDefFunction(tokens) { } function parseDef(tokens) { + if (tokens.length < 2) + throw error.JSyntaxError(0,0,"Unexpected end of source"); + var charnum = fst(tokens)[2]; + var linenum = fst(tokens)[3]; if (fst(tokens)[0] === "left_paren") { // It's a function definition tokens.pop(); return parseDefFunction(tokens); } if (notFollowedBy(tokens, ["identifier"])) { - throw "def must be followed by identifier, not "+fst(tokens)[0]; + throw error.JSyntaxError(linenum, + charnum, + "def must be followed by identifier, not "+fst(tokens)[0]); } else { var identifier = parse(tokens); if (!notFollowedBy(tokens, ["def", "comma", "arrow", "right_brace", "right_square"])) { - throw "def " + identifier.val + " must not be followed by " + fst(tokens)[0]; + throw error.JSyntaxError(linenum, + charnum, + "def " + identifier.val + " must not be followed by " + fst(tokens)[0]); } return new typ.Def(identifier, parse(tokens)); } @@ -240,7 +250,7 @@ function computeApp(tokens, charnum, linenum) { } if (typ.OPInfo[next[1]]) { //it's an infix expression - var result = parseInfix(tokens, 1, lhs); + var result = parseInfix(tokens, 1, lhs, linenum, charnum); if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, charnum, @@ -273,14 +283,16 @@ function computeApp(tokens, charnum, linenum) { See this for more info and an implementation in python http://eli.thegreenplace.net/2012/08/02/parsing-expressions-by-precedence-climbing/ */ -function parseInfix(tokens, minPrec, lhs) { +function parseInfix(tokens, minPrec, lhs, linenum, charnum) { if (!lhs) { var lhs = parse(tokens); } while (true) { var cur = fst(tokens); if (!cur) { - throw "Unexpected end of source"; + throw error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); } var opinfo = typ.OPInfo[cur[1]]; @@ -342,7 +354,9 @@ function parse(tokens) { // return parseLet(tokens); // } else { - throw error.JSyntaxError(linenum, charnum, "Unexpected token: " + toktype); + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Unexpected token: ``" + toktype+"''"); } } -- 2.30.2 From 8257dceac93cf60ead9619e2d5cc3d5373c00a0a Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Thu, 26 Dec 2013 22:21:59 -0500 Subject: [PATCH 15/43] quick update --- parse.js | 59 ++++++++++++++++++++++++++++++++++---------------------- test.jl | 2 +- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/parse.js b/parse.js index f2f2a47..2f05f55 100755 --- a/parse.js +++ b/parse.js @@ -18,20 +18,22 @@ function snd(ts) { return ts[ts.length-2]; } -//Checks if the next token is not followed by any of ``checks'' -function notFollowedBy(tokens, checks) { +/*Checks if the next token is not followed by any of ``checks'' */ +function notFollowedBy(tokens, checks, linenum, charnum) { if (!fst(tokens)) { throw error.JSyntaxError(0,0,"unexpected end of source") } var nextT = fst(tokens)[0]; - if (checks.some(function (x) {return x === nextT;})) + if (checks.some(function (x) { + return x === nextT; + })) return false; else return true; } -//returns a function that takes a parameter and -//checks if it is in the array ``props'' +/* returns a function that takes a parameter and + checks if it is in the array ``props''*/ function makeChecker(props) { return function(x) { return x && props.some(function (y) {return y === x;}); @@ -65,7 +67,6 @@ function parseMany(exprType, valid, tokens, charnum, linenum) { while (valid(snd(tokens)[0])) { if (!(valid(fst(tokens)[0]))) break; - //print(valid(fst(tokens)[0]), tokens); results.push(parse(tokens)); if (!exprType(fst(results).exprType)) break; @@ -157,24 +158,33 @@ function parseDefFunction(tokens) { return new typ.DefFunc(fname, parameters, body); } -function parseDef(tokens) { +function parseDef(tokens, linenum, charnum) { if (tokens.length < 2) - throw error.JSyntaxError(0,0,"Unexpected end of source"); - var charnum = fst(tokens)[2]; - var linenum = fst(tokens)[3]; + throw error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); if (fst(tokens)[0] === "left_paren") { - // It's a function definition + /* It's a function definition */ tokens.pop(); return parseDefFunction(tokens); } - if (notFollowedBy(tokens, ["identifier"])) { + if (notFollowedBy(tokens, ["identifier"], linenum, charnum)) { throw error.JSyntaxError(linenum, charnum, "def must be followed by identifier, not "+fst(tokens)[0]); } else { var identifier = parse(tokens); - if (!notFollowedBy(tokens, ["def", "comma", "arrow", "right_brace", "right_square"])) { + if (!fst(tokens)) + throw error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); + linenum = fst(tokens)[3]; + charnum = fst(tokens)[2]; + if (!notFollowedBy(tokens, + ["def", "comma", "arrow", "right_brace", "right_square"], + linenum, + charnum)) { throw error.JSyntaxError(linenum, charnum, "def " + identifier.val + " must not be followed by " + fst(tokens)[0]); @@ -185,9 +195,14 @@ function parseDef(tokens) { function parseIf(tokens) { - if (!notFollowedBy(tokens, ["def","comma","lambda"])) { - throw error.JSyntaxError(fst(tokens)[3], - fst(tokens)[2], + var linenum = fst(tokens)[3]; + var charnum = fst(tokens)[2]; + if (!notFollowedBy(tokens, + ["def","comma","lambda"], + linenum, + charnum)) { + throw error.JSyntaxError(linenum, + charnum, "``if'' cannot be followed by "+fst(tokens)[0]) ; } else { @@ -238,7 +253,7 @@ var validArgument = tool.compose(tool.not, makeChecker(invalidArguments)); var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); var validOperator = makeChecker(["identifier"]); -//Parses function application (either infix or prefix) +/* Parses function application (either infix or prefix) */ function computeApp(tokens, charnum, linenum) { var lhs = parse(tokens); if (fst(tokens)) @@ -249,7 +264,7 @@ function computeApp(tokens, charnum, linenum) { "Unexpected end of source"); } if (typ.OPInfo[next[1]]) { - //it's an infix expression + /* it's an infix expression */ var result = parseInfix(tokens, 1, lhs, linenum, charnum); if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, @@ -257,13 +272,12 @@ function computeApp(tokens, charnum, linenum) { "Mismatched parentheses or missing parenthesis on right-hand side"); } else { - //return the result tokens.pop(); return result; } } else { - //it's a prefix application + /* it's a prefix application */ var parameters = parseMany(validArgTypes, validArgument, tokens, charnum, linenum); if ((!fst(tokens)) || fst(tokens)[0] !== "right_paren") { @@ -272,7 +286,6 @@ function computeApp(tokens, charnum, linenum) { "Mismatched parentheses or missing parenthesis on right-hand side"); } else { - //return the result tokens.pop(); return typ.makeApp(lhs, parameters); } @@ -304,7 +317,7 @@ function parseInfix(tokens, minPrec, lhs, linenum, charnum) { var assoc = opinfo[1]; var nextMinPrec = assoc === "Left" ? prec + 1 : prec; tokens.pop(); - //remove the operator token + /*remove the operator token*/ var rhs = parseInfix(tokens, nextMinPrec); lhs = typ.makeApp(op, [lhs, rhs]); } @@ -337,7 +350,7 @@ function parse(tokens) { else if (toktype === "truelit" || toktype === "falselit") return new typ.BoolT(token); else if (toktype === "def") - return parseDef(tokens); + return parseDef(tokens, fst(tokens)[3], fst(tokens)[2]); else if (toktype === "ifexp") return parseIf(tokens); else if (toktype === "left_paren") { diff --git a/test.jl b/test.jl index 84ef9be..288a449 100644 --- a/test.jl +++ b/test.jl @@ -4,7 +4,7 @@ def (f a b) def (add a b) (a + b) -def (catstrs strs) +def (catstrs strs) (foldr f (head strs) (tail strs)) -- 2.30.2 From 02c20040c6da0585382cf50f3c60374db05a7681 Mon Sep 17 00:00:00 2001 From: wes Date: Fri, 27 Dec 2013 01:19:24 -0500 Subject: [PATCH 16/43] removed some old comments --- parse.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/parse.js b/parse.js index 2f05f55..0d89fb9 100755 --- a/parse.js +++ b/parse.js @@ -168,6 +168,13 @@ function parseDef(tokens, linenum, charnum) { tokens.pop(); return parseDefFunction(tokens); } + + if (fst(tokens)[0] === "left_brace") { + /* It's a let/def form */ + tokens.pop(); + return parseLetForm(tokens); + } + if (notFollowedBy(tokens, ["identifier"], linenum, charnum)) { throw error.JSyntaxError(linenum, charnum, @@ -349,7 +356,8 @@ function parse(tokens) { return new typ.Name(token); else if (toktype === "truelit" || toktype === "falselit") return new typ.BoolT(token); - else if (toktype === "def") + else if (toktype === "def" || + toktype === "let") return parseDef(tokens, fst(tokens)[3], fst(tokens)[2]); else if (toktype === "ifexp") return parseIf(tokens); @@ -363,9 +371,6 @@ function parse(tokens) { else return computeApp(tokens, charnum, linenum); } -// else if (toktype === "let") { -// return parseLet(tokens); -// } else { throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], -- 2.30.2 From fa70001d4b042bf57d558af0576ec72f67098eea Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Fri, 27 Dec 2013 02:53:56 -0500 Subject: [PATCH 17/43] basic support for let/def forms in the parser --- parse.js | 136 ++++++++++++++++++++++++++++++++++++++++++---- representation.js | 11 ++-- 2 files changed, 133 insertions(+), 14 deletions(-) diff --git a/parse.js b/parse.js index 0d89fb9..d2f0317 100755 --- a/parse.js +++ b/parse.js @@ -42,7 +42,7 @@ function makeChecker(props) { /*Tries to parse until the prediction ``valid'' fails or the wrong type is parsed Collects the results into an array and returns it*/ -function parseMany(exprType, valid, tokens, charnum, linenum) { +function parseMany(parse, exprType, valid, tokens, charnum, linenum) { if (!fst(tokens)) { throw error.JSyntaxError(charnum, linenum, @@ -146,7 +146,12 @@ function parseDefFunction(tokens) { var parameters = []; } else { - var parameters = parseMany(validName, validFormPar, tokens, fst(tokens)[2], fst(tokens)[3]); + var parameters = parseMany(parse, + validName, + validFormPar, + tokens, + fst(tokens)[2], + fst(tokens)[3]); } if ((fst(tokens)[0]) !== "right_paren") { throw error.JSyntaxError(fst(tokens)[3], @@ -158,6 +163,110 @@ function parseDefFunction(tokens) { return new typ.DefFunc(fname, parameters, body); } +validLet = makeChecker(["Definition", "FunctionDefinition"]); +letEnd = tool.compose(tool.not, makeChecker(["right_brace"])); + +function parseLetForm(tokens, linenum, charnum) { + if (!fst(tokens)) { + error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); + } + var pairs = parseMany(parseLetItem, + validLet, + letEnd, + tokens, + linenum, + charnum); + if (fst(tokens) && fst(tokens)[0] !== "right_brace") { + throw error.JSyntaxError(fst(tokens)[2], + fst(tokens)[3], + "let/def form must have a closing }"); + } + if (!fst(tokens)) { + throw error.JSyntaxError(linenum, + charnum, + "Unexpected end of source"); + } + tokens.pop(); + var body = parse(tokens); + return new typ.LetExp(pairs, body); + +} + +function parseLetFunction(tokens, linenum, charnum) { + var fname = parse(tokens); + + if (fname.exprType != "Name") { + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Expected an identifier in function definition"); + } + if (fst(tokens)[0] === "right_paren") { + var parameters = []; + } + else { + var parameters = parseMany(parse, + validName, + validFormPar, + tokens, + fst(tokens)[2], + fst(tokens)[3]); + } + if ((fst(tokens)[0]) !== "right_paren") { + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Formal parameters must be followed by )"); + } + tokens.pop(); + if (fst(tokens)[0] !== "arrow") { + throw error.JSyntaxError(fst(tokens)[3], + fst(tokens)[2], + "Function parameters in let/def form must be followed by ->") + } + tokens.pop(); + var body = parse(tokens); + return new typ.DefFunc(fname, parameters, body); +} + +function parseLetBinding(tokens, linenum, charnum) { + var name = parse(tokens); + if (name.exprType != "Name") { + throw error.JSyntaxError(linenum, + charnum, + "Expected an identifier in let/def binding"); + } + if (!fst(tokens) || fst(tokens)[1] !== "=") { + throw error.JSyntaxError(linenum, + charnum, + "An identifier in a let/def binding must be followed by ``=''"); + } + tokens.pop(); + if (!notFollowedBy(tokens, + ["def", "comma", "arrow", "right_brace", "right_square"], + linenum, + charnum)) { + throw error.JSyntaxError(linenum, + charnum, + "The binding of " + identifier.val + " must not be followed by " + fst(tokens)[0]); + } + return new typ.Def(name, parse(tokens)); +} + +function parseLetItem(tokens) { + if (fst(tokens) && fst(tokens)[0] === "left_paren") { + tokens.pop(); + return parseLetFunction(tokens, + fst(tokens)[3], + fst(tokens)[2]); + } + else { + return parseLetBinding(tokens, + fst(tokens)[3], + fst(tokens)[2]); + } +} + function parseDef(tokens, linenum, charnum) { if (tokens.length < 2) throw error.JSyntaxError(linenum, @@ -166,7 +275,7 @@ function parseDef(tokens, linenum, charnum) { if (fst(tokens)[0] === "left_paren") { /* It's a function definition */ tokens.pop(); - return parseDefFunction(tokens); + return parseDefFunction(tokens, linenum, charnum); } if (fst(tokens)[0] === "left_brace") { @@ -240,7 +349,12 @@ var validFormPar = makeChecker(["identifier"]); var validName = makeChecker(["Name"]); function parseLambda(tokens) { - var parameters = parseMany(validName,validFormPar, tokens, fst(tokens)[2], fst(tokens)[3]); + var parameters = parseMany(parse, + validName, + validFormPar, + tokens, + fst(tokens)[2], + fst(tokens)[3]); if (fst(tokens)[0] !== "arrow") { throw error.JSyntaxError(fst(tokens)[3], @@ -252,9 +366,6 @@ function parseLambda(tokens) { return new typ.FuncT(parameters, body); } -//function parseLet(tokens) { - - var invalidArguments = ["def", "comma", "right_paren", "right_square", "right_brace", "left_brace", "right_brace"]; var validArgument = tool.compose(tool.not, makeChecker(invalidArguments)); var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); @@ -286,7 +397,12 @@ function computeApp(tokens, charnum, linenum) { else { /* it's a prefix application */ - var parameters = parseMany(validArgTypes, validArgument, tokens, charnum, linenum); + var parameters = parseMany(parse, + validArgTypes, + validArgument, + tokens, + charnum, + linenum); if ((!fst(tokens)) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, charnum, @@ -392,9 +508,9 @@ function parseFull(tokenized) { process.exit(1); } } -console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); +//console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); //console.log(tokenizer.tokenize(istr)); -//console.log(parseFull(tokenizer.tokenize(istr))[8].val.body); +console.log(parseFull(tokenizer.tokenize(istr))); //module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) }; diff --git a/representation.js b/representation.js index 0521183..3f56eab 100644 --- a/representation.js +++ b/representation.js @@ -20,14 +20,17 @@ var Expression = { } }; -function LetExp(pairs) { +function LetExp(pairs, body) { if (!pairs.every(function(x) { - return x.exprType === "Name"; + return (x.exprType === "Definition" || + x.exprType === "FunctionDefinition"); })) { - throw "let can only be used to bind things to names"; + throw "let can only be used to bind things to names or functions"; } this.exprType = "Let"; - this.val = tool.dict(pairs); + this.val = [pairs, body]; + this.pairs = pairs; + this.body = body; return this; } LetExp.prototype = Expression; -- 2.30.2 From b779025122d5417fdb3c5e17732793e747f44a15 Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Fri, 27 Dec 2013 03:04:51 -0500 Subject: [PATCH 18/43] fixed error handling for let/def --- parse.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/parse.js b/parse.js index d2f0317..fa88765 100755 --- a/parse.js +++ b/parse.js @@ -188,7 +188,14 @@ function parseLetForm(tokens, linenum, charnum) { charnum, "Unexpected end of source"); } + linenum = fst(tokens)[2]; + charnum = fst(tokens)[3]; tokens.pop(); + if (tokens.length <= 0) { + throw error.JSyntaxError(linenum, + charnum, + "let/def form must have a body"); + } var body = parse(tokens); return new typ.LetExp(pairs, body); @@ -281,7 +288,9 @@ function parseDef(tokens, linenum, charnum) { if (fst(tokens)[0] === "left_brace") { /* It's a let/def form */ tokens.pop(); - return parseLetForm(tokens); + return parseLetForm(tokens, + fst(tokens)[2], + fst(tokens)[3]); } if (notFollowedBy(tokens, ["identifier"], linenum, charnum)) { -- 2.30.2 From 7589d5c4d2b6143be4970a3b38d6e47fbce4eb4b Mon Sep 17 00:00:00 2001 From: Wesley Kerfoot Date: Sun, 29 Dec 2013 22:04:23 -0500 Subject: [PATCH 19/43] desugaring for let/def expressions, more error handling, expanded test cases --- desugar.js | 11 +++++++++-- parse.js | 40 ++++++++++++++++++++++++++++++---------- pprint.js | 5 +++++ representation.js | 3 ++- test.jl | 26 ++++++++++++++++++-------- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/desugar.js b/desugar.js index 0f3fdd8..2a1d19e 100644 --- a/desugar.js +++ b/desugar.js @@ -20,11 +20,16 @@ function desugarList(lst) { } function desugarDefFunc(def) { - return new typ.Def(def.ident, new typ.FuncT(desugar(def.params), desugar(def.body))); + return new typ.Def(def.ident, + new typ.FuncT(desugar(def.params), + desugar(def.body))); } -//function desugarString(str) { +function desugarLet(stx) { + var values = stx.pairs.map(desugar); + return new typ.LetExp(values, desugar(stx.body)); +} function desugar(stx) { switch (stx.exprType) { @@ -59,6 +64,8 @@ function desugar(stx) { return stx; case "Integer": return stx; + case "Let": + return desugarLet(stx); default: return stx; } diff --git a/parse.js b/parse.js index fa88765..9f6ca87 100755 --- a/parse.js +++ b/parse.js @@ -188,8 +188,8 @@ function parseLetForm(tokens, linenum, charnum) { charnum, "Unexpected end of source"); } - linenum = fst(tokens)[2]; - charnum = fst(tokens)[3]; + linenum = fst(tokens)[3]; + charnum = fst(tokens)[2]; tokens.pop(); if (tokens.length <= 0) { throw error.JSyntaxError(linenum, @@ -197,6 +197,12 @@ function parseLetForm(tokens, linenum, charnum) { "let/def form must have a body"); } var body = parse(tokens); + if (body.exprType === "Definition" || + body.exprType === "FunctionDefinition") { + throw error.JSyntaxError(linenum, + charnum, + "Body of a let/def expression cannot be a definition"); + } return new typ.LetExp(pairs, body); } @@ -250,14 +256,21 @@ function parseLetBinding(tokens, linenum, charnum) { } tokens.pop(); if (!notFollowedBy(tokens, - ["def", "comma", "arrow", "right_brace", "right_square"], + ["comma", "arrow", "right_brace", "right_square"], linenum, charnum)) { throw error.JSyntaxError(linenum, charnum, "The binding of " + identifier.val + " must not be followed by " + fst(tokens)[0]); } - return new typ.Def(name, parse(tokens)); + var bound = parse(tokens); + if (bound.exprType === "Definition" || + bound.exprType === "FunctionDefinition") { + throw error.JSyntaxError(linenum, + charnum, + "A definition cannot be the value of a binding"); + } + return new typ.Def(name, bound); } function parseLetItem(tokens) { @@ -289,8 +302,8 @@ function parseDef(tokens, linenum, charnum) { /* It's a let/def form */ tokens.pop(); return parseLetForm(tokens, - fst(tokens)[2], - fst(tokens)[3]); + fst(tokens)[3], + fst(tokens)[2]); } if (notFollowedBy(tokens, ["identifier"], linenum, charnum)) { @@ -307,14 +320,21 @@ function parseDef(tokens, linenum, charnum) { linenum = fst(tokens)[3]; charnum = fst(tokens)[2]; if (!notFollowedBy(tokens, - ["def", "comma", "arrow", "right_brace", "right_square"], + ["comma", "arrow", "right_brace", "right_square"], linenum, charnum)) { throw error.JSyntaxError(linenum, charnum, "def " + identifier.val + " must not be followed by " + fst(tokens)[0]); } - return new typ.Def(identifier, parse(tokens)); + var bound = parse(tokens); + if (bound.exprType === "Definition" || + bound.exprType === "FunctionDefinition") { + throw error.JSyntaxError(linenum, + charnum, + "A definition cannot be the value of a binding"); + } + return new typ.Def(identifier, bound); } } @@ -517,9 +537,9 @@ function parseFull(tokenized) { process.exit(1); } } -//console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); +console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); //console.log(tokenizer.tokenize(istr)); -console.log(parseFull(tokenizer.tokenize(istr))); +//console.log(parseFull(tokenizer.tokenize(istr))); //module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) }; diff --git a/pprint.js b/pprint.js index a5cdf36..19a94a8 100644 --- a/pprint.js +++ b/pprint.js @@ -55,6 +55,11 @@ function pprint(expr) { return "[]"; else if (expr.exprType === "Unary") return "("+expr.op.ident+" "+pprint(expr.val)+")"; + else if (expr.exprType === "Let") + return "let {" + expr.pairs.map( + function (v) { + return pprint(v) + }).join(" ; ") + "} in " + pprint(expr.body); } module.exports = {pprint : pprint}; diff --git a/representation.js b/representation.js index 3f56eab..bc19d72 100644 --- a/representation.js +++ b/representation.js @@ -215,4 +215,5 @@ module.exports = DefFunc : DefFunc, UnaryOp : UnaryOp, Nil : Nil, - LetExp : LetExp} + LetExp : LetExp + } diff --git a/test.jl b/test.jl index 288a449..ae33ebe 100644 --- a/test.jl +++ b/test.jl @@ -44,12 +44,22 @@ def (splitHelp acc xs ys) def (splitxs xs) (splitHelp [] xs xs) +def r def + { + a = 4 + } + a + def main - ((print (testUnary 6)) >> - if False - then - undefined - else fileLines - >> - (print - (splitxs [12,3,4,56]))) + let { + unary = (print (testUnary 6)) + splitted = def { + xs = (fst (splitxs [12,3,4,56])) + } (xs ++ [0,9]) + } + if False + then undefined + else + (unary >> + fileLines >> + (print splitted)) -- 2.30.2 From b092190c08bd69672a8abc6b27408a5153eb7d03 Mon Sep 17 00:00:00 2001 From: wes Date: Wed, 1 Jan 2014 04:54:29 -0500 Subject: [PATCH 20/43] preliminary support for user-defined operators --- tokenize.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tokenize.js b/tokenize.js index b80e58a..66d0a7f 100755 --- a/tokenize.js +++ b/tokenize.js @@ -247,8 +247,13 @@ function tokenize(tokstream) { } case 100: // 'd' - var result = peek(tokstream, "def", "def"); - if (result) { + var defop = peek(tokstream, "defop", "defop"); + if (defop) { + tokens.push(["defop", "defop", charnum, linenum]); + tokstream = tokstream.substr(5); + } + var def = peek(tokstream, "def", "def"); + if (def) { tokens.push(["def", "def", charnum, linenum]); tokstream = tokstream.substr(3); break; -- 2.30.2 From 71f9342ceabb0e00d7f64d596373ff5e92bddb07 Mon Sep 17 00:00:00 2001 From: wes Date: Fri, 3 Jan 2014 19:01:03 -0500 Subject: [PATCH 21/43] support for user defined operators, working on type declarations and inference --- parse.js | 49 +++++++++++++++++++++++++++++++++++++++- representation.js | 57 +++++++++++++++++++++++++++++++---------------- tokenize.js | 37 +++++++++++++++++++++--------- 3 files changed, 113 insertions(+), 30 deletions(-) diff --git a/parse.js b/parse.js index 9f6ca87..903ce75 100755 --- a/parse.js +++ b/parse.js @@ -338,6 +338,51 @@ function parseDef(tokens, linenum, charnum) { } } +function parseDefOp(tokens, linenum, charnum) { + if (fst(tokens)[0] !== "integer" || + fst(tokens)[1] < 1) { + throw error.JSyntaxError(linenum, + charnum, + "defop must be followed by integer precedence >= 1"); + } + tokens.pop(); + + if (false) { + throw error.JSyntaxError(linenum, + charnum, + "defop must be followed by precedence and then either Left or Right"); + } + tokens.pop(); + if (fst(tokens)[0] !== "left_paren") { + throw error.JSyntaxError(linenum, + charnum, + "defop arguments must start with ("); + } + tokens.pop(); + if (!(tokens.slice(tokens.length-3, + tokens.length).every(function(x) { + return x[0] === "identifier"; + }))) { + throw error.JSyntaxError(linenum, + charnum, + "defop must be surrounded by exactly 3 identifier"); + } + var pattern = tokens.slice(tokens.length-3, + tokens.length); + tokens.pop(); tokens.pop(); tokens.pop(); + if (fst(tokens)[0] !== "right_paren") { + throw error.JSyntaxError(linenum, + charnum, + "defop pattern must be terminated with )"); + } + tokens.pop(); + return new typ.DefFunc(new typ.Name(pattern[1][1]), + [new typ.Name(pattern[0][1]), + new typ.Name(pattern[2][1])], + parse(tokens)); +} + + function parseIf(tokens) { var linenum = fst(tokens)[3]; @@ -498,12 +543,14 @@ function parse(tokens) { else if (toktype === "float") return new typ.FloatT(token); else if (toktype === "identifier") - return new typ.Name(token); + return new typ.Name(token); else if (toktype === "truelit" || toktype === "falselit") return new typ.BoolT(token); else if (toktype === "def" || toktype === "let") return parseDef(tokens, fst(tokens)[3], fst(tokens)[2]); + else if (toktype === "defop") + return parseDefOp(tokens, fst(tokens)[3], fst(tokens)[2]); else if (toktype === "ifexp") return parseIf(tokens); else if (toktype === "left_paren") { diff --git a/representation.js b/representation.js index bc19d72..09d456f 100644 --- a/representation.js +++ b/representation.js @@ -156,6 +156,24 @@ function If(condition, thenexp, elseexp) { return this; } +function TypeVar(name) { + this.name = name; + return this; +} + +function TypeOp(name, params, body) { + this.name = name; + this.params = params; + this.body = body; + return this; +} + +function TypeBinding(name, type) { + this.name = name; + this.type = type; + return this; +} + //convenience function to construct binary operators //assumes that the identifier refers to the name of a primitive @@ -178,25 +196,26 @@ function makeApp(name, parameters) { } OPInfo = {"+" : [3, "Left"], - "-" : [3, "Left"], - "*" : [4, "Left"], - "/" : [4, "Left"], - "^" : [5, "Right"], - "++" : [3, "Left"], - "==" : [2, "Left"], - ">" : [2, "Left"], - ">=" : [2, "Left"], - "<" : [2, "Left"], - "<=" : [2, "Left"], - "&&" : [2, "Left"], - "||" : [2, "Left"], - ":" : [2, "Left"], - "$" : [1, "Left"], - ">>" : [1, "Left"], - ">>=" : [1, "Left"], - "<$>" : [1, "Left"], - "." : [1, "Left"], - "," : [1, "Left"]} + "-" : [3, "Left"], + "*" : [4, "Left"], + "/" : [4, "Left"], + "^" : [5, "Right"], + "++" : [3, "Left"], + "==" : [2, "Left"], + ">" : [2, "Left"], + ">=" : [2, "Left"], + "<" : [2, "Left"], + "<=" : [2, "Left"], + "&&" : [2, "Left"], + "||" : [2, "Left"], + "::" : [2, "Left"], + ":" : [1, "Left"], + "$" : [1, "Left"], + ">>" : [1, "Left"], + ">>=" : [1, "Left"], + "<$>" : [1, "Left"], + "." : [1, "Left"], + "," : [1, "Left"]} module.exports = { IntT : IntT, diff --git a/tokenize.js b/tokenize.js index 66d0a7f..5cdd1cc 100755 --- a/tokenize.js +++ b/tokenize.js @@ -5,8 +5,6 @@ var tools = require("./tools.js"); var error = require("./errors.js"); var operators = Object.keys(rep.OPInfo); -var matchop = tools.opMatch(operators); - function isDigit(a) { if (!a) return false; @@ -80,7 +78,7 @@ function tokenizeNum(tokstream, charnum, linenum) { * Everything after the operator goes back on to the token stream */ -function tokenizeIdent(tokstream, charnum, linenum) { +function tokenizeIdent(tokstream, matchop, charnum, linenum) { var identifier = []; var n = 0; while ((!isWhitespace(tokstream[0])) && isIdentifier(tokstream[0]) && !matchop(tokstream)) { @@ -134,7 +132,7 @@ function peek(tokstream, toktype, word, charnum, linenum) { return false; } -function tokenize(tokstream) { +function tokenize(tokstream, matchop) { var tokens = []; var charnum = 1; var linenum = 1; @@ -239,10 +237,10 @@ function tokenize(tokstream) { tokstream = tokstream.substr(2); break; } - var inkeyword = peek(tokstream, "in", "in"); + var inkeyword = peek(tokstream, "in", "in "); if (inkeyword) { tokens.push(inkeyword); - tokstream = tokstream.substr(2); + tokstream = tokstream.substr(3); break; } @@ -251,6 +249,7 @@ function tokenize(tokstream) { if (defop) { tokens.push(["defop", "defop", charnum, linenum]); tokstream = tokstream.substr(5); + break; } var def = peek(tokstream, "def", "def"); if (def) { @@ -304,21 +303,21 @@ function tokenize(tokstream) { tokens.push(["identifier", op, charnum, linenum]); } else { - var result = tokenizeIdent(tokstream, charnum, linenum); + var result = tokenizeIdent(tokstream, matchop, charnum, linenum); result.map(function(x) { charnum++; tokens.push(x[1]); tokstream = tokstream.substr(x[0]); }); } - } + } } return tokens; } -function tokenizeFull(input) { +function tokenizeHelp(input, matchop) { try { - return tokenize(input).reverse().filter(function(x) { + return tokenize(input, matchop).reverse().filter(function(x) { return x[0] !== "whitespace"; }); } catch (e) { @@ -327,4 +326,22 @@ function tokenizeFull(input) { } } +function tokenizeFull(input) { + var matchop = tools.opMatch(operators); + var initialPass = tokenizeHelp(input, matchop).reverse();; + for (var i = 0; i < initialPass.length; i++) { + if (initialPass.slice(i, i+7).map(function(x) { return x[0]; }) === + ["defop", "integer", "identifier", "Left", "Right", + "left_paren", "identifier", "identifier", "identifier", + "right_paren"]) + rep.OPInfo[initialPass[i+5][1]] = [parseInt(initialPass[i+1][1], 10), + initialPass[i+2][1]]; + + } + operators = Object.keys(rep.OPInfo); + matchop = tools.opMatch(operators); + return tokenizeHelp(input, matchop); +} + + module.exports = {tokenize : tokenizeFull}; -- 2.30.2 From 527f1f5e3422fdbc1dd425dd056ecc430d0ce74c Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 1 Mar 2014 21:21:56 -0500 Subject: [PATCH 22/43] adding cexp intermediate-representation, still needs work obviously --- cexps.js | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++ optimize.js | 11 ----- 2 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 cexps.js delete mode 100644 optimize.js diff --git a/cexps.js b/cexps.js new file mode 100644 index 0000000..17b484a --- /dev/null +++ b/cexps.js @@ -0,0 +1,135 @@ +/* + * Defines the data structures associated with Continuation Expressions (cexps) + * The object here is to have an intermediate representation that allows a code generator backend + * to easily create sequential code. In other words we are taking an alegraic/applicative language + * and translating it to something more similar to the "one word at a time" architecture of the von-Neumann + * architecture + */ + +var cexp = { + type : "cexp" +} + +function record(values_accesspaths, + w, + next_cexp) { + this.values_accesspaths = values_accesspaths; + this.w = w; + this.next_cexp = next_cexp; + return this; +} + +record.prototype = cexp; + +function select(i, v, w, next_cexp) { + this.i = i; + this.v = v; + this.w = w; + this.next_cexp = next_cexp; + return this; +} +select.prototype = cexp; + +function offset(i, v, w, next_cexp) { + this.i = i; + this.v = v; + this.w = w; + this.next_cexp = next_cexp; + return this; +} +offset.prototype = cexp; + +function app(k, vs) { + this.k =k; + this.vs = vs; + return this; +} +app.prototype = cexp; + +function fix(fs, next_cexp) { + this.fs = fs; + this.next_cexp = next_cexp; + return this; +} +fix.prototype = cexp; + +function switchl(v, cexps) { + this.v = v; + this.cexps = cexps; + return this; +} +switchl.prototype = cexp; + +function primop(op, vals, vars, next_cexp) { + this.op = op; + this.vals = vals; + this.vars = vars; + this.next_cexp = next_cexp; + return this; +} +primop.prototype = cexp; + +function accessPath(offp, selp) { + this.offp = offp; + this.selp = selp; + return this; +} + +var primoptype = { + type : "primop", + equal : function(pOp) { + return this.name === pOp.name + } +}; + +function Primop(name) { + function ptype() { + this.name = name; + return this; + }; + ptype.prototype = primoptype; + return new ptype(); +} + +var times = Primop("+"); +var plus = Primop("+"); +var div = Primop("div"); +var tilde = Primop("~"); +var ieql = Primop("ieql"); +var ineq = Primop("ineq"); +var lessthan = Primop("<"); +var lessoreq = Primop("<="); +var greatthan = Primop(">"); +var greatoreq = Primop(">="); +var bang = Primop("!"); +var subscript = Protoype("subscript"); +var ordof = Primop("ordof"); +var assign = Primop(":="); +var unboxedassign = Primop("unboxedassign"); +var update = Primop("update"); +var unboxedupdate = Primop("unboxedupdate"); +var store = Primop("store"); +var makeref = Primop("makeref"); +var makerefunboxed = Primop("makerefunboxed"); +var alength = Primop("alength"); +var slength = Primop("slength"); +var gethdlr = Primop("gethdlr") +var sethdlr = Primop("sethdlr"); +var boxed = Primop("boxed"); +var fadd = Primop("fadd"); +var fsub = Primop("fsub"); +var fdiv = Primop("fdiv"); +var fmul = Primop("fmul"); +var feql = Primop("feql"); +var fneq = Primop("fneq"); +var fge = Primop("fge"); +var fgt = Primop("fgt"); +var fle = Primop("fle"); +var flt = Primop("flt"); +var rshift = Primop("rshift"); +var lshift = Primop("lshift"); +var orb = Primop("orb"); +var andb = Primop("andb"); +var xorb = Primop("xorb"); +var notb = Primop("notb"); + diff --git a/optimize.js b/optimize.js deleted file mode 100644 index 56b58dd..0000000 --- a/optimize.js +++ /dev/null @@ -1,11 +0,0 @@ -var typ = require("./representation.js"); - -/*function simplify(stx) { - switch (stx.exprType) { - case "Application": - - } -}*/ - -//function simplifyUnary(stx) { - -- 2.30.2 From cb1af982ec365542d90396dec8c41bac41665700 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 1 Mar 2014 23:26:11 -0500 Subject: [PATCH 23/43] fixed bugs where I forgot to use new with constructors --- environments.js | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 environments.js diff --git a/environments.js b/environments.js new file mode 100644 index 0000000..291a494 --- /dev/null +++ b/environments.js @@ -0,0 +1,50 @@ +/* + * An environment is just an object that maps identifiers to JLambda expressions + * with a few built-in (a standard Prelude environment) + */ + +// returns the new environment after mutating it +// values = [(identifier, JLambda expression)] + +var errors = require("./errors.js"); +var rep = require("./representation.js"); + + +function extend(env, values) { + for (var i = 0; i < values.length; i++) { + env[values[i][0].val] = values[i][1]; + } + return env; +} + +// creates a new environment initialized with the pairs in values +function makeEnv(name, values) { + var env = {}; + env.name = name; + for (var i = 0; i < values.length; i++) { + var name = values[i][0].val; + var val = values[i][1]; + env[name] = val; + } + return env; +} + +function lookup(name, env) { + var value = env[name]; + if (!value) { + throw errors.UnboundError(name, env.name); + } + return value; +} + +var prelude = makeEnv("prelude", [[new rep.Name("e"), new rep.FloatT(Math.E)], + [new rep.Name("pi"), new rep.FloatT(Math.PI)]]); + +var prelude_types = makeEnv("prelude_types", + [[new rep.Name("e"), new rep.TypeOp("Float", [], false)], + [new rep.Name("pi"), new rep.TypeOp("Float", [], false)]]); + +module.exports = { prelude : prelude, + prelude_types : prelude_types, + lookup : lookup, + extend : extend }; -- 2.30.2 From b2ea5642a283ab8a27cfaca873fd801f3e082709 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 1 Mar 2014 23:54:55 -0500 Subject: [PATCH 24/43] closure conversion first commit --- closure_conversion.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 closure_conversion.js diff --git a/closure_conversion.js b/closure_conversion.js new file mode 100644 index 0000000..5dff49c --- /dev/null +++ b/closure_conversion.js @@ -0,0 +1,17 @@ +/* Takes an AST and converts all of the functions into closures. + * A closure is a pair of two things: an environment and a function + * The closure has the property that all of the free variables of the function + * are in the environment, or an exception is raised because the variable is not bound + * in the current environment. + * A free variable is simply those that are not in the list of formal parameters. + * We start with the global environment and traverse the AST. Every time a new function is entered + * the current environment gets extended with the formal parameters of the function. + * When a let is encountered the current environment also gets extended. + * The algorithm continues for any further function definitions in that branch + * otherwise it just stops for that particular branch and continues with the rest of the AST + * + * Therefore in order to call a closure one must first extract the actual function and then + * call the function with the environment associated with it. + * For the purposes of type checking it does not matter how the function gets called, the environment + * is only used for looking up the types of names. Formal parameters are given type variables. + */ -- 2.30.2 From f1aa8b324eddd960d79f30131ef2f72dc741b655 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 2 Mar 2014 00:00:38 -0500 Subject: [PATCH 25/43] closure conversion, expanded description more --- closure_conversion.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/closure_conversion.js b/closure_conversion.js index 5dff49c..bcd45cd 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -1,5 +1,8 @@ /* Takes an AST and converts all of the functions into closures. - * A closure is a pair of two things: an environment and a function + * A closure is a triple of: + * a mapping from names to expressions + * a mapping from names to type expressions + * a function (which includes the formal parameters and the body) * The closure has the property that all of the free variables of the function * are in the environment, or an exception is raised because the variable is not bound * in the current environment. @@ -15,3 +18,8 @@ * For the purposes of type checking it does not matter how the function gets called, the environment * is only used for looking up the types of names. Formal parameters are given type variables. */ + +var rep = require("./representation.js"); +var env = require("./environments.js"); + +function convert -- 2.30.2 From 0661a055c4d2bd8d90fd2efacc141130d9c0183e Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 8 Mar 2014 12:47:29 -0500 Subject: [PATCH 26/43] calculate the free variables of a function (not done yet) --- closure_conversion.js | 76 ++++++++++++++++++++++++++++++++++++++++++- tools.js | 7 +++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/closure_conversion.js b/closure_conversion.js index bcd45cd..c102cc2 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -21,5 +21,79 @@ var rep = require("./representation.js"); var env = require("./environments.js"); +var errors = require("./errors.js"); +var parser = require("./parse.js"); +var pprint = require("./pprint.js"); +var tool = require("./tools.js"); -function convert +/*function convert(stx, cur_types, cur_exprs) { + switch (stx.exprType) { + case "If": + case "Definition": + case "Name": + case "Application": + case "Function": + case "Let": + default: + return stx; + } +}*/ + +function fvs(stx) { + /*if (stx.exprType !== "Function" && + stx.exprType !== "Let") { + throw errors.JInternalError( + ["Tried to calculate the free variables of", + "something that was not a function or let.\n", + "That something was a: " + stx.exprType +"\n"].reduce( + function (a,b) { + return a+" "+b + }, "")); + }*/ + + switch (stx.exprType) { + case "Integer": + return []; + case "Float": + return []; + case "String": + return []; + case "Function": + return []; + case "Nil": + return []; + case "List": + return []; + case "Bool": + return []; + case "FunctionDefinition": + return []; + case "Let": + return stx.pairs.map(fvs); + case "Unary": + return fvs(stx.val); + case "Definition": + return [fvs(stx.val)]; + case "Application": + var vs = fvs(stx.p); + var f_fvs = fvs(stx.func); + return [].concat.apply([], [vs, f_fvs]); + case "If": + if (stx.elseexp) { + var cond_fvs = fvs(stx.condition); + var then_fvs = fvs(stx.thenexp); + var else_fvs = fvs(stx.elseexp); + return [cond_fvs, then_fvs, else_fvs]; + } + else { + return [fvs(stx.condition)] + [fvs(stx.thenexp)]; + } + case "Name": + return stx.ident; + } +} + +var ast = parser.parse("(^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rt))")[0]; +console.log(pprint.pprint(ast)); +console.log(tool.unique(fvs(ast))); +//console.log(ast); diff --git a/tools.js b/tools.js index 564fec2..0b4db5c 100644 --- a/tools.js +++ b/tools.js @@ -106,6 +106,10 @@ function groupOps(ops) { return groupBy(eq, ops.sort()); } +function unique(ops) { + return groupOps(ops).map(function(x) { return x[0]; }); +} + function find(f, haystack) { for(var i = 0; i < haystack.length; i++) { if (f(haystack[i])) @@ -163,4 +167,5 @@ module.exports = {compose : compose, len : len, groupOps : groupOps, opMatch : operatorMatch, - dict: dict} + dict: dict, + unique : unique}; -- 2.30.2 From b890053d20881dca044c33fc3db728f86b5f92ce Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 9 Mar 2014 13:50:04 -0400 Subject: [PATCH 27/43] readability fixes, a few bug fixes, can calculate free variables of let blocks, environments not being passed yet or closures created --- cexps.js | 2 +- closure_conversion.js | 94 ++++++++++++++++++++++--------------------- desugar.js | 2 +- environments.js | 13 ++++-- errors.js | 10 ++++- evaluate.js | 18 +++++++++ parse.js | 49 +++++++++++++--------- representation.js | 33 +++++++++------ tokenize.js | 68 ++++++++++++++++++++----------- tools.js | 40 ++++++++++++++++-- typecheck.js | 19 +++++++++ 11 files changed, 237 insertions(+), 111 deletions(-) create mode 100644 evaluate.js diff --git a/cexps.js b/cexps.js index 17b484a..b7726d0 100644 --- a/cexps.js +++ b/cexps.js @@ -40,7 +40,7 @@ function offset(i, v, w, next_cexp) { offset.prototype = cexp; function app(k, vs) { - this.k =k; + this.k = k; this.vs = vs; return this; } diff --git a/closure_conversion.js b/closure_conversion.js index c102cc2..eeb59c0 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -8,8 +8,8 @@ * in the current environment. * A free variable is simply those that are not in the list of formal parameters. * We start with the global environment and traverse the AST. Every time a new function is entered - * the current environment gets extended with the formal parameters of the function. - * When a let is encountered the current environment also gets extended. + * the current environment gets $.extended with the formal parameters of the function. + * When a let is encountered the current environment also gets $.extended. * The algorithm continues for any further function definitions in that branch * otherwise it just stops for that particular branch and continues with the rest of the AST * @@ -24,33 +24,11 @@ var env = require("./environments.js"); var errors = require("./errors.js"); var parser = require("./parse.js"); var pprint = require("./pprint.js"); -var tool = require("./tools.js"); +var $ = require("./tools.js"); -/*function convert(stx, cur_types, cur_exprs) { - switch (stx.exprType) { - case "If": - case "Definition": - case "Name": - case "Application": - case "Function": - case "Let": - default: - return stx; - } -}*/ - -function fvs(stx) { - /*if (stx.exprType !== "Function" && - stx.exprType !== "Let") { - throw errors.JInternalError( - ["Tried to calculate the free variables of", - "something that was not a function or let.\n", - "That something was a: " + stx.exprType +"\n"].reduce( - function (a,b) { - return a+" "+b - }, "")); - }*/ +var notEmpty = $.compose($.not, $.eq([])); +function fvs_helper(stx) { switch (stx.exprType) { case "Integer": return []; @@ -62,38 +40,62 @@ function fvs(stx) { return []; case "Nil": return []; - case "List": - return []; case "Bool": return []; - case "FunctionDefinition": - return []; case "Let": - return stx.pairs.map(fvs); + return []; case "Unary": - return fvs(stx.val); + return $.flatten([stx.op.ident, fvs_helper(stx.val)]); case "Definition": - return [fvs(stx.val)]; + return $.flatten(fvs_helper(stx.val)); case "Application": - var vs = fvs(stx.p); - var f_fvs = fvs(stx.func); - return [].concat.apply([], [vs, f_fvs]); + var vs = $.flatten(fvs_helper(stx.p)); + var f_fvs = $.flatten(fvs_helper(stx.func)); + return $.flatten([vs, f_fvs]); case "If": if (stx.elseexp) { - var cond_fvs = fvs(stx.condition); - var then_fvs = fvs(stx.thenexp); - var else_fvs = fvs(stx.elseexp); - return [cond_fvs, then_fvs, else_fvs]; + var cond_fvs = fvs_helper(stx.condition); + var then_fvs = fvs_helper(stx.thenexp); + var else_fvs = fvs_helper(stx.elseexp); + return $.flatten([cond_fvs, then_fvs, else_fvs]); } else { - return [fvs(stx.condition)] + [fvs(stx.thenexp)]; + return $.flatten([fvs_helper(stx.condition), fvs_helper(stx.thenexp)]); } case "Name": return stx.ident; } } -var ast = parser.parse("(^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rt))")[0]; -console.log(pprint.pprint(ast)); -console.log(tool.unique(fvs(ast))); -//console.log(ast); +function fvs(stx) { + if (stx.exprType !== "Function" && + stx.exprType !== "Let") { + throw errors.JInternalError( + ["Tried to calculate the free variables of", + "something that was not a function or let.\n", + "That something was a: " + stx.exprType +"\n"].reduce( + function (a,b) { + return a+" "+b; + }, "")); + } + var variables; + switch (stx.exprType) { + case "Let": + var bound_vars = stx.pairs.map( + function (stx) { + return stx.ident.ident; + }); + var let_fvs = stx.pairs.map(fvs_helper); + var body_fvs = fvs_helper(stx.body); + variables = $.flatten(let_fvs); + $.extend(variables, $.flatten(body_fvs)); + } + return $.difference($.unique($.flatten(variables)), bound_vars); + +} + +//var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rt))")[0]; +//var ast = parser.parse("let { a = let { b = let {dsdfgf = sddd } fdgfg } gggggg } t")[0]; +//console.log(pprint.pprint(ast)); +var ast = parser.parse("let { a = 12 b = (a + t) } (a + b * d)")[0]; +console.log(fvs(ast)); diff --git a/desugar.js b/desugar.js index 2a1d19e..daef9eb 100644 --- a/desugar.js +++ b/desugar.js @@ -32,7 +32,7 @@ function desugarLet(stx) { } function desugar(stx) { - switch (stx.exprType) { + switch (stx.exprType) { case "If": if (stx.elseexp) return new typ.If(desugar(stx.condition), desugar(stx.thenexp), desugar(stx.elseexp)); diff --git a/environments.js b/environments.js index 291a494..51ee39e 100644 --- a/environments.js +++ b/environments.js @@ -11,10 +11,15 @@ var rep = require("./representation.js"); function extend(env, values) { - for (var i = 0; i < values.length; i++) { - env[values[i][0].val] = values[i][1]; + var new_env = {}; + var env_keys = Object.keys(env); + for (var i = 0; i < env_keys.length; i++) { + new_env[env_keys[i]] = env[env_keys[i]]; } - return env; + for (i = 0; i < values.length; i++) { + new_env[values[i][0].val] = values[i][1]; + } + return new_env; } // creates a new environment initialized with the pairs in values @@ -22,7 +27,7 @@ function makeEnv(name, values) { var env = {}; env.name = name; for (var i = 0; i < values.length; i++) { - var name = values[i][0].val; + name = values[i][0].val; var val = values[i][1]; env[name] = val; } diff --git a/errors.js b/errors.js index 05a1dab..b7855f0 100644 --- a/errors.js +++ b/errors.js @@ -26,6 +26,14 @@ function JTypeError(linenum, charnum, token, message) { return this; } +function JInternalError(message) { + this.errormessage = message; + console.log(message); + return this; +} + module.exports = {JSyntaxError : JSyntaxError, - JTypeError : JTypeError}; + JTypeError : JTypeError, + JInternalError : JInternalError + }; diff --git a/evaluate.js b/evaluate.js new file mode 100644 index 0000000..c87286e --- /dev/null +++ b/evaluate.js @@ -0,0 +1,18 @@ +var parse = require("./parse.js").parse; + +function isAtom(x) { + return stx.exprType != "List"; +} + +function evaluate(exp, env) { + if (isAtom(exp)) { + switch (exp.exprType) { + case "Function": + return evaluate(invoke(exp, env), env); + case "Name": + return lookup(expr.val); + default: + return expr.val; + } + } +} diff --git a/parse.js b/parse.js index 903ce75..2f52e4f 100755 --- a/parse.js +++ b/parse.js @@ -21,7 +21,7 @@ function snd(ts) { /*Checks if the next token is not followed by any of ``checks'' */ function notFollowedBy(tokens, checks, linenum, charnum) { if (!fst(tokens)) { - throw error.JSyntaxError(0,0,"unexpected end of source") + throw error.JSyntaxError(0,0,"unexpected end of source"); } var nextT = fst(tokens)[0]; if (checks.some(function (x) { @@ -71,7 +71,7 @@ function parseMany(parse, exprType, valid, tokens, charnum, linenum) { if (!exprType(fst(results).exprType)) break; if (fst(tokens)) - current = fst(tokens)[0] + current = fst(tokens)[0]; else throw error.JSyntaxError(charnum, linenum, "Unexpected end of source"); if (tokens.length <= 1) @@ -115,15 +115,16 @@ function parseBetween(exprType, between, tokens, charnum, linenum) { } function parseList(tokens) { + var xs; if (fst(tokens)[0] === "right_square") { - var xs = []; + xs = []; } else if (fst(tokens)[0] === "comma") { tokens.pop(); - var xs = []; + xs = []; } else { - var xs = parseBetween(function (x) { return true; }, "comma", tokens, fst(tokens)[3], fst(tokens)[2]); + xs = parseBetween(function (x) { return true; }, "comma", tokens, fst(tokens)[3], fst(tokens)[2]); } if (!fst(tokens) || fst(tokens)[0] !== "right_square") { throw error.JSyntaxError(fst(tokens)[3], @@ -137,16 +138,17 @@ function parseList(tokens) { function parseDefFunction(tokens) { var fname = parse(tokens); + var parameters; if (fname.exprType != "Name") { throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], "Expected an identifier in function definition"); } if (fst(tokens)[0] === "right_paren") { - var parameters = []; + parameters = []; } else { - var parameters = parseMany(parse, + parameters = parseMany(parse, validName, validFormPar, tokens, @@ -209,6 +211,7 @@ function parseLetForm(tokens, linenum, charnum) { function parseLetFunction(tokens, linenum, charnum) { var fname = parse(tokens); + var parameters; if (fname.exprType != "Name") { throw error.JSyntaxError(fst(tokens)[3], @@ -216,10 +219,10 @@ function parseLetFunction(tokens, linenum, charnum) { "Expected an identifier in function definition"); } if (fst(tokens)[0] === "right_paren") { - var parameters = []; + parameters = []; } else { - var parameters = parseMany(parse, + parameters = parseMany(parse, validName, validFormPar, tokens, @@ -235,7 +238,7 @@ function parseLetFunction(tokens, linenum, charnum) { if (fst(tokens)[0] !== "arrow") { throw error.JSyntaxError(fst(tokens)[3], fst(tokens)[2], - "Function parameters in let/def form must be followed by ->") + "Function parameters in let/def form must be followed by ->"); } tokens.pop(); var body = parse(tokens); @@ -435,7 +438,7 @@ function parseLambda(tokens) { fst(tokens)[2], "arrow must follow parameters in lambda, not "+fst(tokens)[0]); } - tokens.pop() + tokens.pop(); var body = parse(tokens); return new typ.FuncT(parameters, body); } @@ -448,8 +451,10 @@ var validOperator = makeChecker(["identifier"]); /* Parses function application (either infix or prefix) */ function computeApp(tokens, charnum, linenum) { var lhs = parse(tokens); + var next; + var result; if (fst(tokens)) - var next = fst(tokens); + next = fst(tokens); else { throw error.JSyntaxError(linenum, charnum, @@ -457,7 +462,7 @@ function computeApp(tokens, charnum, linenum) { } if (typ.OPInfo[next[1]]) { /* it's an infix expression */ - var result = parseInfix(tokens, 1, lhs, linenum, charnum); + result = parseInfix(tokens, 1, lhs, linenum, charnum); if (!fst(tokens) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, charnum, @@ -495,7 +500,7 @@ function computeApp(tokens, charnum, linenum) { */ function parseInfix(tokens, minPrec, lhs, linenum, charnum) { if (!lhs) { - var lhs = parse(tokens); + lhs = parse(tokens); } while (true) { var cur = fst(tokens); @@ -524,8 +529,9 @@ function parseInfix(tokens, minPrec, lhs, linenum, charnum) { function parse(tokens) { var charnum = fst(tokens)[2]; var linenum = fst(tokens)[3]; + var toktype; if (fst(tokens)) { - var toktype = fst(tokens)[0]; + toktype = fst(tokens)[0]; } else { process.exit(code=1); @@ -570,9 +576,9 @@ function parse(tokens) { } } -var istr = fs.readFileSync('/dev/stdin').toString(); +//var istr = fs.readFileSync('/dev/stdin').toString(); function parseFull(tokenized) { - var ast = new Array(); + var ast = []; try { while (tokenized.length > 0) { var parsed = desugarer.desugar(parse(tokenized)); @@ -584,7 +590,14 @@ function parseFull(tokenized) { process.exit(1); } } -console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); + +module.exports = { parse : function(str) { + return parseFull(tokenizer.tokenize(str)); + } + }; + + +//console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); //console.log(tokenizer.tokenize(istr)); //console.log(parseFull(tokenizer.tokenize(istr))); diff --git a/representation.js b/representation.js index 09d456f..ccc708c 100644 --- a/representation.js +++ b/representation.js @@ -168,13 +168,6 @@ function TypeOp(name, params, body) { return this; } -function TypeBinding(name, type) { - this.name = name; - this.type = type; - return this; -} - - //convenience function to construct binary operators //assumes that the identifier refers to the name of a primitive //operation @@ -195,11 +188,22 @@ function makeApp(name, parameters) { } +function makeGensym() { + var n = 0; + return function() { + var x = "G"+n; + n = n + 1; + return x; + }; +} + +var gensym = makeGensym(); + OPInfo = {"+" : [3, "Left"], - "-" : [3, "Left"], - "*" : [4, "Left"], - "/" : [4, "Left"], - "^" : [5, "Right"], + "-" : [3, "Left"], + "*" : [4, "Left"], + "/" : [4, "Left"], + "^" : [5, "Right"], "++" : [3, "Left"], "==" : [2, "Left"], ">" : [2, "Left"], @@ -234,5 +238,8 @@ module.exports = DefFunc : DefFunc, UnaryOp : UnaryOp, Nil : Nil, - LetExp : LetExp - } + LetExp : LetExp, + gensym : gensym, + TypeVar : TypeVar, + TypeOp : TypeOp + }; diff --git a/tokenize.js b/tokenize.js index 5cdd1cc..0db5e72 100755 --- a/tokenize.js +++ b/tokenize.js @@ -136,70 +136,83 @@ function tokenize(tokstream, matchop) { var tokens = []; var charnum = 1; var linenum = 1; + var i, result, lambda, num; while (tokstream) { switch (tokstream[0].charCodeAt()) { + /* falls through */ case 9: // '\t' charnum++; tokens.push(["whitespace", '\t', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 32: // ' ' charnum++; tokens.push(["whitespace", ' ', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 10: // '\n' linenum++; charnum = 1; tokens.push(["whitespace", '\n', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 44: // ',' charnum++; tokens.push(["comma", ",", charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 40: // '(' charnum++; tokens.push(["left_paren", '(', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 41: // ')' charnum++; tokens.push(["right_paren", ')', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 123: // '{' charnum++; tokens.push(["left_brace", '{', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 125: // '}' charnum++; tokens.push(["right_brace", '}', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 91: // '[' charnum++; tokens.push(["left_square", '[', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 93: // ']' charnum++; tokens.push(["right_square", ']', charnum, linenum]); tokstream = tokstream.substr(1); break; + /* falls through */ case 34: // '"' - var result = tokenizeStr(tokstream, charnum, linenum); + result = tokenizeStr(tokstream, charnum, linenum); var str = result[1]; - var i = result[0]; + i = result[0]; tokens.push(str); tokstream = tokstream.substr(i); break; + /* falls through */ case 45: // '-' - var lambda = peek(tokstream, "arrow", "->"); + lambda = peek(tokstream, "arrow", "->"); if (lambda) { tokens.push(lambda); tokstream = tokstream.substr(2); @@ -212,24 +225,27 @@ function tokenize(tokstream, matchop) { break; } + /* falls through */ case 46: // '.' if (isDigit(tokstream[1])) { - var result = tokenizeNum(tokstream, charnum, linenum); - var num = result[1]; - var i = result[0]; - if (num[1] !== NaN) + result = tokenizeNum(tokstream, charnum, linenum); + num = result[1]; + i = result[0]; + if (!isNaN(num[1])) { tokens.push(num); + } tokstream = tokstream.substr(i); break; } + /* falls through */ case 116: // 't' - var result = tokenizeT(tokstream); + result = tokenizeT(tokstream); if (result) { tokens.push(result); tokstream = tokstream.substr(4); // 4 = length of either token break; } - + /* falls through */ case 105: // 'i' var ifexp = peek(tokstream, "ifexp", "if"); if (ifexp) { @@ -244,6 +260,7 @@ function tokenize(tokstream, matchop) { break; } + /* falls through */ case 100: // 'd' var defop = peek(tokstream, "defop", "defop"); if (defop) { @@ -257,22 +274,25 @@ function tokenize(tokstream, matchop) { tokstream = tokstream.substr(3); break; } + /* falls through */ case 101: // e - var result = peek(tokstream, "elsexp", "else"); + result = peek(tokstream, "elsexp", "else"); if (result) { tokens.push(result); tokstream = tokstream.substr(4); break; } + /* falls through */ case 102: // f - var result = peek(tokstream, "falselit", "false"); + result = peek(tokstream, "falselit", "false"); if (result) { tokens.push(result); tokstream = tokstream.substr(5); break; } + /* falls through */ case 108: // l - var lambda = peek(tokstream, "lambda", "lambda"); + lambda = peek(tokstream, "lambda", "lambda"); if (lambda) { tokens.push(lambda); tokstream = tokstream.substr(6); @@ -285,13 +305,15 @@ function tokenize(tokstream, matchop) { break; } + /* falls through */ default: if (isDigit(tokstream[0])) { - var result = tokenizeNum(tokstream, charnum, linenum); - var num = result[1]; - var i = result[0]; - if (num[1] !== NaN) + result = tokenizeNum(tokstream, charnum, linenum); + num = result[1]; + i = result[0]; + if (!isNaN(num[1])) { tokens.push(num); + } tokstream = tokstream.substr(i); break; } @@ -303,12 +325,12 @@ function tokenize(tokstream, matchop) { tokens.push(["identifier", op, charnum, linenum]); } else { - var result = tokenizeIdent(tokstream, matchop, charnum, linenum); - result.map(function(x) { + result = tokenizeIdent(tokstream, matchop, charnum, linenum); + for(var index = 0; index < result.length; index++) { charnum++; - tokens.push(x[1]); - tokstream = tokstream.substr(x[0]); - }); + tokens.push(result[index][1]); + tokstream = tokstream.substr(result[index][0]); + } } } } @@ -328,9 +350,9 @@ function tokenizeHelp(input, matchop) { function tokenizeFull(input) { var matchop = tools.opMatch(operators); - var initialPass = tokenizeHelp(input, matchop).reverse();; + var initialPass = tokenizeHelp(input, matchop).reverse(); for (var i = 0; i < initialPass.length; i++) { - if (initialPass.slice(i, i+7).map(function(x) { return x[0]; }) === + if (initialPass.slice(i, i+7).map(tools.fst) === ["defop", "integer", "identifier", "Left", "Right", "left_paren", "identifier", "identifier", "identifier", "right_paren"]) diff --git a/tools.js b/tools.js index 0b4db5c..9d682a0 100644 --- a/tools.js +++ b/tools.js @@ -85,6 +85,10 @@ function eq(a) { }; } +function fst(xs) { + return xs[0]; +} + function equal(a) { return function(b) { return a === b; @@ -103,11 +107,11 @@ function groupBy(eq, xs) { } function groupOps(ops) { - return groupBy(eq, ops.sort()); + return groupBy(function (x) { return function(y) { return x === y; };}, ops.sort()); } function unique(ops) { - return groupOps(ops).map(function(x) { return x[0]; }); + return groupOps(ops).map(fst); } function find(f, haystack) { @@ -126,13 +130,34 @@ function dict(pairs) { return o; } +function flatten(xs) { + if (!(xs instanceof Array)) + return xs; + if (xs.every(function (x) { return !(x instanceof Array); })) + return xs; + return [].concat.apply([], xs); +} + +function extend(xs, ys) { + xs.push.apply(xs, ys); + return xs; +} + +function difference(xs, ys) { + var difflist = groupOps(extend(xs, ys)); + return difflist.filter(function (group) { + if (group.length > 1) { + return false; + } + return true; + }).map(fst); +} /* * Problem: * >> > >>^ <- longest one must be matched * regex? */ - RegExp.escape= function(s) { return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); }; @@ -151,6 +176,8 @@ function operatorMatch(ops) { return false; }; } + + /* var print = console.log; @@ -168,4 +195,9 @@ module.exports = {compose : compose, groupOps : groupOps, opMatch : operatorMatch, dict: dict, - unique : unique}; + unique : unique, + fst : fst, + eq: eq, + extend : extend, + flatten : flatten, + difference : difference} diff --git a/typecheck.js b/typecheck.js index e69de29..7c198c6 100644 --- a/typecheck.js +++ b/typecheck.js @@ -0,0 +1,19 @@ +/* + * Typecheck an AST with a given environment + * the environment maps variables to types + * a variable can either be bound or free + * when we say a variable is free that means that it is either + * unbound (which causes an exception to be raised immediately) + * or it is bound in the outer scope + * + * So the AST must first be converted to a form where each function body is tied + * to an environment mapping identifiers to types + */ + +var rep = require("./representation.js"); +var env = require("./environments.js"); + +var TypeOp = rep.TypeOp; +var TypeVar = rep.TypeVar; + + -- 2.30.2 From 02baaa019241b4d6093a46c5702798d3a3393221 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 9 Mar 2014 16:10:44 -0400 Subject: [PATCH 28/43] some minor cosmetic fixes --- cexps.js | 8 ++++---- closure_conversion.js | 1 + pprint.js | 2 +- representation.js | 34 +++++++++++++++++----------------- tools.js | 8 ++++---- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/cexps.js b/cexps.js index b7726d0..8a41373 100644 --- a/cexps.js +++ b/cexps.js @@ -8,7 +8,7 @@ var cexp = { type : "cexp" -} +}; function record(values_accesspaths, w, @@ -78,7 +78,7 @@ function accessPath(offp, selp) { var primoptype = { type : "primop", equal : function(pOp) { - return this.name === pOp.name + return this.name === pOp.name; } }; @@ -86,7 +86,7 @@ function Primop(name) { function ptype() { this.name = name; return this; - }; + } ptype.prototype = primoptype; return new ptype(); } @@ -113,7 +113,7 @@ var makeref = Primop("makeref"); var makerefunboxed = Primop("makerefunboxed"); var alength = Primop("alength"); var slength = Primop("slength"); -var gethdlr = Primop("gethdlr") +var gethdlr = Primop("gethdlr"); var sethdlr = Primop("sethdlr"); var boxed = Primop("boxed"); var fadd = Primop("fadd"); diff --git a/closure_conversion.js b/closure_conversion.js index eeb59c0..44c3a06 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -62,6 +62,7 @@ function fvs_helper(stx) { else { return $.flatten([fvs_helper(stx.condition), fvs_helper(stx.thenexp)]); } + break; case "Name": return stx.ident; } diff --git a/pprint.js b/pprint.js index 19a94a8..023b1f8 100644 --- a/pprint.js +++ b/pprint.js @@ -58,7 +58,7 @@ function pprint(expr) { else if (expr.exprType === "Let") return "let {" + expr.pairs.map( function (v) { - return pprint(v) + return pprint(v); }).join(" ; ") + "} in " + pprint(expr.body); } diff --git a/representation.js b/representation.js index ccc708c..9e23498 100644 --- a/representation.js +++ b/representation.js @@ -200,10 +200,10 @@ function makeGensym() { var gensym = makeGensym(); OPInfo = {"+" : [3, "Left"], - "-" : [3, "Left"], - "*" : [4, "Left"], - "/" : [4, "Left"], - "^" : [5, "Right"], + "-" : [3, "Left"], + "*" : [4, "Left"], + "/" : [4, "Left"], + "^" : [5, "Right"], "++" : [3, "Left"], "==" : [2, "Left"], ">" : [2, "Left"], @@ -219,22 +219,22 @@ OPInfo = {"+" : [3, "Left"], ">>=" : [1, "Left"], "<$>" : [1, "Left"], "." : [1, "Left"], - "," : [1, "Left"]} + "," : [1, "Left"]}; module.exports = { IntT : IntT, - FloatT : FloatT, - StrT : StrT, - BoolT : BoolT, - ListT : ListT, - FuncT : FuncT, - App : App, - Name : Name, - Def : Def, - OpT : OpT, - OPInfo : OPInfo, - makeApp : makeApp, - If : If, + FloatT : FloatT, + StrT : StrT, + BoolT : BoolT, + ListT : ListT, + FuncT : FuncT, + App : App, + Name : Name, + Def : Def, + OpT : OpT, + OPInfo : OPInfo, + makeApp : makeApp, + If : If, DefFunc : DefFunc, UnaryOp : UnaryOp, Nil : Nil, diff --git a/tools.js b/tools.js index 9d682a0..8be3018 100644 --- a/tools.js +++ b/tools.js @@ -123,7 +123,7 @@ function find(f, haystack) { } function dict(pairs) { - var o = new Object(); + var o = {}; pairs.map(function(p) { o[p[0]] = p[1]; }); @@ -188,8 +188,8 @@ print(matcher(">=")); */ module.exports = {compose : compose, - not : not, - on : on, + not : not, + on : on, maxBy : maxBy, len : len, groupOps : groupOps, @@ -200,4 +200,4 @@ module.exports = {compose : compose, eq: eq, extend : extend, flatten : flatten, - difference : difference} + difference : difference}; -- 2.30.2 From 1e9e5c560710403d0d4a9086817211c554ebfc74 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 9 Mar 2014 16:41:28 -0400 Subject: [PATCH 29/43] removing old code --- evaluate.js | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 evaluate.js diff --git a/evaluate.js b/evaluate.js deleted file mode 100644 index c87286e..0000000 --- a/evaluate.js +++ /dev/null @@ -1,18 +0,0 @@ -var parse = require("./parse.js").parse; - -function isAtom(x) { - return stx.exprType != "List"; -} - -function evaluate(exp, env) { - if (isAtom(exp)) { - switch (exp.exprType) { - case "Function": - return evaluate(invoke(exp, env), env); - case "Name": - return lookup(expr.val); - default: - return expr.val; - } - } -} -- 2.30.2 From d5424b9c4b1d63928496fb8274936672aa43ec54 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Mon, 10 Mar 2014 14:58:49 -0400 Subject: [PATCH 30/43] closure stuff, desugar to curried functions --- closure_conversion.js | 18 +++++++++++++----- desugar.js | 17 ++++++++++++++--- representation.js | 11 ++++++++++- tools.js | 12 +++++++++++- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/closure_conversion.js b/closure_conversion.js index 44c3a06..55a8b25 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -79,7 +79,8 @@ function fvs(stx) { return a+" "+b; }, "")); } - var variables; + var variables, free_variables; + switch (stx.exprType) { case "Let": var bound_vars = stx.pairs.map( @@ -90,13 +91,20 @@ function fvs(stx) { var body_fvs = fvs_helper(stx.body); variables = $.flatten(let_fvs); $.extend(variables, $.flatten(body_fvs)); - } - return $.difference($.unique($.flatten(variables)), bound_vars); + free_variables = $.difference($.unique($.flatten(variables)), bound_vars); + break; + case "Function": + + } } //var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rt))")[0]; //var ast = parser.parse("let { a = let { b = let {dsdfgf = sddd } fdgfg } gggggg } t")[0]; //console.log(pprint.pprint(ast)); -var ast = parser.parse("let { a = 12 b = (a + t) } (a + b * d)")[0]; -console.log(fvs(ast)); +//var ast = parser.parse("let { a = 12 b = (a + t) } (a + b * d)")[0]; +//console.log(fvs(ast)); +//var ast = parser.parse("((lambda a b c -> (+ a b c)) 2 3.0 4)"); +var ast = parser.parse("def (f a b c) 12")[0]; +//console.log(JSON.stringify(ast, null, 4)); +console.log(pprint.pprint(ast)); diff --git a/desugar.js b/desugar.js index daef9eb..e9e1feb 100644 --- a/desugar.js +++ b/desugar.js @@ -5,6 +5,7 @@ */ var typ = require("./representation.js"); +var $ = require("./tools.js"); // Lists get desugared to nested function calls // i.e. (cons (cons (cons ...))) @@ -21,8 +22,18 @@ function desugarList(lst) { function desugarDefFunc(def) { return new typ.Def(def.ident, - new typ.FuncT(desugar(def.params), - desugar(def.body))); + curryFunc(def.params, + def.body)); +} + +function curryFunc(ps, body) { + if ($.empty(ps)) { + return desugar(body); + } + else { + return new typ.FuncT(desugar($.fst(ps)), + curryFunc($.rst(ps), body)); + } } @@ -53,7 +64,7 @@ function desugar(stx) { return new typ.App(desugar(stx.func), desugar(stx.p)); return new typ.App(stx.func); case "Function": - return new typ.FuncT(stx.p, desugar(stx.body)); + return curryFunc(stx.p, stx.body); case "List": return desugarList(stx); case "Bool": diff --git a/representation.js b/representation.js index 9e23498..e357c9e 100644 --- a/representation.js +++ b/representation.js @@ -20,6 +20,14 @@ var Expression = { } }; +function Closure(bound_vars, free_vars, body, env) { + this.bound_vars = bound_vars; + this.free_vars = free_vars; + this.body = body; + this.env = env; + return this; +} + function LetExp(pairs, body) { if (!pairs.every(function(x) { return (x.exprType === "Definition" || @@ -241,5 +249,6 @@ module.exports = LetExp : LetExp, gensym : gensym, TypeVar : TypeVar, - TypeOp : TypeOp + TypeOp : TypeOp, + Closure : Closure }; diff --git a/tools.js b/tools.js index 8be3018..a2b2084 100644 --- a/tools.js +++ b/tools.js @@ -1,3 +1,7 @@ +function empty(xs) { + return xs.length === 0; +} + function identity(a) { return a; } @@ -89,6 +93,10 @@ function fst(xs) { return xs[0]; } +function rst(xs) { + return xs.slice(1,xs.length); +} + function equal(a) { return function(b) { return a === b; @@ -197,7 +205,9 @@ module.exports = {compose : compose, dict: dict, unique : unique, fst : fst, + rst : rst, eq: eq, extend : extend, flatten : flatten, - difference : difference}; + difference : difference, + empty : empty }; -- 2.30.2 From 27eee4bc10229d01e32aeaa40972280096065eef Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Tue, 11 Mar 2014 22:52:20 -0400 Subject: [PATCH 31/43] free variable calculation and naive closure creation works --- closure_conversion.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/closure_conversion.js b/closure_conversion.js index 55a8b25..023f6d9 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -64,7 +64,7 @@ function fvs_helper(stx) { } break; case "Name": - return stx.ident; + return [stx.ident]; } } @@ -79,11 +79,11 @@ function fvs(stx) { return a+" "+b; }, "")); } - var variables, free_variables; + var variables, free_variables, bound_vars; switch (stx.exprType) { case "Let": - var bound_vars = stx.pairs.map( + bound_vars = stx.pairs.map( function (stx) { return stx.ident.ident; }); @@ -91,20 +91,23 @@ function fvs(stx) { var body_fvs = fvs_helper(stx.body); variables = $.flatten(let_fvs); $.extend(variables, $.flatten(body_fvs)); - free_variables = $.difference($.unique($.flatten(variables)), bound_vars); break; case "Function": - - + bound_vars = [stx.p.ident,]; + variables = fvs_helper(stx.body); } + free_variables = $.difference($.unique(variables, bound_vars)); + return new rep.Closure(bound_vars, free_variables, stx, []); } -//var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rt))")[0]; +//var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rtcccc))")[0]; //var ast = parser.parse("let { a = let { b = let {dsdfgf = sddd } fdgfg } gggggg } t")[0]; //console.log(pprint.pprint(ast)); //var ast = parser.parse("let { a = 12 b = (a + t) } (a + b * d)")[0]; //console.log(fvs(ast)); //var ast = parser.parse("((lambda a b c -> (+ a b c)) 2 3.0 4)"); -var ast = parser.parse("def (f a b c) 12")[0]; +//var ast = parser.parse("def (f a b c) 12")[0]; //console.log(JSON.stringify(ast, null, 4)); -console.log(pprint.pprint(ast)); +//console.log(pprint.pprint(ast)); +var ast = parser.parse("(lambda a -> (pi*e+c-a))")[0]; +console.log(fvs(ast)); -- 2.30.2 From cb0dbd0948277efb1e1fe618135b0ae2a5a3096a Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Thu, 13 Mar 2014 01:48:47 -0400 Subject: [PATCH 32/43] fixed bugs in defop parsing, both in the tokenizer and parser (was doing something stupid with array comparisons) --- parse.js | 8 ++++---- tokenize.js | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/parse.js b/parse.js index 2f52e4f..1adb19b 100755 --- a/parse.js +++ b/parse.js @@ -350,7 +350,7 @@ function parseDefOp(tokens, linenum, charnum) { } tokens.pop(); - if (false) { + if (fst(tokens)[1] !== "Left" && fst(tokens)[1] !== "Right") { throw error.JSyntaxError(linenum, charnum, "defop must be followed by precedence and then either Left or Right"); @@ -368,7 +368,7 @@ function parseDefOp(tokens, linenum, charnum) { }))) { throw error.JSyntaxError(linenum, charnum, - "defop must be surrounded by exactly 3 identifier"); + "defop must be surrounded by exactly 3 identifiers"); } var pattern = tokens.slice(tokens.length-3, tokens.length); @@ -576,7 +576,7 @@ function parse(tokens) { } } -//var istr = fs.readFileSync('/dev/stdin').toString(); +var istr = fs.readFileSync('/dev/stdin').toString(); function parseFull(tokenized) { var ast = []; try { @@ -597,7 +597,7 @@ module.exports = { parse : function(str) { }; -//console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); +console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); //console.log(tokenizer.tokenize(istr)); //console.log(parseFull(tokenizer.tokenize(istr))); diff --git a/tokenize.js b/tokenize.js index 0db5e72..9397b47 100755 --- a/tokenize.js +++ b/tokenize.js @@ -347,17 +347,21 @@ function tokenizeHelp(input, matchop) { process.exit(1); } } +var defop_pattern = ["defop", "integer", "identifier", + "left_paren", "identifier", + "identifier", "identifier", "right_paren"]; function tokenizeFull(input) { var matchop = tools.opMatch(operators); var initialPass = tokenizeHelp(input, matchop).reverse(); for (var i = 0; i < initialPass.length; i++) { - if (initialPass.slice(i, i+7).map(tools.fst) === - ["defop", "integer", "identifier", "Left", "Right", - "left_paren", "identifier", "identifier", "identifier", - "right_paren"]) + if (initialPass.slice(i, i+8).map(tools.fst).every( + function(x, i) { + return x === defop_pattern[i]; + })) { rep.OPInfo[initialPass[i+5][1]] = [parseInt(initialPass[i+1][1], 10), initialPass[i+2][1]]; + } } operators = Object.keys(rep.OPInfo); -- 2.30.2 From d366da152a9eef770ca99bff610046119a74f711 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Thu, 13 Mar 2014 03:29:24 -0400 Subject: [PATCH 33/43] full closure conversion sort of works, doesn't map variables on to types yet, will have to add that --- closure_conversion.js | 96 +++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 27 deletions(-) diff --git a/closure_conversion.js b/closure_conversion.js index 023f6d9..765565d 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -1,17 +1,12 @@ /* Takes an AST and converts all of the functions into closures. * A closure is a triple of: - * a mapping from names to expressions - * a mapping from names to type expressions - * a function (which includes the formal parameters and the body) - * The closure has the property that all of the free variables of the function + * the bound variables in a function or let + * the free variables in a function or let + * a function body or let body and bound values + * The closure has the property that all of the free variables of the function or let * are in the environment, or an exception is raised because the variable is not bound * in the current environment. - * A free variable is simply those that are not in the list of formal parameters. - * We start with the global environment and traverse the AST. Every time a new function is entered - * the current environment gets $.extended with the formal parameters of the function. - * When a let is encountered the current environment also gets $.extended. - * The algorithm continues for any further function definitions in that branch - * otherwise it just stops for that particular branch and continues with the rest of the AST + * A free variable is simply those that are not in the list of formal parameters or bound variables if it is a let * * Therefore in order to call a closure one must first extract the actual function and then * call the function with the environment associated with it. @@ -28,7 +23,7 @@ var $ = require("./tools.js"); var notEmpty = $.compose($.not, $.eq([])); -function fvs_helper(stx) { +function fvs(stx) { switch (stx.exprType) { case "Integer": return []; @@ -45,22 +40,22 @@ function fvs_helper(stx) { case "Let": return []; case "Unary": - return $.flatten([stx.op.ident, fvs_helper(stx.val)]); + return $.flatten([stx.op.ident, fvs(stx.val)]); case "Definition": - return $.flatten(fvs_helper(stx.val)); + return $.flatten(fvs(stx.val)); case "Application": - var vs = $.flatten(fvs_helper(stx.p)); - var f_fvs = $.flatten(fvs_helper(stx.func)); + var vs = $.flatten(fvs(stx.p)); + var f_fvs = $.flatten(fvs(stx.func)); return $.flatten([vs, f_fvs]); case "If": if (stx.elseexp) { - var cond_fvs = fvs_helper(stx.condition); - var then_fvs = fvs_helper(stx.thenexp); - var else_fvs = fvs_helper(stx.elseexp); + var cond_fvs = fvs(stx.condition); + var then_fvs = fvs(stx.thenexp); + var else_fvs = fvs(stx.elseexp); return $.flatten([cond_fvs, then_fvs, else_fvs]); } else { - return $.flatten([fvs_helper(stx.condition), fvs_helper(stx.thenexp)]); + return $.flatten([fvs(stx.condition), fvs(stx.thenexp)]); } break; case "Name": @@ -68,7 +63,12 @@ function fvs_helper(stx) { } } -function fvs(stx) { +function closure_convert(stx) { + /* Takes a stx object that is either + * a lambda + * a let + * and returns a closure wrapped around that stx object + */ if (stx.exprType !== "Function" && stx.exprType !== "Let") { throw errors.JInternalError( @@ -79,7 +79,7 @@ function fvs(stx) { return a+" "+b; }, "")); } - var variables, free_variables, bound_vars; + var variables, free_variables, bound_vars, stx_type; switch (stx.exprType) { case "Let": @@ -87,19 +87,60 @@ function fvs(stx) { function (stx) { return stx.ident.ident; }); - var let_fvs = stx.pairs.map(fvs_helper); - var body_fvs = fvs_helper(stx.body); + var let_fvs = stx.pairs.map(fvs); + var body_fvs = fvs(stx.body); variables = $.flatten(let_fvs); $.extend(variables, $.flatten(body_fvs)); break; case "Function": bound_vars = [stx.p.ident,]; - variables = fvs_helper(stx.body); + variables = fvs(stx.body); + break; } - free_variables = $.difference($.unique(variables, bound_vars)); + free_variables = $.difference($.unique(variables), bound_vars); return new rep.Closure(bound_vars, free_variables, stx, []); } +function closure_convert_all(stx) { + var closure; + switch (stx.exprType) { + case "Let": + closure = closure_convert(stx); + closure.body.pairs = closure.body.pairs.map(closure_convert_all); + closure.body = closure_convert_all(closure.body.body); + return closure; + case "Function": + closure = closure_convert(stx); + closure.body.body = closure_convert_all(closure.body.body); + return closure; + case "Unary": + stx.val = closure_convert_all(stx.val); + return stx; + case "Application": + stx.func = closure_convert_all(stx.func); + stx.p = closure_convert_all(stx.p); + return stx; + case "If": + if (stx.elseexp) { + stx.condition = closure_convert_all(stx.condition); + stx.thenexp = closure_convert_all(stx.thenexp); + stx.elseexp = closure_convert_all(stx.elseexp); + return stx; + } + else { + stx.condition = closure_convert_all(stx.condition); + stx.thenexp = closure_convert_all(stx.thenexp); + return stx; + } + case "Definition": + stx.val = closure_convert_all(stx.val); + return stx; + default: + return stx; + } +} + + //var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rtcccc))")[0]; //var ast = parser.parse("let { a = let { b = let {dsdfgf = sddd } fdgfg } gggggg } t")[0]; //console.log(pprint.pprint(ast)); @@ -109,5 +150,6 @@ function fvs(stx) { //var ast = parser.parse("def (f a b c) 12")[0]; //console.log(JSON.stringify(ast, null, 4)); //console.log(pprint.pprint(ast)); -var ast = parser.parse("(lambda a -> (pi*e+c-a))")[0]; -console.log(fvs(ast)); +var ast = parser.parse("(lambda a -> if true then ((lambda b -> (+ a b)) q))")[0]; +console.log(JSON.stringify(closure_convert_all(ast), null, 4)); +//console.log(pprint.pprint(ast)); -- 2.30.2 From 13201d8662129cf42551d8bf34f674d25619dc78 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 16:23:13 -0400 Subject: [PATCH 34/43] switching to underscore.js for most functionality --- closure_conversion.js | 46 +++++------ desugar.js | 8 +- parse.js | 20 ++--- patterns.js | 8 ++ representation.js | 3 +- tokenize.js | 24 +++--- tools.js | 182 +++++++----------------------------------- 7 files changed, 87 insertions(+), 204 deletions(-) create mode 100644 patterns.js diff --git a/closure_conversion.js b/closure_conversion.js index 765565d..d8efe07 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -20,8 +20,9 @@ var errors = require("./errors.js"); var parser = require("./parse.js"); var pprint = require("./pprint.js"); var $ = require("./tools.js"); +var _ = require("underscore"); -var notEmpty = $.compose($.not, $.eq([])); +var notEmpty = _.compose($.not, _.partial(_.equal, [])); function fvs(stx) { switch (stx.exprType) { @@ -40,22 +41,22 @@ function fvs(stx) { case "Let": return []; case "Unary": - return $.flatten([stx.op.ident, fvs(stx.val)]); + return _.flatten([stx.op.ident, fvs(stx.val)]); case "Definition": - return $.flatten(fvs(stx.val)); + return _.flatten(fvs(stx.val)); case "Application": - var vs = $.flatten(fvs(stx.p)); - var f_fvs = $.flatten(fvs(stx.func)); - return $.flatten([vs, f_fvs]); + var vs = _.flatten(fvs(stx.p)); + var f_fvs = _.flatten(fvs(stx.func)); + return _.flatten([vs, f_fvs]); case "If": if (stx.elseexp) { var cond_fvs = fvs(stx.condition); var then_fvs = fvs(stx.thenexp); var else_fvs = fvs(stx.elseexp); - return $.flatten([cond_fvs, then_fvs, else_fvs]); + return _.flatten([cond_fvs, then_fvs, else_fvs]); } else { - return $.flatten([fvs(stx.condition), fvs(stx.thenexp)]); + return _.flatten([fvs(stx.condition), fvs(stx.thenexp)]); } break; case "Name": @@ -89,15 +90,15 @@ function closure_convert(stx) { }); var let_fvs = stx.pairs.map(fvs); var body_fvs = fvs(stx.body); - variables = $.flatten(let_fvs); - $.extend(variables, $.flatten(body_fvs)); + variables = _.flatten(let_fvs); + $.extend(variables, _.flatten(body_fvs)); break; case "Function": bound_vars = [stx.p.ident,]; variables = fvs(stx.body); break; } - free_variables = $.difference($.unique(variables), bound_vars); + free_variables = _.difference(_.uniq(variables), bound_vars); return new rep.Closure(bound_vars, free_variables, stx, []); } @@ -141,15 +142,14 @@ function closure_convert_all(stx) { } -//var ast = parser.parse("let { c = trtr a = let {tttt = (rtertret^yyyy) } let { dfsdf = let { asdsd = 3434 } gdfgdfg } (45+(asdddy*uyuy)) q = ((lambda x y -> (x+y)) 4 ui) } (^ wat (a+(ar*b*c^twerp+\"sdfdsfsdfsdfsdf\")*rtcccc))")[0]; -//var ast = parser.parse("let { a = let { b = let {dsdfgf = sddd } fdgfg } gggggg } t")[0]; -//console.log(pprint.pprint(ast)); -//var ast = parser.parse("let { a = 12 b = (a + t) } (a + b * d)")[0]; -//console.log(fvs(ast)); -//var ast = parser.parse("((lambda a b c -> (+ a b c)) 2 3.0 4)"); -//var ast = parser.parse("def (f a b c) 12")[0]; -//console.log(JSON.stringify(ast, null, 4)); -//console.log(pprint.pprint(ast)); -var ast = parser.parse("(lambda a -> if true then ((lambda b -> (+ a b)) q))")[0]; -console.log(JSON.stringify(closure_convert_all(ast), null, 4)); -//console.log(pprint.pprint(ast)); +function test(src) { + var ast = parser.parse(src)[0]; + console.log(JSON.stringify(closure_convert_all(ast), null, 4)); +} + +test("(lambda a -> (+ a b c))"); + +module.export = { + test : test, + closureConvert : closure_convert_all +}; diff --git a/desugar.js b/desugar.js index e9e1feb..cf79a35 100644 --- a/desugar.js +++ b/desugar.js @@ -5,7 +5,7 @@ */ var typ = require("./representation.js"); -var $ = require("./tools.js"); +var _ = require("underscore"); // Lists get desugared to nested function calls // i.e. (cons (cons (cons ...))) @@ -27,12 +27,12 @@ function desugarDefFunc(def) { } function curryFunc(ps, body) { - if ($.empty(ps)) { + if (_.isEmpty(ps)) { return desugar(body); } else { - return new typ.FuncT(desugar($.fst(ps)), - curryFunc($.rst(ps), body)); + return new typ.FuncT(desugar(_.first(ps)), + curryFunc(_.rest(ps), body)); } } diff --git a/parse.js b/parse.js index 1adb19b..1d7d4ac 100755 --- a/parse.js +++ b/parse.js @@ -2,7 +2,8 @@ var fs = require("fs"); var typ = require("./representation.js"); -var tool = require("./tools.js"); +var $ = require("./tools.js"); +var _ = require("underscore"); var tokenizer = require("./tokenize.js"); var desugarer = require("./desugar.js"); var pprint = require("./pprint.js"); @@ -166,7 +167,7 @@ function parseDefFunction(tokens) { } validLet = makeChecker(["Definition", "FunctionDefinition"]); -letEnd = tool.compose(tool.not, makeChecker(["right_brace"])); +letEnd = _.compose($.not, makeChecker(["right_brace"])); function parseLetForm(tokens, linenum, charnum) { if (!fst(tokens)) { @@ -444,8 +445,8 @@ function parseLambda(tokens) { } var invalidArguments = ["def", "comma", "right_paren", "right_square", "right_brace", "left_brace", "right_brace"]; -var validArgument = tool.compose(tool.not, makeChecker(invalidArguments)); -var validArgTypes = tool.compose(tool.not, makeChecker(["Definition"])); +var validArgument = _.compose($.not, makeChecker(invalidArguments)); +var validArgTypes = _.compose($.not, makeChecker(["Definition"])); var validOperator = makeChecker(["identifier"]); /* Parses function application (either infix or prefix) */ @@ -576,7 +577,7 @@ function parse(tokens) { } } -var istr = fs.readFileSync('/dev/stdin').toString(); + function parseFull(tokenized) { var ast = []; try { @@ -595,11 +596,4 @@ module.exports = { parse : function(str) { return parseFull(tokenizer.tokenize(str)); } }; - - -console.log(parseFull(tokenizer.tokenize(istr)).map(pprint.pprint).join("\n")); - -//console.log(tokenizer.tokenize(istr)); -//console.log(parseFull(tokenizer.tokenize(istr))); - -//module.exports = {parse : tool.compose(parseFull, tokenizer.tokenize) }; +//var istr = fs.readFileSync('/dev/stdin').toString(); diff --git a/patterns.js b/patterns.js new file mode 100644 index 0000000..3dcf3e9 --- /dev/null +++ b/patterns.js @@ -0,0 +1,8 @@ +/* Lightweight pattern matching for lists of strings + * + */ +var $ = require("./tools.js"); + + + + diff --git a/representation.js b/representation.js index e357c9e..ea4ba57 100644 --- a/representation.js +++ b/representation.js @@ -1,5 +1,3 @@ -var tool = require("./tools.js"); - var Expression = { display : function() { @@ -25,6 +23,7 @@ function Closure(bound_vars, free_vars, body, env) { this.free_vars = free_vars; this.body = body; this.env = env; + this.exprType = "Closure"; return this; } diff --git a/tokenize.js b/tokenize.js index 9397b47..47ffcc4 100755 --- a/tokenize.js +++ b/tokenize.js @@ -1,9 +1,10 @@ #! /usr/bin/node var rep = require("./representation.js"); -var tools = require("./tools.js"); +var $ = require("./tools.js"); var error = require("./errors.js"); var operators = Object.keys(rep.OPInfo); +var _ = require("underscore"); function isDigit(a) { if (!a) @@ -337,25 +338,31 @@ function tokenize(tokstream, matchop) { return tokens; } -function tokenizeHelp(input, matchop) { +function tokenizeHelp(input, matchop, strip_whitespace) { try { return tokenize(input, matchop).reverse().filter(function(x) { - return x[0] !== "whitespace"; + if (strip_whitespace) { + return x[0] !== "whitespace"; + } + else { + return true; + } }); } catch (e) { console.log(e.stxerror()); process.exit(1); } } + var defop_pattern = ["defop", "integer", "identifier", "left_paren", "identifier", "identifier", "identifier", "right_paren"]; function tokenizeFull(input) { - var matchop = tools.opMatch(operators); - var initialPass = tokenizeHelp(input, matchop).reverse(); + var matchop = $.opMatch(operators); + var initialPass = tokenizeHelp(input, _.constant(false), true).reverse(); for (var i = 0; i < initialPass.length; i++) { - if (initialPass.slice(i, i+8).map(tools.fst).every( + if (initialPass.slice(i, i+8).map(_.first).every( function(x, i) { return x === defop_pattern[i]; })) { @@ -365,9 +372,8 @@ function tokenizeFull(input) { } operators = Object.keys(rep.OPInfo); - matchop = tools.opMatch(operators); - return tokenizeHelp(input, matchop); + matchop = $.opMatch(operators); + return tokenizeHelp(input, matchop, true); } - module.exports = {tokenize : tokenizeFull}; diff --git a/tools.js b/tools.js index a2b2084..bccb8b1 100644 --- a/tools.js +++ b/tools.js @@ -1,131 +1,34 @@ -function empty(xs) { - return xs.length === 0; -} - -function identity(a) { - return a; -} +var _ = require("underscore"); -function compose(f, g) { - return function(x) { - return f(g(x)); - }; +function empty(xs) { + return _.size(xs) < 1; } function not(x) { return !x; } -function on(g, f) { - return function(x,y) { - return g(f(x), f(y)); - }; -} - -function maxf(f, a, b) { - if (f(a) >= f(b)) - return a; - return b; -} - -function max(a, b) { - if (a > b) - return 1; - else if (a < b) - return -1; - else - return 0; -} - function min(a, b) { - if (a < b) + if (a < b) { return 1; - else if (a > b) - return -1; - else - return 0; -} - -function maxBy(f, xs) { - if (xs.length < 1) - return false; - return xs.reduce(function(maxval, a) { return maxf(f, maxval, a); }); -} - -function sortBy(f, xs) { - return xs.sort(f); -} - -function len(xs) { - return xs.length; -} - -function takeWhile(f, xs) { - var result = []; - for (var i = 0; i < xs.length; i++) { - if (f(xs[i])) - result.push(xs[i]); - else - break; } - return result; -} - -function dropWhile(f, xs) { - for (i = 0; i < xs.length; i++) { - if (!f(xs[i])) - break; + else if (a > b) { + return -1; } - return xs.slice(i); -} - -function span(f, xs) { - return [takeWhile(f, xs), dropWhile(f, xs)]; -} - -function eq(a) { - return function(b) { - return a[0] === b[0]; - }; -} - -function fst(xs) { - return xs[0]; -} - -function rst(xs) { - return xs.slice(1,xs.length); -} - -function equal(a) { - return function(b) { - return a === b; - }; -} - -function groupBy(eq, xs) { - var groups = []; - var spanned; - while (xs.length > 0) { - spanned = span(eq(xs[0]), xs.slice(1)); - groups.push([xs[0]].concat(spanned[0])); - xs = spanned[1]; + else { + return 0; } - return groups; } function groupOps(ops) { - return groupBy(function (x) { return function(y) { return x === y; };}, ops.sort()); -} - -function unique(ops) { - return groupOps(ops).map(fst); + return _.groupBy(ops.sort(), _.isEqual); } function find(f, haystack) { for(var i = 0; i < haystack.length; i++) { - if (f(haystack[i])) + if (f(haystack[i])) { return i; + } } return false; } @@ -139,10 +42,14 @@ function dict(pairs) { } function flatten(xs) { - if (!(xs instanceof Array)) + if (!(xs instanceof Array)) { return xs; - if (xs.every(function (x) { return !(x instanceof Array); })) + } + if (xs.every(function (x) { + return !(x instanceof Array); + })) { return xs; + } return [].concat.apply([], xs); } @@ -151,21 +58,6 @@ function extend(xs, ys) { return xs; } -function difference(xs, ys) { - var difflist = groupOps(extend(xs, ys)); - return difflist.filter(function (group) { - if (group.length > 1) { - return false; - } - return true; - }).map(fst); -} - -/* - * Problem: - * >> > >>^ <- longest one must be matched - * regex? - */ RegExp.escape= function(s) { return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); }; @@ -178,36 +70,20 @@ function operatorMatch(ops) { var reg = new RegExp(rstring); return function(x) { var matched = reg.exec(x); - if (matched[0]) + if (matched[0]) { return matched[0]; - else + } + else { return false; + } }; } - -/* -var print = console.log; - -var testOps = [">>", ">>&", ">", "aaaaa:", ">="]; - -var matcher = operatorMatch(testOps); -print(matcher(">=")); -*/ - -module.exports = {compose : compose, - not : not, - on : on, - maxBy : maxBy, - len : len, - groupOps : groupOps, - opMatch : operatorMatch, - dict: dict, - unique : unique, - fst : fst, - rst : rst, - eq: eq, - extend : extend, - flatten : flatten, - difference : difference, - empty : empty }; +module.exports = { + not : not, + groupOps : groupOps, + opMatch : operatorMatch, + dict: dict, + extend : extend, + empty : empty, +}; -- 2.30.2 From e11dcc36c81e86da1fd15450051002a3f1b84e23 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 16:33:07 -0400 Subject: [PATCH 35/43] changed example filename to example.jl --- test.jl => example.jl | 3 +++ 1 file changed, 3 insertions(+) rename test.jl => example.jl (96%) diff --git a/test.jl b/example.jl similarity index 96% rename from test.jl rename to example.jl index ae33ebe..9854467 100644 --- a/test.jl +++ b/example.jl @@ -1,3 +1,6 @@ +defop 2 Left (a ## b) + (a - b) + def (f a b) (a ++ b) -- 2.30.2 From 24c29c3d0e4a219f66d8381980f1544cd043021c Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 16:34:06 -0400 Subject: [PATCH 36/43] using javascript version of QuickCheck for testing --- test.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test.js b/test.js index eff6f14..38d7b1a 100755 --- a/test.js +++ b/test.js @@ -1,10 +1,4 @@ #! /usr/bin/node -var p = require("./parse.js"); -var pp = require("./pprint.js"); -var tools = require("./tools.js"); -var parse = tools.compose(pp.pprint, p.parse); -//console.log(parse("((map g [1,2,3]) >> (print 34))")); -//p.parse("((f [1,2,3,4,5]) >> (* 2 3))"); -//p.parse("[1] 45"); -//p.parse("(+ 2 3 4)"); +var parser = require("./parse.js"); +var qc = require("quickcheck"); -- 2.30.2 From 4580a8c509dc390f6e3bbb0003e8f4bf3fba039b Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 16:36:14 -0400 Subject: [PATCH 37/43] removing accidentally commited file that I never needed --- patterns.js | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 patterns.js diff --git a/patterns.js b/patterns.js deleted file mode 100644 index 3dcf3e9..0000000 --- a/patterns.js +++ /dev/null @@ -1,8 +0,0 @@ -/* Lightweight pattern matching for lists of strings - * - */ -var $ = require("./tools.js"); - - - - -- 2.30.2 From 74b2c9b31013c9c5fa1fb25b81c9ff57df141548 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 20:22:23 -0400 Subject: [PATCH 38/43] tests --- cexps.js | 2 +- closure_conversion.js | 2 -- test.js | 72 +++++++++++++++++++++++++++++++++++++++++++ tools.js | 17 ++-------- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/cexps.js b/cexps.js index 8a41373..f996a65 100644 --- a/cexps.js +++ b/cexps.js @@ -102,7 +102,7 @@ var lessoreq = Primop("<="); var greatthan = Primop(">"); var greatoreq = Primop(">="); var bang = Primop("!"); -var subscript = Protoype("subscript"); +var subscript = Primop("subscript"); var ordof = Primop("ordof"); var assign = Primop(":="); var unboxedassign = Primop("unboxedassign"); diff --git a/closure_conversion.js b/closure_conversion.js index d8efe07..34a26d4 100644 --- a/closure_conversion.js +++ b/closure_conversion.js @@ -147,8 +147,6 @@ function test(src) { console.log(JSON.stringify(closure_convert_all(ast), null, 4)); } -test("(lambda a -> (+ a b c))"); - module.export = { test : test, closureConvert : closure_convert_all diff --git a/test.js b/test.js index 38d7b1a..7c254ac 100755 --- a/test.js +++ b/test.js @@ -1,4 +1,76 @@ #! /usr/bin/node var parser = require("./parse.js"); +var cexps = require("./cexps.js"); +var closures = require("./closure_conversion.js"); +var desugar = require("./desugar.js"); +var environments = require("./environments.js"); +var errors = require("./errors.js"); +var tokens = require("./tokenize.js"); +var tools = require("./tools.js"); +var typecheck = require("./typecheck.js"); +var _ = require("underscore"); + var qc = require("quickcheck"); +var assert = require("assert"); + + +/* my own generators */ +function arbArray(gen) { + return qc.arbArray(gen); +} + + +function arbPair(gen) { + return function() { + return [gen(), gen()]; + }; +} + +function arbArrayofPairs() { + return arbArray(function() { + return arbArray(arbPair(qc.arbString)); + }); +} + +function arbPairs() { + return arbArray(arbPair(qc.arbString)); +} + + +/* Tests for misc tools */ +function emptyProp(xs) { + return (tools.empty(xs) === tools.empty(xs) && + ((tools.empty(xs) === true) || + (tools.empty(xs) === false))); +} + + +function dictProp(pairs) { + var dict = tools.dict(pairs); + return _.map(pairs, + function(pair) { + dict[pair[0]] === pair[1]; + }); +} + +function extendProp(pair) { + if (pair.length < 2) { + // empty lists or lists with one item are undefined + // so just return true because extend can't handle them + return true; + } + var x = _.first(pair); + var y = _.first(_.rest(pair)); + var extended = tools.extend(x,y); + return x.length + y.length === extended.length; +} + +function toolsTests() { + assert.equal(true, tools.empty([])); + assert.equal(true, qc.forAll(dictProp, arbArrayofPairs)); + assert.equal(true, qc.forAll(extendProp, arbPairs)); +} + + +toolsTests(); diff --git a/tools.js b/tools.js index bccb8b1..a20f399 100644 --- a/tools.js +++ b/tools.js @@ -41,21 +41,10 @@ function dict(pairs) { return o; } -function flatten(xs) { - if (!(xs instanceof Array)) { - return xs; - } - if (xs.every(function (x) { - return !(x instanceof Array); - })) { - return xs; - } - return [].concat.apply([], xs); -} - function extend(xs, ys) { - xs.push.apply(xs, ys); - return xs; + var result = _.clone(xs); + result.push.apply(result, ys); + return result; } RegExp.escape= function(s) { -- 2.30.2 From ffd3b005907b1fea3dadb6061b9a6d302b3c09b6 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sat, 15 Mar 2014 23:02:22 -0400 Subject: [PATCH 39/43] testing stuff --- test.js | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/test.js b/test.js index 7c254ac..d69b063 100755 --- a/test.js +++ b/test.js @@ -20,6 +20,15 @@ function arbArray(gen) { return qc.arbArray(gen); } +function arbArrays(gen) { + return function() { + return qc.arbArray( + function () { + return qc.arbArray(gen); + }) + }; +} + function arbPair(gen) { return function() { @@ -48,9 +57,30 @@ function emptyProp(xs) { function dictProp(pairs) { var dict = tools.dict(pairs); - return _.map(pairs, + var result = _.map(pairs, function(pair) { - dict[pair[0]] === pair[1]; + if ((_.size(pair) < 2) || + (_.size(pair[0]) < 1) || + (_.size(pair[1]) < 1)) { + return true; + } + return dict[pair[0]] === pair[1]; + }); + if (_.every(result, _.identity)) { + return true; + } + return false; +} + +function opMatchProp(strings) { + console.log(typeof strings); + if (strings.length < 1) { + return true; + } + var match = tools.opMatch(strings); + return _.map(strings, + function(str) { + return match(str); }); } @@ -70,6 +100,7 @@ function toolsTests() { assert.equal(true, tools.empty([])); assert.equal(true, qc.forAll(dictProp, arbArrayofPairs)); assert.equal(true, qc.forAll(extendProp, arbPairs)); + assert.equal(true, qc.forAll(opMatchProp, arbArrays(qc.arbString))); } -- 2.30.2 From 8bd8e7385db357a546c23e2c0effdd971075eb8e Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 16 Mar 2014 00:30:36 -0400 Subject: [PATCH 40/43] adding more tests, commenting out regex test until I can figure out why some unicode chars cause it to fail --- failing.js | 3 +++ test.js | 33 +++++++++++++++++---------------- tools.js | 10 +++++++++- 3 files changed, 29 insertions(+), 17 deletions(-) create mode 100644 failing.js diff --git a/failing.js b/failing.js new file mode 100644 index 0000000..40a85a1 --- /dev/null +++ b/failing.js @@ -0,0 +1,3 @@ +var xs = [ 'ÎßLä˜ÜyY;çiM´\u0011\u0007òЊ—H¹Ý', + 'I®s\u001eƒ\u0000(³\u0000`×\u001eÉ ‚—Õ\tNž(E=˜‹ô]àlŒpž\u0002c+ü÷Nr\u000ej¢V͑\u0003#\u0018#' ]; +module.exports = { xs: xs}; diff --git a/test.js b/test.js index d69b063..efbfe2c 100755 --- a/test.js +++ b/test.js @@ -20,13 +20,8 @@ function arbArray(gen) { return qc.arbArray(gen); } -function arbArrays(gen) { - return function() { - return qc.arbArray( - function () { - return qc.arbArray(gen); - }) - }; +function arbStrings() { + return qc.arbArray(qc.arbString); } @@ -73,15 +68,21 @@ function dictProp(pairs) { } function opMatchProp(strings) { - console.log(typeof strings); - if (strings.length < 1) { - return true; - } var match = tools.opMatch(strings); - return _.map(strings, - function(str) { - return match(str); - }); + var result = _.every(_.map(strings, + function (str) { + if (str.replace(/ /g,'').length < 1) { + return true; + } + var res = match(str); + if (res !== false) { + console.log(str); + return true; + } + return false; + }), + _.identity); + return result; } function extendProp(pair) { @@ -100,7 +101,7 @@ function toolsTests() { assert.equal(true, tools.empty([])); assert.equal(true, qc.forAll(dictProp, arbArrayofPairs)); assert.equal(true, qc.forAll(extendProp, arbPairs)); - assert.equal(true, qc.forAll(opMatchProp, arbArrays(qc.arbString))); + //assert.equal(true, qc.forAll(opMatchProp, arbStrings)); } diff --git a/tools.js b/tools.js index a20f399..8fe5c36 100644 --- a/tools.js +++ b/tools.js @@ -1,4 +1,5 @@ var _ = require("underscore"); +var example = require("./failing.js"); function empty(xs) { return _.size(xs) < 1; @@ -52,14 +53,21 @@ RegExp.escape= function(s) { }; function operatorMatch(ops) { + ops = _.filter(ops, + function (op) { + return op.replace(/ /g,'').length > 1; + }); var rstring = ops.sort(min).reduce( function(acc, x) { + if (!x || x.length < 1) { + return ""; + } return acc + "(" + RegExp.escape(x) + ")|"; }, ""); var reg = new RegExp(rstring); return function(x) { var matched = reg.exec(x); - if (matched[0]) { + if ((!(_.isNull(matched))) && matched[0]) { return matched[0]; } else { -- 2.30.2 From 3794ab9271ffb23b8c2101243fc5146e97df6498 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 16 Mar 2014 00:31:13 -0400 Subject: [PATCH 41/43] removed test file --- failing.js | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 failing.js diff --git a/failing.js b/failing.js deleted file mode 100644 index 40a85a1..0000000 --- a/failing.js +++ /dev/null @@ -1,3 +0,0 @@ -var xs = [ 'ÎßLä˜ÜyY;çiM´\u0011\u0007òЊ—H¹Ý', - 'I®s\u001eƒ\u0000(³\u0000`×\u001eÉ ‚—Õ\tNž(E=˜‹ô]àlŒpž\u0002c+ü÷Nr\u000ej¢V͑\u0003#\u0018#' ]; -module.exports = { xs: xs}; -- 2.30.2 From 58f08f257881dd108fc20a75abc97445b8502ecd Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 16 Mar 2014 01:08:01 -0400 Subject: [PATCH 42/43] handle empty applications --- parse.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/parse.js b/parse.js index 1d7d4ac..301515c 100755 --- a/parse.js +++ b/parse.js @@ -454,8 +454,9 @@ function computeApp(tokens, charnum, linenum) { var lhs = parse(tokens); var next; var result; - if (fst(tokens)) + if (fst(tokens)) { next = fst(tokens); + } else { throw error.JSyntaxError(linenum, charnum, @@ -476,13 +477,18 @@ function computeApp(tokens, charnum, linenum) { } else { /* it's a prefix application */ - - var parameters = parseMany(parse, + var parameters; + if (fst(tokens)[0] !== "right_paren") { + parameters = parseMany(parse, validArgTypes, validArgument, tokens, charnum, linenum); + } + else { + parameters = []; + } if ((!fst(tokens)) || fst(tokens)[0] !== "right_paren") { throw error.JSyntaxError(linenum, charnum, -- 2.30.2 From 3e1c5528e1a44da8b787e8207383e09ce8e0ef98 Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Sun, 16 Mar 2014 01:11:52 -0400 Subject: [PATCH 43/43] fixed bug in operator matching function --- test.js | 3 +++ tools.js | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test.js b/test.js index efbfe2c..a360582 100755 --- a/test.js +++ b/test.js @@ -97,6 +97,9 @@ function extendProp(pair) { return x.length + y.length === extended.length; } +/* Tokenizer tests */ + + function toolsTests() { assert.equal(true, tools.empty([])); assert.equal(true, qc.forAll(dictProp, arbArrayofPairs)); diff --git a/tools.js b/tools.js index 8fe5c36..5e65e34 100644 --- a/tools.js +++ b/tools.js @@ -1,5 +1,4 @@ var _ = require("underscore"); -var example = require("./failing.js"); function empty(xs) { return _.size(xs) < 1; @@ -55,7 +54,7 @@ RegExp.escape= function(s) { function operatorMatch(ops) { ops = _.filter(ops, function (op) { - return op.replace(/ /g,'').length > 1; + return op.length > 0; }); var rstring = ops.sort(min).reduce( function(acc, x) { -- 2.30.2