Lexer
class
class Lexer {
static const String QUOTES = "\"'";
static const String DOT = ".";
static const String SPECIAL = "(){}[].,;:";
static const String JSON_SEP = "{,";
static const String JSON_OPEN = "{[";
static const String JSON_CLOSE = "}]";
static const String WHITESPACE = " \r\t\n\v\u00A0";
static const String EXP_OP = "Ee";
static const String SIGN_OP = "+-";
static Map<String, String> ESCAPE =
{"n":"\n", "f":"\f", "r":"\r", "t":"\t", "v":"\v", "'":"'", '"':'"'};
List<Token> call(String text) {
List<Token> tokens = [];
Token token;
int index = 0;
int lastIndex;
int textLength = text.length;
String ch;
String lastCh = ":";
isIn(String charSet, [String c]) =>
charSet.indexOf(c != null ? c : ch) != -1;
was(String charSet) => charSet.indexOf(lastCh) != -1;
cc(String s) => s.codeUnitAt(0);
bool isNumber([String c]) {
int cch = cc(c != null ? c : ch);
return cc('0') <= cch && cch <= cc('9');
}
isIdent() {
int cch = cc(ch);
return
cc('a') <= cch && cch <= cc('z') ||
cc('A') <= cch && cch <= cc('Z') ||
cc('_') == cch || cch == cc('\$');
}
isWhitespace([String c]) => isIn(WHITESPACE, c);
isExpOperator([String c]) => isIn(SIGN_OP, c) || isNumber(c);
String peek() => index + 1 < textLength ? text[index + 1] : "EOF";
lexError(String s) { throw "Lexer Error: $s at column $index in expression [$text]"; }
// whileChars takes two functions: One called for each character
// and a second, optional function call at the end of the file.
// If the first function returns false, the the loop stops and endFn
// is not run.
whileChars(fn(), [endFn()]) {
while (index < textLength) {
ch = text[index];
int lastIndex = index;
if (fn() == false) {
return;
}
if (lastIndex >= index) {
throw "while chars loop must advance at index $index";
}
}
if (endFn != null) { endFn(); }
}
readString() {
int start = index;
String string = "";
String rawString = ch;
String quote = ch;
index++;
whileChars(() {
rawString += ch;
if (ch == '\\') {
index++;
whileChars(() {
rawString += ch;
if (ch == 'u') {
String hex = text.substring(index + 1, index + 5);
int charCode = int.parse(hex, radix: 16,
onError: (s) { lexError('Invalid unicode escape [\\u$hex]'); });
string += new String.fromCharCode(charCode);
index += 5;
} else {
var rep = ESCAPE[ch];
if (rep != null) {
string += rep;
} else {
string += ch;
}
index++;
}
return false; // BREAK
});
} else if (ch == quote) {
index++;
tokens.add(new Token(start, rawString)
..withValue(string));
return false; // BREAK
} else {
string += ch;
index++;
}
}, () {
lexError('Unterminated quote starting at $start');
});
}
readNumber() {
String number = "";
int start = index;
bool simpleInt = true;
whileChars(() {
if (ch == '.') {
number += ch;
simpleInt = false;
} else if (isNumber()) {
number += ch;
} else {
String peekCh = peek();
if (isIn(EXP_OP) && isExpOperator(peekCh)) {
simpleInt = false;
number += ch;
} else if (isExpOperator() && peekCh != '' && isNumber(peekCh) && isIn(EXP_OP, number[number.length - 1])) {
simpleInt = false;
number += ch;
} else if (isExpOperator() && (peekCh == '' || !isNumber(peekCh)) &&
isIn(EXP_OP, number[number.length - 1])) {
lexError('Invalid exponent');
} else {
return false; // BREAK
}
}
index++;
});
var ret = simpleInt ? int.parse(number) : double.parse(number);
tokens.add(new Token(start, number)..withValue(ret));
}
readIdent() {
String ident = "";
int start = index;
int lastDot = -1, peekIndex = -1;
String methodName;
whileChars(() {
if (ch == '.' || isIdent() || isNumber()) {
if (ch == '.') {
lastDot = index;
}
ident += ch;
} else {
return false; // BREAK
}
index++;
});
// The identifier had a . in the identifier
if (lastDot != -1) {
peekIndex = index;
while (peekIndex < textLength) {
String peekChar = text[peekIndex];
if (peekChar == "(") {
methodName = ident.substring(lastDot - start + 1);
ident = ident.substring(0, lastDot - start);
index = peekIndex;
}
if (isWhitespace(peekChar)) {
peekIndex++;
} else {
break;
}
}
}
var token = new Token(start, ident);
if (OPERATORS.containsKey(ident)) {
token.withOp(ident);
} else {
token.withGetterSetter(ident);
}
tokens.add(token);
if (methodName != null) {
tokens.add(new Token(lastDot, '.'));
tokens.add(new Token(lastDot + 1, methodName));
}
}
oneLexLoop() {
if (isIn(QUOTES)) {
readString();
} else if (isNumber() || isIn(DOT) && isNumber(peek())) {
readNumber();
} else if (isIdent()) {
readIdent();
} else if (isIn(SPECIAL)) {
tokens.add(new Token(index, ch));
index++;
} else if (isWhitespace()) {
index++;
} else {
// Check for two character operators (e.g. "==")
String ch2 = ch + peek();
if (OPERATORS.containsKey(ch2)) {
tokens.add(new Token(index, ch2)..withOp(ch2));
index += 2;
} else if (OPERATORS.containsKey(ch)) {
tokens.add(new Token(index, ch)..withOp(ch));
index++;
} else {
lexError('Unexpected next character [$ch]');
}
}
}
whileChars(() {
oneLexLoop();
});
return tokens;
}
}
Static Properties
const String DOT #
static const String DOT = "."
Map<String, String> ESCAPE #
static Map<String, String> ESCAPE =
{"n":"\n", "f":"\f", "r":"\r", "t":"\t", "v":"\v", "'":"'", '"':'"'}
const String EXP_OP #
static const String EXP_OP = "Ee"
const String JSON_CLOSE #
static const String JSON_CLOSE = "}]"
const String JSON_OPEN #
static const String JSON_OPEN = "{["
const String JSON_SEP #
static const String JSON_SEP = "{,"
const String QUOTES #
static const String QUOTES = "\"'"
const String SIGN_OP #
static const String SIGN_OP = "+-"
const String SPECIAL #
static const String SPECIAL = "(){}[].,;:"
const String WHITESPACE #
static const String WHITESPACE = " \r\t\n\v\u00A0"
Methods
List<Token> call(String text) #
List<Token> call(String text) {
List<Token> tokens = [];
Token token;
int index = 0;
int lastIndex;
int textLength = text.length;
String ch;
String lastCh = ":";
isIn(String charSet, [String c]) =>
charSet.indexOf(c != null ? c : ch) != -1;
was(String charSet) => charSet.indexOf(lastCh) != -1;
cc(String s) => s.codeUnitAt(0);
bool isNumber([String c]) {
int cch = cc(c != null ? c : ch);
return cc('0') <= cch && cch <= cc('9');
}
isIdent() {
int cch = cc(ch);
return
cc('a') <= cch && cch <= cc('z') ||
cc('A') <= cch && cch <= cc('Z') ||
cc('_') == cch || cch == cc('\$');
}
isWhitespace([String c]) => isIn(WHITESPACE, c);
isExpOperator([String c]) => isIn(SIGN_OP, c) || isNumber(c);
String peek() => index + 1 < textLength ? text[index + 1] : "EOF";
lexError(String s) { throw "Lexer Error: $s at column $index in expression [$text]"; }
// whileChars takes two functions: One called for each character
// and a second, optional function call at the end of the file.
// If the first function returns false, the the loop stops and endFn
// is not run.
whileChars(fn(), [endFn()]) {
while (index < textLength) {
ch = text[index];
int lastIndex = index;
if (fn() == false) {
return;
}
if (lastIndex >= index) {
throw "while chars loop must advance at index $index";
}
}
if (endFn != null) { endFn(); }
}
readString() {
int start = index;
String string = "";
String rawString = ch;
String quote = ch;
index++;
whileChars(() {
rawString += ch;
if (ch == '\\') {
index++;
whileChars(() {
rawString += ch;
if (ch == 'u') {
String hex = text.substring(index + 1, index + 5);
int charCode = int.parse(hex, radix: 16,
onError: (s) { lexError('Invalid unicode escape [\\u$hex]'); });
string += new String.fromCharCode(charCode);
index += 5;
} else {
var rep = ESCAPE[ch];
if (rep != null) {
string += rep;
} else {
string += ch;
}
index++;
}
return false; // BREAK
});
} else if (ch == quote) {
index++;
tokens.add(new Token(start, rawString)
..withValue(string));
return false; // BREAK
} else {
string += ch;
index++;
}
}, () {
lexError('Unterminated quote starting at $start');
});
}
readNumber() {
String number = "";
int start = index;
bool simpleInt = true;
whileChars(() {
if (ch == '.') {
number += ch;
simpleInt = false;
} else if (isNumber()) {
number += ch;
} else {
String peekCh = peek();
if (isIn(EXP_OP) && isExpOperator(peekCh)) {
simpleInt = false;
number += ch;
} else if (isExpOperator() && peekCh != '' && isNumber(peekCh) && isIn(EXP_OP, number[number.length - 1])) {
simpleInt = false;
number += ch;
} else if (isExpOperator() && (peekCh == '' || !isNumber(peekCh)) &&
isIn(EXP_OP, number[number.length - 1])) {
lexError('Invalid exponent');
} else {
return false; // BREAK
}
}
index++;
});
var ret = simpleInt ? int.parse(number) : double.parse(number);
tokens.add(new Token(start, number)..withValue(ret));
}
readIdent() {
String ident = "";
int start = index;
int lastDot = -1, peekIndex = -1;
String methodName;
whileChars(() {
if (ch == '.' || isIdent() || isNumber()) {
if (ch == '.') {
lastDot = index;
}
ident += ch;
} else {
return false; // BREAK
}
index++;
});
// The identifier had a . in the identifier
if (lastDot != -1) {
peekIndex = index;
while (peekIndex < textLength) {
String peekChar = text[peekIndex];
if (peekChar == "(") {
methodName = ident.substring(lastDot - start + 1);
ident = ident.substring(0, lastDot - start);
index = peekIndex;
}
if (isWhitespace(peekChar)) {
peekIndex++;
} else {
break;
}
}
}
var token = new Token(start, ident);
if (OPERATORS.containsKey(ident)) {
token.withOp(ident);
} else {
token.withGetterSetter(ident);
}
tokens.add(token);
if (methodName != null) {
tokens.add(new Token(lastDot, '.'));
tokens.add(new Token(lastDot + 1, methodName));
}
}
oneLexLoop() {
if (isIn(QUOTES)) {
readString();
} else if (isNumber() || isIn(DOT) && isNumber(peek())) {
readNumber();
} else if (isIdent()) {
readIdent();
} else if (isIn(SPECIAL)) {
tokens.add(new Token(index, ch));
index++;
} else if (isWhitespace()) {
index++;
} else {
// Check for two character operators (e.g. "==")
String ch2 = ch + peek();
if (OPERATORS.containsKey(ch2)) {
tokens.add(new Token(index, ch2)..withOp(ch2));
index += 2;
} else if (OPERATORS.containsKey(ch)) {
tokens.add(new Token(index, ch)..withOp(ch));
index++;
} else {
lexError('Unexpected next character [$ch]');
}
}
}
whileChars(() {
oneLexLoop();
});
return tokens;
}