1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
package no.eliashaugsbakk.kompilator.parsing;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.ASSIGN;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COMMA;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.KEYWORD;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.LPAREN;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.NULLABLE;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING_LITERAL;
import static no.eliashaugsbakk.kompilator.tokenization.TokenType.TYPE;
import java.util.ArrayList;
import java.util.List;
import no.eliashaugsbakk.kompilator.parsing.node.Program;
import no.eliashaugsbakk.kompilator.parsing.node.expression.Expression;
import no.eliashaugsbakk.kompilator.parsing.node.expression.FunctionCall;
import no.eliashaugsbakk.kompilator.parsing.node.expression.Identifier;
import no.eliashaugsbakk.kompilator.parsing.node.expression.literal.StringLiteral;
import no.eliashaugsbakk.kompilator.parsing.node.statement.Assignment;
import no.eliashaugsbakk.kompilator.parsing.node.statement.ExpressionStatement;
import no.eliashaugsbakk.kompilator.parsing.node.statement.IdentifierDeclaration;
import no.eliashaugsbakk.kompilator.parsing.node.statement.Statement;
import no.eliashaugsbakk.kompilator.tokenization.Token;
/*
The goal of parsing is to create the AST.
The root node is the program itself, which can only hold statements.
Statements:
- Is an action which executes something
- Has no return or implicit value
- Must end with a semicolon
- A statement may contain expressions
- An expression cannot contain a statement
Expressions:
- Returns or produces a value
- Does not end with a semicolon
- Can be nested instide other expressions or a statement
- Examples:
- Literals
- Identifiers
- Binary Operations
- Function calls
*/
public class Parser {
private final List<Token> tokens;
private final Program rootNode;
private int current = 0;
public Parser(List<Token> tokens) {
this.tokens = tokens;
this.rootNode = new Program();
}
public AST parse() throws ParserException {
while (current < tokens.size()) {
Token token = tokens.get(current);
// End of File
if (token.type() == EOF) {
break;
}
// root node must contain only statements
Statement stmt = parseStatement();
rootNode.addStatement(stmt);
}
return new AST(rootNode);
}
private Statement parseStatement() throws ParserException {
// multiple statements are supported for v0.0.2;
// identifier declaration: x: string = "hello"; (with expression)
// identifier declaration: x: string; (without expression)
// assignment: x = "hello"; (identifier gets assigned an expression)
// expression statements: print(x); (function without a return value)
// each statement must either start with a keyword or an identifier
Token token = tokens.get(current);
// only keyword implemented is "skriv", so calling parseExpression right away
if (token.type() == KEYWORD) {
return parseExpressionStatement();
}
// must be identifier declaration or an assignment
else if (token.type() == IDENTIFIER) {
return parseIdentifierStatement();
}
// must be some other token which is not a statement
else {
throw new ParserException(token.line(), token.column(), "Unexpected token: " + token.value());
}
}
private ExpressionStatement parseExpressionStatement() throws ParserException {
// this can be any expression used as a statement
// print(x);
// 5 + 6;
// "hello";
Expression expression = parseExpression();
expectSemicolon();
return new ExpressionStatement(expression);
}
private Statement parseIdentifierStatement() throws ParserException {
// this is either:
// - declaration of a new variable with an associated value
// - declaration of a new variable without an associated value
// - reassigning an existing variable
/*
x: string?; IDENTIFIER, COLON, TYPE, SEMICOLON
x: string = "string"; IDENTIFIER, COLON, TYPE, ASSIGN, STRING, SEMICOLON
x = "string"; IDENTIFIER, ASSIGN, STRING, SEMICOLON
*/
String identifier = tokens.get(current).value();
current++;
Token next = tokens.get(current);
if (next.type() == COLON) {
// x: string = "hello";
return parseDeclaration(identifier);
} else if (next.type() == ASSIGN) {
// x = "hello";
return parseAssignment(identifier);
} else {
throw new ParserException(next.line(), next.column(), "Expected : or = after identifier");
}
}
private Statement parseDeclaration(String identifier) throws ParserException {
// x: type;
current++; // skip colon
Type type = parseType();
Expression initializer = null;
// x: type = "hello";
if (tokens.get(current).type() == ASSIGN) {
current++; // skip =
initializer = parseExpression();
}
expectSemicolon();
return new IdentifierDeclaration(identifier, type, initializer);
}
private Statement parseAssignment(String identifier) throws ParserException {
// x = value;
current++; // skip =
Expression value = parseExpression();
expectSemicolon();
return new Assignment(identifier, value);
}
private Type parseType() throws ParserException {
Token token = tokens.get(current);
if (token.type() != TYPE) {
throw new ParserException(token.line(), token.column(),
"Expected type, got: " + token.value());
}
current++;
String typeName = token.value();
boolean nullable = false;
if (current < tokens.size() && tokens.get(current).type() == NULLABLE) {
nullable = true;
current++;
}
return new Type(typeName, nullable);
}
private Expression parseExpression() throws ParserException {
// an expression produces a value and may contain other expressions
// implemented for v0.0.2 are:
// "string" - STRING_LITERAL
// my_var - IDENTIFIER
Token token = tokens.get(current);
current++;
if (token.type() == STRING_LITERAL) {
return new StringLiteral(token.value());
} else if (token.type() == IDENTIFIER || token.type() == KEYWORD) {
// could be function call or just identifier reference
if (current < tokens.size() && tokens.get(current).type() == LPAREN) {
current--;
return parseFunctionCall();
}
return new Identifier(token.value());
} else {
throw new ParserException(token.line(), token.column(),
"Unexpected token: " + token.value() + ". Expected an expression");
}
}
private FunctionCall parseFunctionCall() throws ParserException {
String name = tokens.get(current).value();
current++;
List<Expression> arguments = parseFunctionArguments();
return new FunctionCall(name, arguments);
}
private List<Expression> parseFunctionArguments() throws ParserException {
List<Expression> arguments = new ArrayList<>();
if (tokens.get(current).type() != LPAREN) {
throw new ParserException(tokens.get(current).line(), tokens.get(current).column(),
"Expected (");
}
current++; // skip (
while (current < tokens.size() && tokens.get(current).type() != RPAREN) {
arguments.add(parseExpression()); // parseExpression() increments current
if (tokens.get(current).type() == COMMA) {
current++; // skip comma
}
}
if (current >= tokens.size() || tokens.get(current).type() != RPAREN) {
throw new ParserException(tokens.get(current).line(), tokens.get(current).column(),
"Expected )");
}
current++; // skip )
return arguments;
}
private void expectSemicolon() throws ParserException {
if (current >= tokens.size() || tokens.get(current).type() != SEMICOLON) {
throw new ParserException(-1, -1, "Expected ;");
}
current++;
}
}
|