diff options
Diffstat (limited to 'src/main/java/no')
3 files changed, 51 insertions, 30 deletions
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java index 3b3bec9..9637153 100644 --- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java +++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java @@ -3,5 +3,9 @@ package no.eliashaugsbakk.kompilator.tokenization; import java.util.Set; public class Keywords { - public static final Set<String> KEYWORDS = Set.of("skriv"); + public static final Set<String> KEYWORDS = Set.of( + "skriv", + "set", + "mut" + ); } diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java index 53c7b53..d705ca5 100644 --- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java +++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java @@ -6,6 +6,7 @@ import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_TYPE; import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_WORD; import static no.eliashaugsbakk.kompilator.tokenization.LexerState.NORMAL; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.ASSIGN; +import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COMMA; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER; @@ -16,7 +17,6 @@ import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING_LITERAL; import static no.eliashaugsbakk.kompilator.tokenization.TokenType.TYPE; -import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON; import java.util.ArrayList; import java.util.List; @@ -41,6 +41,8 @@ public class Lexer { public List<Token> tokenize() { while (position < input.length()) { + current = input.charAt(position); + if (current == '\n') { line++; column = 0; @@ -48,58 +50,66 @@ public class Lexer { column++; } - current = input.charAt(position); - if (state == IN_STRING) { + if (state == IN_WORD) { + inWord(); + } else if (state == IN_TYPE) { + inType(); + } else if (state == IN_STRING) { inString(); - } else if (!Character.isWhitespace(current)) { - if (state == IN_WORD) { - inWord(); - } else if (state == IN_TYPE) { - inType(); - } - - if (state == NORMAL) { - normal(); - } } + + if (state == NORMAL) { + normal(); + } + position++; } + tokens.add(new Token(EOF, "End of File", line, column)); - // tokens.forEach(token -> IO.println(token.type().toString() + ": " + token.value())); + tokens.forEach(token -> IO.println(token.type().toString() + ": " + token.value())); return tokens; } private void normal() { + if (Character.isWhitespace(current)) { + return; + } + if (current == '"') { state = IN_STRING; - } else if (isLetterOrDigit(current)) { + } else if (isWordCharacter(current)) { state = IN_WORD; wordBuffer.append(current); - } else if (current == ':') { tokens.add(new Token(COLON, ":", line, column)); state = IN_TYPE; + position++; // Skip whitespace } else if (current == '=') { tokens.add(new Token(ASSIGN, "=", line, column)); } else if (current == '(') { - tokens.add(new Token(LPAREN, Character.toString(current), line, column)); + tokens.add(new Token(LPAREN, "(", line, column)); } else if (current == ')') { - tokens.add(new Token(RPAREN, Character.toString(current), line, column)); + tokens.add(new Token(RPAREN, ")", line, column)); } else if (current == ';') { - tokens.add(new Token(SEMICOLON, Character.toString(current), line, column)); + tokens.add(new Token(SEMICOLON, ";", line, column)); } else if (current == ',') { - tokens.add(new Token(COMMA, Character.toString(current), line, column)); + tokens.add(new Token(COMMA, ",", line, column)); } } + private boolean isWordCharacter(char current) { + return isLetterOrDigit(current) || current == '_'; + } + private void inType() { + if (current == '?') { state = NORMAL; tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length())); tokens.add(new Token(NULLABLE, "?", line, column)); clearWordBuffer(); - } else if (current == '=' || current == ';') { + } else if (Character.isWhitespace(current) || current == '=' || current == ';') { state = NORMAL; tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length())); clearWordBuffer(); @@ -109,29 +119,36 @@ public class Lexer { } private void inString() { - if (current == '"') { // closing " gets skipped implicitly - current = input.charAt(position); - state = NORMAL; + if (current == '"') { tokens.add(new Token(STRING_LITERAL, wordBuffer.toString(), line, column)); wordBuffer.delete(0, wordBuffer.length()); + state = NORMAL; + position++; // skip closing " + current = input.charAt(position); } else { wordBuffer.append(current); } } private void inWord() { - if (!isLetterOrDigit(current) && current != '_') { + if (Character.isWhitespace(current) || current == ':' || current == '(' || current == ')') { state = NORMAL; characterizeWord(); } else { wordBuffer.append(current); } - } private void characterizeWord() { if (Keywords.KEYWORDS.contains(wordBuffer.toString())) { - tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length())); + if (wordBuffer.toString().contentEquals("set")) { + tokens.add(new Token(KEYWORD, "set", line, column)); + } else if (wordBuffer.toString().contentEquals("mut")) { + tokens.add(new Token(KEYWORD, "mut", line, column)); + } else { + // this is a function call + tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length())); + } } else { tokens.add(new Token(IDENTIFIER, wordBuffer.toString(), line, column - wordBuffer.length())); } diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java index 3a1d4ba..40b9d2e 100644 --- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java +++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java @@ -4,10 +4,10 @@ public enum TokenType { KEYWORD, // print, var, if, while, function, etc. IDENTIFIER, // variable_1 STRING_LITERAL, // "Hello, World!" - COLON, // : (x[:] int = ...) + COLON, // : (set x[:] int = ...) TYPE, // string, i32, i16?, my_type, ... (? makes nullable) ASSIGN, // = - NULLABLE, // ? (x = string?;) + NULLABLE, // ? (x = string?;) LPAREN, // ( RPAREN, // ) SEMICOLON, // ; |