Implement tokenization

1 files changed, 93 insertions, 0 deletions

diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
new file mode 100644
index 0000000..4149b4f
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
@@ -0,0 +1,93 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+import static java.lang.Character.isLetterOrDigit;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_STRING;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_WORD;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.NORMAL;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.KEYWORD;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.LPAREN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Lexer {
+  private int line = 1;
+  private int column = 0;
+
+  private final String input;
+  private int position = 0;
+  private final List<Token> tokens = new ArrayList<>();
+  private final StringBuilder wordBuffer = new StringBuilder();
+  LexerState state = NORMAL;
+
+
+  public Lexer(String input) {
+    this.input = input;
+  }
+
+  List<Token> tokenize() {
+
+    while (position < input.length()) {
+      char current = input.charAt(position);
+
+      if (current == '\n') {
+        line++;
+        column = 0;
+      } else {
+        column++;
+      }
+
+      if (state == IN_WORD) {
+        if (!isLetterOrDigit(current) || Character.isWhitespace(current)) {
+          state = NORMAL;
+          characterizeWord();
+        } else {
+          wordBuffer.append(current);
+        }
+      }
+      if (state == NORMAL) {
+        if (!Character.isWhitespace(current)) {
+          if (current == '"') {
+            state = IN_STRING;
+          } else if (isLetterOrDigit(current)) {
+            state = IN_WORD;
+            wordBuffer.append(current);
+
+          } else if (current == '(') {
+            tokens.add(new Token(LPAREN, Character.toString(current), line, column));
+          } else if (current == ')') {
+            tokens.add(new Token(RPAREN, Character.toString(current), line, column));
+          } else if (current == ';') {
+            tokens.add(new Token(SEMICOLON, Character.toString(current), line, column));
+          }
+        }
+
+      } else if (state == IN_STRING) {
+        if (current == '"') {
+          state = NORMAL;
+          tokens.add(new Token(STRING, wordBuffer.toString(), line, column));
+          wordBuffer.delete(0, wordBuffer.length());
+        } else {
+          wordBuffer.append(current);
+        }
+      }
+      position++;
+    }
+    tokens.add(new Token(EOF, "End of File", line, column));
+    return tokens;
+  }
+
+  private void characterizeWord() {
+    if (Keywords.KEYWORDS.contains(wordBuffer.toString())) {
+      tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length()));
+    } else {
+      tokens.add(new Token(IDENTIFIER, wordBuffer.toString(), line, column - wordBuffer.length()));
+    }
+    wordBuffer.delete(0, wordBuffer.length());
+  }
+}