Implement tokenization

9 files changed, 149 insertions, 0 deletions

diff --git a/src/main/java/no/eliashaugsbakk/kompilator/IRGeneration/IRGeneration.java b/src/main/java/no/eliashaugsbakk/kompilator/IRGeneration/IRGeneration.java
new file mode 100644
index 0000000..f37db3d
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/IRGeneration/IRGeneration.java
@@ -0,0 +1,4 @@
+package no.eliashaugsbakk.kompilator.IRGeneration;
+
+public class IRGeneration {
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/parsing/AST.java b/src/main/java/no/eliashaugsbakk/kompilator/parsing/AST.java
new file mode 100644
index 0000000..3f54e3c
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/parsing/AST.java
@@ -0,0 +1,4 @@
+package no.eliashaugsbakk.kompilator.parsing;
+
+public class AST {
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/parsing/Parser.java b/src/main/java/no/eliashaugsbakk/kompilator/parsing/Parser.java
new file mode 100644
index 0000000..a41a1bf
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/parsing/Parser.java
@@ -0,0 +1,4 @@
+package no.eliashaugsbakk.kompilator.parsing;
+
+public class Parser {
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/semanticAnalysis/Analyzer.java b/src/main/java/no/eliashaugsbakk/kompilator/semanticAnalysis/Analyzer.java
new file mode 100644
index 0000000..891018c
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/semanticAnalysis/Analyzer.java
@@ -0,0 +1,4 @@
+package no.eliashaugsbakk.kompilator.semanticAnalysis;
+
+public class Analyzer {
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java
new file mode 100644
index 0000000..f2f70d3
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Keywords.java
@@ -0,0 +1,7 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+import java.util.Set;
+
+class Keywords {
+    static final Set<String> KEYWORDS = Set.of("print");
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
new file mode 100644
index 0000000..4149b4f
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
@@ -0,0 +1,93 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+import static java.lang.Character.isLetterOrDigit;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_STRING;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_WORD;
+import static no.eliashaugsbakk.kompilator.tokenization.LexerState.NORMAL;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.KEYWORD;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.LPAREN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Lexer {
+  private int line = 1;
+  private int column = 0;
+
+  private final String input;
+  private int position = 0;
+  private final List<Token> tokens = new ArrayList<>();
+  private final StringBuilder wordBuffer = new StringBuilder();
+  LexerState state = NORMAL;
+
+
+  public Lexer(String input) {
+    this.input = input;
+  }
+
+  List<Token> tokenize() {
+
+    while (position < input.length()) {
+      char current = input.charAt(position);
+
+      if (current == '\n') {
+        line++;
+        column = 0;
+      } else {
+        column++;
+      }
+
+      if (state == IN_WORD) {
+        if (!isLetterOrDigit(current) || Character.isWhitespace(current)) {
+          state = NORMAL;
+          characterizeWord();
+        } else {
+          wordBuffer.append(current);
+        }
+      }
+      if (state == NORMAL) {
+        if (!Character.isWhitespace(current)) {
+          if (current == '"') {
+            state = IN_STRING;
+          } else if (isLetterOrDigit(current)) {
+            state = IN_WORD;
+            wordBuffer.append(current);
+
+          } else if (current == '(') {
+            tokens.add(new Token(LPAREN, Character.toString(current), line, column));
+          } else if (current == ')') {
+            tokens.add(new Token(RPAREN, Character.toString(current), line, column));
+          } else if (current == ';') {
+            tokens.add(new Token(SEMICOLON, Character.toString(current), line, column));
+          }
+        }
+
+      } else if (state == IN_STRING) {
+        if (current == '"') {
+          state = NORMAL;
+          tokens.add(new Token(STRING, wordBuffer.toString(), line, column));
+          wordBuffer.delete(0, wordBuffer.length());
+        } else {
+          wordBuffer.append(current);
+        }
+      }
+      position++;
+    }
+    tokens.add(new Token(EOF, "End of File", line, column));
+    return tokens;
+  }
+
+  private void characterizeWord() {
+    if (Keywords.KEYWORDS.contains(wordBuffer.toString())) {
+      tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length()));
+    } else {
+      tokens.add(new Token(IDENTIFIER, wordBuffer.toString(), line, column - wordBuffer.length()));
+    }
+    wordBuffer.delete(0, wordBuffer.length());
+  }
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java
new file mode 100644
index 0000000..8f0a060
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java
@@ -0,0 +1,7 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+public enum LexerState {
+  NORMAL,           // reading regular tokens
+  IN_STRING,        // inside a string (after ")
+  IN_WORD           // reading a keyword or identifier
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java
new file mode 100644
index 0000000..519b6b6
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java
@@ -0,0 +1,15 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+class Token {
+  TokenType type;
+  String value;
+  int line;
+  int colum;
+
+  Token(TokenType type, String value, int line, int colum) {
+    this.type = type;
+    this.value = value;
+    this.line = line;
+    this.colum = colum;
+  }
+}
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java
new file mode 100644
index 0000000..408abb2
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java
@@ -0,0 +1,11 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+enum TokenType {
+  KEYWORD,      // print, var, if, while, function, etc.
+  IDENTIFIER,   // variable_1
+  STRING,       // "Hello, World!"
+  LPAREN,       // (
+  RPAREN,       // )
+  SEMICOLON,    // ;
+  EOF           // End of File
+}