1 files changed, 44 insertions, 27 deletions

diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
index 53c7b53..d705ca5 100644
--- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
@@ -6,6 +6,7 @@ import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_TYPE;
 import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_WORD;
 import static no.eliashaugsbakk.kompilator.tokenization.LexerState.NORMAL;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.ASSIGN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COMMA;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER;
@@ -16,7 +17,6 @@ import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING_LITERAL;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.TYPE;
-import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,6 +41,8 @@ public class Lexer {
   public List<Token> tokenize() {
 
     while (position < input.length()) {
+      current = input.charAt(position);
+
       if (current == '\n') {
         line++;
         column = 0;
@@ -48,58 +50,66 @@ public class Lexer {
         column++;
       }
 
-      current = input.charAt(position);
-      if (state == IN_STRING) {
+      if (state == IN_WORD) {
+        inWord();
+      } else if (state == IN_TYPE) {
+        inType();
+      } else if (state == IN_STRING) {
         inString();
-      } else if (!Character.isWhitespace(current)) {
-        if (state == IN_WORD) {
-          inWord();
-        } else if (state == IN_TYPE) {
-          inType();
-        }
-
-        if (state == NORMAL) {
-          normal();
-        }
       }
+
+      if (state == NORMAL) {
+        normal();
+      }
+
       position++;
     }
+
     tokens.add(new Token(EOF, "End of File", line, column));
-    // tokens.forEach(token -> IO.println(token.type().toString() + ": " + token.value()));
+    tokens.forEach(token -> IO.println(token.type().toString() + ": " + token.value()));
     return tokens;
   }
 
   private void normal() {
+    if (Character.isWhitespace(current)) {
+      return;
+    }
+
     if (current == '"') {
       state = IN_STRING;
-    } else if (isLetterOrDigit(current)) {
+    } else if (isWordCharacter(current)) {
       state = IN_WORD;
       wordBuffer.append(current);
-
     } else if (current == ':') {
       tokens.add(new Token(COLON, ":", line, column));
       state = IN_TYPE;
+      position++; // Skip whitespace
 
     } else if (current == '=') {
       tokens.add(new Token(ASSIGN, "=", line, column));
     } else if (current == '(') {
-      tokens.add(new Token(LPAREN, Character.toString(current), line, column));
+      tokens.add(new Token(LPAREN, "(", line, column));
     } else if (current == ')') {
-      tokens.add(new Token(RPAREN, Character.toString(current), line, column));
+      tokens.add(new Token(RPAREN, ")", line, column));
     } else if (current == ';') {
-      tokens.add(new Token(SEMICOLON, Character.toString(current), line, column));
+      tokens.add(new Token(SEMICOLON, ";", line, column));
     } else if (current == ',') {
-      tokens.add(new Token(COMMA, Character.toString(current), line, column));
+      tokens.add(new Token(COMMA, ",", line, column));
     }
   }
 
+  private boolean isWordCharacter(char current) {
+    return isLetterOrDigit(current) || current == '_';
+  }
+
   private void inType() {
+
     if (current == '?') {
       state = NORMAL;
       tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length()));
       tokens.add(new Token(NULLABLE, "?", line, column));
       clearWordBuffer();
-    } else if (current == '=' || current == ';') {
+    } else if (Character.isWhitespace(current) || current == '=' || current == ';') {
       state = NORMAL;
       tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length()));
       clearWordBuffer();
@@ -109,29 +119,36 @@ public class Lexer {
   }
 
   private void inString() {
-    if (current == '"') { // closing " gets skipped implicitly
-      current = input.charAt(position);
-      state = NORMAL;
+    if (current == '"') {
       tokens.add(new Token(STRING_LITERAL, wordBuffer.toString(), line, column));
       wordBuffer.delete(0, wordBuffer.length());
+      state = NORMAL;
+      position++; // skip closing "
+      current = input.charAt(position);
     } else {
       wordBuffer.append(current);
     }
   }
 
   private void inWord() {
-    if (!isLetterOrDigit(current) && current != '_') {
+    if (Character.isWhitespace(current) || current == ':' || current == '(' || current == ')') {
       state = NORMAL;
       characterizeWord();
     } else {
       wordBuffer.append(current);
     }
-
   }
 
   private void characterizeWord() {
     if (Keywords.KEYWORDS.contains(wordBuffer.toString())) {
-      tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length()));
+      if (wordBuffer.toString().contentEquals("set")) {
+        tokens.add(new Token(KEYWORD, "set", line, column));
+      } else if (wordBuffer.toString().contentEquals("mut")) {
+        tokens.add(new Token(KEYWORD, "mut", line, column));
+      } else {
+        // this is a function call
+        tokens.add(new Token(KEYWORD, wordBuffer.toString(), line, column - wordBuffer.length()));
+      }
     } else {
       tokens.add(new Token(IDENTIFIER, wordBuffer.toString(), line, column - wordBuffer.length()));
     }