3 files changed, 24 insertions, 15 deletions
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
index c185650..53c7b53 100644
--- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Lexer.java
@@ -6,15 +6,17 @@ import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_TYPE;
 import static no.eliashaugsbakk.kompilator.tokenization.LexerState.IN_WORD;
 import static no.eliashaugsbakk.kompilator.tokenization.LexerState.NORMAL;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.ASSIGN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COMMA;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.EOF;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.IDENTIFIER;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.KEYWORD;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.LPAREN;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.NULLABLE;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.RPAREN;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.SEMICOLON;
-import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.STRING_LITERAL;
 import static no.eliashaugsbakk.kompilator.tokenization.TokenType.TYPE;
-import static no.eliashaugsbakk.kompilator.tokenization.TokenType.TYPE_DECLARATION;
+import static no.eliashaugsbakk.kompilator.tokenization.TokenType.COLON;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,12 +49,11 @@ public class Lexer {
       }
 
       current = input.charAt(position);
-      if (!Character.isWhitespace(current)) {
-
+      if (state == IN_STRING) {
+        inString();
+      } else if (!Character.isWhitespace(current)) {
         if (state == IN_WORD) {
           inWord();
-        } else if (state == IN_STRING) {
-          inString();
         } else if (state == IN_TYPE) {
           inType();
         }
@@ -76,7 +77,7 @@ public class Lexer {
       wordBuffer.append(current);
 
     } else if (current == ':') {
-      tokens.add(new Token(TYPE_DECLARATION, ":", line, column));
+      tokens.add(new Token(COLON, ":", line, column));
       state = IN_TYPE;
 
     } else if (current == '=') {
@@ -87,11 +88,18 @@ public class Lexer {
       tokens.add(new Token(RPAREN, Character.toString(current), line, column));
     } else if (current == ';') {
       tokens.add(new Token(SEMICOLON, Character.toString(current), line, column));
+    } else if (current == ',') {
+      tokens.add(new Token(COMMA, Character.toString(current), line, column));
     }
   }
 
   private void inType() {
-    if (current == '=' || current == ';') {
+    if (current == '?') {
+      state = NORMAL;
+      tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length()));
+      tokens.add(new Token(NULLABLE, "?", line, column));
+      clearWordBuffer();
+    } else if (current == '=' || current == ';') {
       state = NORMAL;
       tokens.add(new Token(TYPE, wordBuffer.toString(), line, column - wordBuffer.length()));
       clearWordBuffer();
@@ -101,11 +109,10 @@ public class Lexer {
   }
 
   private void inString() {
-    if (current == '"') {
-      position++; // skip closing "
+    if (current == '"') { // closing " gets skipped implicitly
       current = input.charAt(position);
       state = NORMAL;
-      tokens.add(new Token(STRING, wordBuffer.toString(), line, column));
+      tokens.add(new Token(STRING_LITERAL, wordBuffer.toString(), line, column));
       wordBuffer.delete(0, wordBuffer.length());
     } else {
       wordBuffer.append(current);
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java
index 09c5d83..1cab703 100644
--- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/Token.java
@@ -1,4 +1,4 @@
 package no.eliashaugsbakk.kompilator.tokenization;
 
-public record Token(TokenType type, String value, int line, int colum) {
+public record Token(TokenType type, String value, int line, int column) {
 }
diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java
index 1d6382e..3a1d4ba 100644
--- a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/TokenType.java
@@ -3,12 +3,14 @@ package no.eliashaugsbakk.kompilator.tokenization;
 public enum TokenType {
   KEYWORD,          // print, var, if, while, function, etc.
   IDENTIFIER,       // variable_1
-  STRING,           // "Hello, World!"
-  TYPE_DECLARATION, // : (x[:] int = ...)
-  TYPE,             // String, i32, i16?, my_type, ... (? makes nullable)
+  STRING_LITERAL,   // "Hello, World!"
+  COLON,            // : (x[:] int = ...)
+  TYPE,             // string, i32, i16?, my_type, ... (? makes nullable)
   ASSIGN,           // =
+  NULLABLE,    // ? (x = string?;)
   LPAREN,           // (
   RPAREN,           // )
   SEMICOLON,        // ;
+  COMMA,            // ,
   EOF               // End of File
 }