Implement tokenization

1 files changed, 7 insertions, 0 deletions

diff --git a/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java
new file mode 100644
index 0000000..8f0a060
--- /dev/null
+++ b/src/main/java/no/eliashaugsbakk/kompilator/tokenization/LexerState.java
@@ -0,0 +1,7 @@
+package no.eliashaugsbakk.kompilator.tokenization;
+
+public enum LexerState {
+  NORMAL,           // reading regular tokens
+  IN_STRING,        // inside a string (after ")
+  IN_WORD           // reading a keyword or identifier
+}