From cb29252f1e0d29d555fb232f39d343930fc76105 Mon Sep 17 00:00:00 2001
From: AidenRushbrooke <72034940+AidenRushbrooke@users.noreply.github.com>
Date: Sat, 9 Oct 2021 23:12:42 +0100
Subject: Added basic lexical analysis
---
code/Interpreter/TokenScanner.java | 132 +++++++++++++++++++++++++++++++++++++
1 file changed, 132 insertions(+)
create mode 100644 code/Interpreter/TokenScanner.java
(limited to 'code/Interpreter/TokenScanner.java')
diff --git a/code/Interpreter/TokenScanner.java b/code/Interpreter/TokenScanner.java
new file mode 100644
index 0000000..87f1e4b
--- /dev/null
+++ b/code/Interpreter/TokenScanner.java
@@ -0,0 +1,132 @@
+package Interpreter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TokenScanner {
+ private String sourceCode;
+ List<Token> tokens = new ArrayList<>();
+ private int tokenStart=0;
+ private int currentLoc=0;
+
+ //Extract tokens from the source code by reading character by character
+ List<Token> extractTokens(String sourceCode){
+ this.sourceCode=sourceCode;
+ while (!checkEOF()){
+ tokenStart=currentLoc;
+ readToken();
+ }
+ return tokens;
+ }
+
+ //Extract a single token
+ private void readToken(){
+ char checkChar = sourceCode.charAt(currentLoc);
+ switch(checkChar){
+ case '(': createTokenNull(TokenType.LEFT_PAREN); break;
+ case ')': createTokenNull(TokenType.RIGHT_PAREN); break;
+ case '+': createTokenNull(TokenType.PLUS); break;
+ case '-': createTokenNull(TokenType.MINUS); break;
+ case '*': createTokenNull(TokenType.STAR); break;
+ case '/': createTokenNull(TokenType.SLASH); break;
+
+ //Some tokens are multiple characters long (==, <=) etc
+ //so need to check next char as well
+ case '=':
+ if (checkNextChar('=')){
+ createTokenNull(TokenType.EQUALITY);
+ break;
+ } else {
+ createTokenNull(TokenType.EQUALS);
+ break;
+ }
+ case '<':
+ if (checkNextChar('=')){
+ createTokenNull(TokenType.LESS_EQUAL);
+ break;
+ } else {
+ createTokenNull(TokenType.LESS);
+ break;
+ }
+ case '>':
+ if (checkNextChar('=')){
+ createTokenNull(TokenType.GREATER_EQUAL);
+ break;
+ } else {
+ createTokenNull(TokenType.GREATER);
+ break;
+ }
+ default:
+
+ //Check for numer
+ if (checkIsDigit(checkChar)){
+ while (checkIsDigit(lookAhead())){
+ currentLoc++;
+ }
+ //Check if number contains a decimal point
+ if (lookAhead()=='.' && checkIsDigit(lookTwoAhead())){
+ currentLoc++;
+ while (checkIsDigit(lookAhead())){
+ currentLoc++;
+ }
+ }
+ createToken(TokenType.NUMBER, Double.parseDouble(sourceCode.substring(tokenStart, currentLoc+1)));
+ }
+ }
+ currentLoc++;
+
+ }
+
+ //Test for end of file
+ private boolean checkEOF(){
+ return currentLoc>=sourceCode.length();
+ }
+
+ //Create a token without a value
+ private void createTokenNull(TokenType type){
+ createToken(type, null);
+ }
+
+ //Create token
+ private void createToken(TokenType type, Object value){
+ String tokenText = sourceCode.substring(tokenStart, currentLoc+1);
+ tokens.add(new Token(type, tokenText, value));
+ }
+
+ //Check if the next char matches a given char
+ private boolean checkNextChar(char matchChar){
+ if (checkEOF()){
+ return false;
+ }
+ if (sourceCode.charAt(currentLoc+1)==matchChar){
+ currentLoc++;
+ return true;
+ }
+ return false;
+ }
+
+ //Look at the next char in the source code
+ private char lookAhead(){
+ if (currentLoc+1>=sourceCode.length()){
+ return ' ';
+
+ } else {
+ return sourceCode.charAt(currentLoc+1);
+ }
+ }
+
+ //Look 2 chars ahead in the source code
+ private char lookTwoAhead(){
+ if (currentLoc+2>=sourceCode.length()){
+ return ' ';
+
+ } else {
+ return sourceCode.charAt(currentLoc+2);
+ }
+ }
+
+ //Check if a given char is a digit
+ private boolean checkIsDigit(char checkChar){
+ return checkChar>='0' && checkChar<='9';
+ }
+}
--
cgit v1.2.3