package Compiler; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public class TokenScanner { private String sourceCode; List tokens = new ArrayList<>(); private int tokenStart=0; private int currentLoc=0; //Extract tokens from the source code by reading character by character List extractTokens(String sourceCode){ this.sourceCode=sourceCode; while (!checkEOF()){ tokenStart=currentLoc; readToken(); } tokens.add(new Token(TokenType.EOF, "", null)); return tokens; } //Extract a single token private void readToken(){ char checkChar = sourceCode.charAt(currentLoc); switch(checkChar){ case ' ':break; case '\n':break; case '\r':break; case '\t': break; case '(': createTokenNull(TokenType.LEFT_PAREN); break; case ')': createTokenNull(TokenType.RIGHT_PAREN); break; case '+': createTokenNull(TokenType.PLUS); break; case '-': createTokenNull(TokenType.MINUS); break; case '*': createTokenNull(TokenType.STAR); break; case '/': createTokenNull(TokenType.SLASH); break; case ';': createTokenNull(TokenType.SEMI_COLON); break; //Some tokens are multiple characters long (==, <=) etc //so need to check next char as well case '=': if (checkNextChar('=')){ createTokenNull(TokenType.EQUALITY); break; } else { createTokenNull(TokenType.EQUALS); break; } case ':': if (checkNextChar(':')){ createTokenNull(TokenType.DEFINE); break; } else { createTokenNull(TokenType.COLON); break; } case '<': if (checkNextChar('=')){ createTokenNull(TokenType.LESS_EQUAL); break; } else { createTokenNull(TokenType.LESS); break; } case '>': if (checkNextChar('=')){ createTokenNull(TokenType.GREATER_EQUAL); break; } else { createTokenNull(TokenType.GREATER); break; } default: //Check for numer if (checkIsDigit(checkChar)){ String type = "int"; while (checkIsDigit(lookAhead())){ currentLoc++; } //Check if number contains a decimal point if (lookAhead()=='.' && checkIsDigit(lookTwoAhead())){ type="double"; currentLoc++; while (checkIsDigit(lookAhead())){ currentLoc++; } } if (type.equals("double")){ createToken(TokenType.NUMBER, Double.parseDouble(sourceCode.substring(tokenStart, currentLoc+1))); } else{ createToken(TokenType.NUMBER, Integer.parseInt(sourceCode.substring(tokenStart, currentLoc+1))); } } else if (checkIsAlpha(checkChar)){ while (checkIsAlpha(lookAhead())){ currentLoc++; } String text = sourceCode.substring(tokenStart, currentLoc+1); TokenType type = keywords.get(text); if(type == null){ createToken(TokenType.IDENTIFIER, text); } else{ createToken(type, text); } } else { Language.displayError("Unexpected Character"); } } currentLoc++; } //Test for end of file private boolean checkEOF(){ return currentLoc>=sourceCode.length(); } //Create a token without a value private void createTokenNull(TokenType type){ createToken(type, null); } //Create token private void createToken(TokenType type, Object value){ String tokenText = sourceCode.substring(tokenStart, currentLoc+1); tokens.add(new Token(type, tokenText, value)); } //Check if the next char matches a given char private boolean checkNextChar(char matchChar){ if (checkEOF()){ return false; } if (sourceCode.charAt(currentLoc+1)==matchChar){ currentLoc++; return true; } return false; } //Look at the next char in the source code private char lookAhead(){ if (currentLoc+1>=sourceCode.length()){ return ' '; } else { return sourceCode.charAt(currentLoc+1); } } //Look 2 chars ahead in the source code private char lookTwoAhead(){ if (currentLoc+2>=sourceCode.length()){ return ' '; } else { return sourceCode.charAt(currentLoc+2); } } //Check if a given char is a digit private boolean checkIsDigit(char checkChar){ return checkChar>='0' && checkChar<='9'; } private boolean checkIsAlpha(char checkChar){ return ('a'<=checkChar && checkChar<='z')|| ('A'<=checkChar && checkChar<='Z'); } private static final Map keywords; static { keywords = new HashMap<>(); keywords.put("int", TokenType.INT); keywords.put("real", TokenType.REAL); keywords.put("print", TokenType.PRINT); keywords.put("if", TokenType.IF); keywords.put("then", TokenType.THEN); keywords.put("endif", TokenType.ENDIF); keywords.put("else", TokenType.ELSE); } }