Finished Tokenizer

Added part to match keywords.

Author: Gabriel-T-Harris

parser/Token.java

@@ -4,7 +4,7 @@
 <b>
 Purpose: Types of Tokens to be created by Lexical Analyzer.<br>
 Programmer: Gabriel Toban Harris <br>
-Date: 2021-07-26
+Date: 2021-07-26, 2021-7-30
 </b>
 */
 
@@ -45,13 +45,13 @@ public static boolean is_error(final Lexeme_Types INPUT)
      * 
      * @param TYPE of LEXEME
      * @param LINE_NUMBER location in source file
-     * @param LEXEME content of LEXEME
+     * @param LEXEME content of lexeme which is then trimmed by {@link String#trim()}
      */
     public Token(final Token.Lexeme_Types TYPE, final long LINE_NUMBER, final String LEXEME)
     {
         this.TYPE = TYPE;
         this.LINE_NUMBER = LINE_NUMBER;
-        this.LEXEME = LEXEME;
+        this.LEXEME = LEXEME.trim();
     }
 
     //getters

parser/Tokenizer.java

@@ -8,7 +8,7 @@
 <b>
 Purpose: Perform tokenization on input files, such that later stages may read a stream of tokens one by one.<br>
 Programmer: Gabriel Toban Harris <br>
-Date: 2021-07-[27, 28]
+Date: 2021-07-[27, 28], 2021-7-30
 </b>
 */
 
@@ -91,6 +91,37 @@ public Returned_Data(final String EXTRA_DATA, final Token FULLY_FORMED_PART)
      */
     public final static String CONDITION_EXPR_END = ")";
 
+    /**
+     * Simple concatenation of chars which are not allowed to be a part of any keyword, ID, or special marker of a part.
+     */
+    public final static String RESTRICTED_CHARS = ";=" + SENTINEL_START + SENTINEL_END + CONDITION_CARD_START + CONDITION_CARD_END + CONDITION_SCENARIO_START +
+                                                  CONDITION_SCENARIO_END + CONDITION_EXPR_START + CONDITION_EXPR_END;
+
+    /**
+     * Representation of unary operator not.
+     */
+    public final static Pattern NOT = Pattern.compile("\\s*NOT\\s*");
+
+    /**
+     * Representation of binary operator and.
+     */
+    public final static Pattern AND = Pattern.compile("\\s*AND\\s*");
+
+    /**
+     * Representation of binary operator or.
+     */
+    public final static Pattern OR = Pattern.compile("\\s*OR\\s*");
+
+    /**
+     * Representation of binary operator xor.
+     */
+    public final static Pattern XOR = Pattern.compile("\\s*XOR\\s*");
+
+    /**
+     * Simply the the predefined class \s.
+     */
+    public final static Pattern WHITE_SPACE_CHAR = Pattern.compile("\\s");
+
     /**
      * Pattern indicating the definition of the {@link Lexeme_Types#DECK_START} {@link Token}.
      */
@@ -114,8 +145,7 @@ public Returned_Data(final String EXTRA_DATA, final Token FULLY_FORMED_PART)
     /**
      * Definition of valid char that an the ID lexeme can have.
      */
-    public final static Pattern ID_CHAR_SET = Pattern.compile("[^;=" + SENTINEL_START + SENTINEL_END + CONDITION_CARD_START + CONDITION_CARD_END + CONDITION_SCENARIO_START +
-                                                              CONDITION_SCENARIO_END + CONDITION_EXPR_START + CONDITION_EXPR_END + "]");
+    public final static Pattern ID_CHAR_SET = Pattern.compile("[^" + RESTRICTED_CHARS + "]");
 
     /**
      * Performs tokenization, after which other functions are called internally.
@@ -199,68 +229,66 @@ else if (placeholder.equals("*"))
                 //defer to ID_CHAR_SET
                 return new Returned_Data(new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEME.toString()));
             }
-            //TODO:finish
-            //binary operators
-
-            //display values
-            case "t":
-            {
-                if (INPUT.hasNext())
-                {
-                    String placeholder = INPUT.next();
-                    LEXEME.append(placeholder);
-                    //true keyword
-                    if (placeholder.equals("r"))
-                    {
-                        
-                        
-                        LEXEME.append(placeholder = INPUT.next());
-                        
-                        if (placeholder.equals("u"))
-                        {
-                            LEXEME.append(placeholder= INPUT.next());
-                            
-                            if (placeholder.equals("e"))
-                            {
-                                placeholder= INPUT.next();
-                                
-                                if ()
-                            }
-                        }
-                    }
-
-                    return parse_ID(LINE_NUMBER, LEXEME, INPUT);
-                }
-
-                return new Returned_Data(new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEME.toString()));
-            }
-            //guess it is some sort of ID
+            //parse multichar sequences
             default:
-                return gather_ID_chars(LINE_NUMBER, LEXEME, INPUT);
+                return gather_keyword_chars(LINE_NUMBER, LEXEME, INPUT);
         }
     }
 
     /**
-     * Determines which {@link Token} should be formed given the lexeme provided.
+     * Function to attempt to find keywords, other wise defers to {@link #gather_ID_chars(long, StringBuilder, Scanner)}
      * 
      * @param LINE_NUMBER of source file being read
-     * @param REMAINDER which is the extra data that should be feed back into {@link #tokenize(long, String, Scanner)}
-     * @param COMPLETE_LEXEME which is fully formed
+     * @param LEXEME_START is the lexem formed thus far
+     * @param INPUT source to be read from
      * @return the created {@link Token} wrapped in a {@link Returned_Data}
      */
-    private static Returned_Data parse_potentional_ID(final long LINE_NUMBER, final String REMAINDER, final String COMPLETE_LEXEME)
+    private static Returned_Data gather_keyword_chars(final long LINE_NUMBER, final StringBuilder LEXEME_START, final Scanner INPUT)
     {
-        //Test lexeme for special sequences, if all fail then is in fact ID.
-        if (TREE_START.matcher(COMPLETE_LEXEME).matches())
-            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.TREE_START, LINE_NUMBER, COMPLETE_LEXEME));
-        else if (DISPLAY_START.matcher(COMPLETE_LEXEME).matches())
-            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.DISPLAY_START, LINE_NUMBER, COMPLETE_LEXEME));
-        else if (PROBABILITY_START.matcher(COMPLETE_LEXEME).matches())
-            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.PROBABILITY_START, LINE_NUMBER, COMPLETE_LEXEME));
-        else if (DECK_START.matcher(COMPLETE_LEXEME).matches())
-            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.DECK_START, LINE_NUMBER, COMPLETE_LEXEME));
-        else
-            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.ID, LINE_NUMBER, COMPLETE_LEXEME));
+        String placeholder;
+
+        while (INPUT.hasNext())
+        {
+            placeholder = INPUT.next();
+            
+            //check for keyword end
+            if (WHITE_SPACE_CHAR.matcher(placeholder).matches())
+            {
+                final String LEXEM = LEXEME_START.toString();
+                
+                if (AND.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.AND, LINE_NUMBER, LEXEM));
+                else if (OR.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.OR, LINE_NUMBER, LEXEM));
+                else if (NOT.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.NOT, LINE_NUMBER, LEXEM));
+                else if (XOR.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.XOR, LINE_NUMBER, LEXEM));
+                else
+                    return gather_ID_chars(LINE_NUMBER, LEXEME_START.append(placeholder), INPUT);
+            }
+            //check for restricted char
+            else if (RESTRICTED_CHARS.contains(placeholder))
+            {
+                final String LEXEM = LEXEME_START.toString();
+                
+                if (AND.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.AND, LINE_NUMBER, LEXEM));
+                else if (OR.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.OR, LINE_NUMBER, LEXEM));
+                else if (NOT.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.NOT, LINE_NUMBER, LEXEM));
+                else if (XOR.matcher(LEXEM).matches())
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.XOR, LINE_NUMBER, LEXEM));
+                else
+                    return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEM));
+            }
+            //keep building
+            else
+                LEXEME_START.append(placeholder);
+        }
+        
+        return new Returned_Data(new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEME_START.toString()));
     }
 
     /**
@@ -289,27 +317,25 @@ private static Returned_Data gather_ID_chars(final long LINE_NUMBER, final Strin
     }
 
     /**
-     * Subroutine to create a {@link Lexeme_Types#ID} {@link Token}.
+     * Determines which {@link Token} should be formed given the lexeme provided.
      * 
      * @param LINE_NUMBER of source file being read
-     * @param LEXEME_START is the lexem formed thus far
-     * @param INPUT source to be read from
+     * @param REMAINDER which is the extra data that should be feed back into {@link #tokenize(long, String, Scanner)}
+     * @param COMPLETE_LEXEME which is fully formed
      * @return the created {@link Token} wrapped in a {@link Returned_Data}
      */
-    private static Returned_Data parse_ID(final long LINE_NUMBER, final StringBuilder LEXEME_START, final Scanner INPUT)
+    private static Returned_Data parse_potentional_ID(final long LINE_NUMBER, final String REMAINDER, final String COMPLETE_LEXEME)
     {
-        String placeholder;
-
-        while (INPUT.hasNext())
-        {
-            placeholder = INPUT.next();
-
-            if (ID_CHAR_SET.matcher(placeholder).matches())
-                LEXEME_START.append(placeholder);
-            else
-                return new Returned_Data(placeholder, new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEME_START.toString()));
-        }
-
-        return new Returned_Data(new Token(Token.Lexeme_Types.ID, LINE_NUMBER, LEXEME_START.toString()));
+        //Test lexeme for special sequences, if all fail then is in fact ID.
+        if (TREE_START.matcher(COMPLETE_LEXEME).matches())
+            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.TREE_START, LINE_NUMBER, COMPLETE_LEXEME));
+        else if (DISPLAY_START.matcher(COMPLETE_LEXEME).matches())
+            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.DISPLAY_START, LINE_NUMBER, COMPLETE_LEXEME));
+        else if (PROBABILITY_START.matcher(COMPLETE_LEXEME).matches())
+            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.PROBABILITY_START, LINE_NUMBER, COMPLETE_LEXEME));
+        else if (DECK_START.matcher(COMPLETE_LEXEME).matches())
+            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.DECK_START, LINE_NUMBER, COMPLETE_LEXEME));
+        else
+            return new Returned_Data(REMAINDER, new Token(Token.Lexeme_Types.ID, LINE_NUMBER, COMPLETE_LEXEME));
     }
 }