diff options
| author | Elizabeth Alexander Hunt <me@liz.coffee> | 2026-07-02 11:55:17 -0700 |
|---|---|---|
| committer | Elizabeth Alexander Hunt <me@liz.coffee> | 2026-07-02 11:55:17 -0700 |
| commit | 6bf4b90c90f15f4ab60833bddf5b5756d1a6b1f6 (patch) | |
| tree | ed97e39ec77c5231ffd2c394493e68d00ddac5a4 /Homework/cs5300/project-three/parser | |
| download | misc-undergrad-6bf4b90c90f15f4ab60833bddf5b5756d1a6b1f6.tar.gz misc-undergrad-6bf4b90c90f15f4ab60833bddf5b5756d1a6b1f6.zip | |
Diffstat (limited to 'Homework/cs5300/project-three/parser')
| -rw-r--r-- | Homework/cs5300/project-three/parser/Action.java | 70 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Grammar.java | 125 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Item.java | 151 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Main.java | 337 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Parser.java | 341 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/ParserException.java | 81 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Rule.java | 93 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/State.java | 94 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/States.java | 44 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Tests.java | 69 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/Util.java | 132 | ||||
| -rw-r--r-- | Homework/cs5300/project-three/parser/util/SymbolComparator.java | 38 |
12 files changed, 1575 insertions, 0 deletions
diff --git a/Homework/cs5300/project-three/parser/Action.java b/Homework/cs5300/project-three/parser/Action.java new file mode 100644 index 0000000..040daab --- /dev/null +++ b/Homework/cs5300/project-three/parser/Action.java @@ -0,0 +1,70 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser; + +/** + * + * @author edwajohn + */ +public class Action { + + enum Act { + SHIFT, REDUCE, ACCEPT + }; + + private final Act act; + private final Integer state; + private final Rule rule; + + static Action createShift(Integer state) { + return new Action(Act.SHIFT, state, null); + } + + static Action createReduce(Rule rule) { + return new Action(Act.REDUCE, null, rule); + } + + static Action createAccept() { + return new Action(Act.ACCEPT, null, null); + } + + private Action(Act act, Integer state, Rule rule) { + this.act = act; + this.state = state; + this.rule = rule; + } + + public boolean isShift() { + return act == Act.SHIFT; + } + + public boolean isReduce() { + return act == Act.REDUCE; + } + + public boolean isAccept() { + return act == Act.ACCEPT; + } + + public Integer getState() { + return state; + } + + public Rule getRule() { + return rule; + } + + @Override + public String toString() { + if (isShift()) { + return "S" + state; + } + if (isReduce()) { + return "R" + rule.getName(); + } + return "acc"; + } +} diff --git a/Homework/cs5300/project-three/parser/Grammar.java b/Homework/cs5300/project-three/parser/Grammar.java new file mode 100644 index 0000000..0050cd1 --- /dev/null +++ b/Homework/cs5300/project-three/parser/Grammar.java @@ -0,0 +1,125 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser; + +import java.io.IOException; +import java.util.*; + +import parser.util.SymbolComparator; + +/** + * + * @author edwajohn + */ +public class Grammar { + + /** + * Name of the grammar from the grammar file. This is computed for you. + */ + final String grammarName; + /** + * The list of rules found in the input grammar file. This is computed for + * you. + */ + final ArrayList<Rule> rules; + /** + * The list of all symbols found in the input grammar file. This is computed + * for you. + */ + final ArrayList<String> symbols; + /** + * The set of all terminal symbols. To see if a symbol is terminal or not: + * terminals.contains(symbol). This is computed for you. + */ + final HashSet<String> terminals; + /** + * The set of all nonterminal symbols. To see if a symbol is nonterminal: + * nonterminals.contains(symbol). This is computed for you. + */ + final HashSet<String> nonterminals; + /** + * Start symbol that is provided for you. + */ + final String startSymbol; + /** + * Start production rule that is provided for you. + */ + final Rule startRule; + /** + * Maps non-terminals to all rules that have the non-terminal on the lhs. + */ + HashMap<String, ArrayList<Rule>> nt2rules; + + final HashMap<String, HashSet<String>> first; + final HashMap<String, HashSet<String>> follow; + final HashMap<Rule, HashSet<String>> firstPlus; + + public Grammar(String grammarFilename) throws IOException { + System.out.println("Reading grammar " + grammarFilename); + cfgparser.Grammar grammar = new cfgparser.Grammar(grammarFilename); + + this.grammarName = grammar.getName(); + this.rules = grammar.getRules(); + this.symbols = grammar.getSymbols(); + this.startSymbol = grammar.getStartSymbol(); + this.terminals = new HashSet<>(symbols); + this.nonterminals = new HashSet<>(); + + Rule start = null; + for (Rule rule : this.rules) { + if (rule.getLhs().equals(this.startSymbol)) { + start = rule; + } + } + if (start == null) { + throw new RuntimeException("Unexpected null start production rule"); + } + this.startRule = start; + + nt2rules = new HashMap<>(); + System.out.println("\nRules"); + for (Rule rule : rules) { + System.out.println(rule.toString()); + final String lhs = rule.getLhs(); + terminals.remove(lhs); + nonterminals.add(lhs); + if (!nt2rules.containsKey(lhs)) { + nt2rules.put(lhs, new ArrayList<>()); + } + nt2rules.get(lhs).add(rule); + } + + // Sort the list of symbols for easy reading of output. + symbols.sort(new SymbolComparator(terminals)); + + // Compute first, follow and firstPlus sets. + first = Util.computeFirst(symbols, terminals, rules); + follow = Util.computeFollow(symbols, terminals, rules, first); + firstPlus = Util.computeFirstPlus(symbols, terminals, rules, first, follow); + } + + public boolean isTerminal(String symbol) { + return terminals.contains(symbol); + } + + public boolean isNonterminal(String symbol) { + return nonterminals.contains(symbol); + } + + public Rule findRule(String lhs, List<String> rhs) { + for (Rule rule : rules) { + if (rule.getLhs().equals(lhs) && rule.getRhs().equals(rhs)) { + return rule; + } + } + return null; + } + + public Rule findRule(String s) { + List<String> symbols = Arrays.asList(s.split(" ")); + return findRule(symbols.get(0), symbols.subList(2, symbols.size())); + } +} diff --git a/Homework/cs5300/project-three/parser/Item.java b/Homework/cs5300/project-three/parser/Item.java new file mode 100644 index 0000000..1cdb134 --- /dev/null +++ b/Homework/cs5300/project-three/parser/Item.java @@ -0,0 +1,151 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser; + +import java.util.ArrayList; +import java.util.Objects; + +/** + * An LR(1) item. + * + */ +public class Item implements Comparable { + + private Rule rule; + /** + * Position of the stack top in the production's RHS. + */ + private int dot; + /** + * The lookahead symbol + */ + private String a; + + /** + * Creates a set of items given the rule and lookahead, one item per potential + * stack position placement. The number of item is the number of symbols on + * the rhs of the rule plus one. + * + * @param rule + * @param a + * @return + */ + static ArrayList<Item> create(Rule rule, String a) { + ArrayList<Item> ret = new ArrayList<>(); + final ArrayList<String> rhs = rule.getRhs(); + for (int i = 0; i < rhs.size(); ++i) { + ret.add(new Item(rule, i, a)); + } + ret.add(new Item(rule, rhs.size(), a)); + return ret; + } + + public Item(Rule rule, int dot, String a) { + this.rule = rule; + this.dot = dot; + this.a = a; + } + + public Rule getRule() { + return rule; + } + + public int getDot() { + return dot; + } + + /** + * @return the lookahead symbol + */ + public String getA() { + return a; + } + + public String getLookahead() { + return getA(); + } + + /** + * Returns the symbol following the stack pointer. Returns null if the pointer + * is at the end of the rhs list. + * + * @return + */ + public String getNextSymbol() { + if (dot < rule.getRhs().size()) { + return rule.getRhs().get(dot); + } + return null; + } + + public String getNextNextSymbol() { + if (dot < rule.getRhs().size() - 1) { + return rule.getRhs().get(dot + 1); + } + return null; + } + + public Item advance() { + return new Item(rule, dot + 1, a); + } + + @Override + public int hashCode() { + int hash = 7; + hash = 37 * hash + Objects.hashCode(this.rule); + hash = 37 * hash + this.dot; + hash = 37 * hash + Objects.hashCode(this.a); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final Item other = (Item) obj; + if (!Objects.equals(this.rule, other.rule)) { + return false; + } + if (this.dot != other.dot) { + return false; + } + if (!Objects.equals(this.a, other.a)) { + return false; + } + return true; + } + + @Override + public String toString() { + String ret = "[" + rule.getLhs() + " -> "; + final ArrayList<String> rhs = rule.getRhs(); + for (int i = 0; i < dot; ++i) { + ret += rhs.get(i) + " "; + } + ret += Character.toString((char) 0x25CF); + if (dot < rhs.size()) { + ret += " "; + } + for (int i = dot; i < rhs.size(); ++i) { + ret += rhs.get(i); + if (i < rhs.size() - 1) { + ret += " "; + } + } + ret += ", " + a + "]"; + return ret; + } + + @Override + public int compareTo(Object o) { + return toString().compareTo(o.toString()); + } + +} diff --git a/Homework/cs5300/project-three/parser/Main.java b/Homework/cs5300/project-three/parser/Main.java new file mode 100644 index 0000000..62ca0a3 --- /dev/null +++ b/Homework/cs5300/project-three/parser/Main.java @@ -0,0 +1,337 @@ +package parser; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.*; + +/** + * + */ +public class Main { + static final String dot = Character.toString((char) 0x25CF); + + /** + * @param args the command line arguments + * @throws java.io.IOException + */ + public static void main(String[] args) throws IOException { + Tests tests = new Tests(); + testClosure(tests); + testStates(tests); + testTables(tests); + testParser(tests); + + // ------------------------------------------------------------ + // Output number of tests that succeeded + // ------------------------------------------------------------ + System.out.println(tests.getSuccesses() + "/" + tests.getN() + + " tests succeeded"); + // return tests.getFailures(); + + } + + public static void testClosure(Tests tests) throws FileNotFoundException, IOException { + // TODO: uncomment tests as you develop code + { + // TODO: Become familiar with the Grammar class. You will use it a lot. + Grammar grammar = new Grammar("data/Simple.cfg"); + + // Find the closure of [N -> ● X, $] + Rule rule = grammar.findRule("N -> X"); + State state = Parser.computeClosure(new Item(rule, 0, Util.EOF), grammar); + state.setName(0); + tests.test(state, "0: [[N -> ● X, $]]"); + + // Find the closure of [N -> ● N X, $] + rule = grammar.findRule("N -> N X"); + state = Parser.computeClosure(new Item(rule, 0, Util.EOF), grammar); + state.setName(0); + // [[N -> ● N X, $], [N -> ● N X, X], [N -> ● X, X]] + tests.test(state.size(), 3); + + // Find the start state + Item head = new Item(grammar.startRule, 0, Util.EOF); + state = Parser.computeClosure(head, grammar); + // [[G -> ● N, $], [N -> ● N X, $], [N -> ● N X, X], [N -> ● X, X], [N -> ● X, + // $]] + tests.test(state.size(), 5); + } + { + Grammar grammar = new Grammar("data/Paren.cfg"); + + // Find the closure of [list -> list ● pair, $] + Rule rule = grammar.findRule("list -> list pair"); + State state = Parser.computeClosure(new Item(rule, 1, Util.EOF), grammar); + // [[list -> list ● pair, $], [pair -> ● OPAREN list CPAREN, $], [pair -> ● + // OPAREN CPAREN, $]] + tests.test(state.size(), 3); + + // Find the closure of [pair -> OPAREN ● list CPAREN, $] + rule = grammar.findRule("pair -> OPAREN list CPAREN"); + state = Parser.computeClosure(new Item(rule, 1, Util.EOF), grammar); + // [[pair -> OPAREN ● list CPAREN, $], [list -> ● list pair, CPAREN] + // [list -> ● list pair, OPAREN], [list -> ● pair, OPAREN], [pair -> ● OPAREN + // list CPAREN, OPAREN] + // [pair -> ● OPAREN CPAREN, OPAREN], [list -> ● pair, CPAREN], [pair -> ● + // OPAREN list CPAREN, CPAREN] + // [pair -> ● OPAREN CPAREN, CPAREN]] + tests.test(state.size(), 9); + + // Find the start state + Item head = new Item(grammar.startRule, 0, Util.EOF); + state = Parser.computeClosure(head, grammar); + // [[goal -> ● list, $], [list -> ● list pair, $], [list -> ● list pair, OPAREN] + // [list -> ● pair, OPAREN], [pair -> ● OPAREN list CPAREN, OPAREN], [pair -> ● + // OPAREN CPAREN, OPAREN] + // [list -> ● pair, $], [pair -> ● OPAREN list CPAREN, $], [pair -> ● OPAREN + // CPAREN, $]] + tests.test(state.size(), 9); + } + { + Grammar grammar = new Grammar("data/Expr.cfg"); + + // Find the start state + Item head = new Item(grammar.startRule, 0, Util.EOF); + State state = Parser.computeClosure(head, grammar); + // [[goal -> ● expr, $], [expr -> ● expr PLUS term, $], [expr -> ● expr PLUS + // term, PLUS], [expr -> ● expr MINUS term, PLUS], [expr -> ● expr PLUS term, + // MINUS], [expr -> ● term, PLUS], [term -> ● term MULTIPLY factor, PLUS], [term + // -> ● term MULTIPLY factor, MULTIPLY], [term -> ● term DIVIDE factor, + // MULTIPLY], [term -> ● term MULTIPLY factor, DIVIDE], [term -> ● factor, + // MULTIPLY], [factor -> ● OPAREN expr CPAREN, MULTIPLY], [factor -> ● INT, + // MULTIPLY], [factor -> ● FLOAT, MULTIPLY], [factor -> ● IDENTIFIER, MULTIPLY], + // [term -> ● term DIVIDE factor, DIVIDE], [term -> ● factor, DIVIDE], [factor + // -> ● OPAREN expr CPAREN, DIVIDE], [factor -> ● INT, DIVIDE], [factor -> ● + // FLOAT, DIVIDE], [factor -> ● IDENTIFIER, DIVIDE], [term -> ● term DIVIDE + // factor, PLUS], [term -> ● factor, PLUS], [factor -> ● OPAREN expr CPAREN, + // PLUS], [factor -> ● INT, PLUS], [factor -> ● FLOAT, PLUS], [factor -> ● + // IDENTIFIER, PLUS], [expr -> ● expr MINUS term, MINUS], [expr -> ● term, + // MINUS], [term -> ● term MULTIPLY factor, MINUS], [term -> ● term DIVIDE + // factor, MINUS], [term -> ● factor, MINUS], [factor -> ● OPAREN expr CPAREN, + // MINUS], [factor -> ● INT, MINUS], [factor -> ● FLOAT, MINUS], [factor -> ● + // IDENTIFIER, MINUS], [expr -> ● expr MINUS term, $], [expr -> ● term, $], + // [term -> ● term MULTIPLY factor, $], [term -> ● term DIVIDE factor, $], [term + // -> ● factor, $], [factor -> ● OPAREN expr CPAREN, $], [factor -> ● INT, $], + // [factor -> ● FLOAT, $], [factor -> ● IDENTIFIER, $]] + tests.test(state.size(), 45); + } + } + + public static void testStates(Tests tests) throws FileNotFoundException, IOException { + { + Parser parser = new Parser("data/Simple.cfg"); + States states = parser.getStates(); + tests.test(states.size(), 4); + } + { + Parser parser = new Parser("data/Paren.cfg"); + States states = parser.getStates(); + tests.test(states.size(), 14); + } + { + Parser parser = new Parser("data/Expr.cfg"); + States states = parser.getStates(); + tests.test(states.size(), 34); + } + } + + public static void testTables(Tests tests) throws FileNotFoundException, IOException { + { + Parser parser = new Parser("data/Simple.cfg"); + String actionTable = parser.actionTableToString(); + tests.test(countMatches(actionTable, 'S'), 2); // 2 shifts + tests.test(countMatches(actionTable, 'R'), 4); // 4 reduces + tests.test(countMatches(actionTable, "acc"), 1); // 1 accept + } + { + Parser parser = new Parser("data/Paren.cfg"); + String actionTable = parser.actionTableToString(); + tests.test(countMatches(actionTable, 'S'), 10); // 10 shifts + tests.test(countMatches(actionTable, 'R'), 18); // 18 reduces (2 of the Rs are in the header) + tests.test(countMatches(actionTable, "acc"), 1); // 1 accept + } + { + Parser parser = new Parser("data/Expr.cfg"); + String actionTable = parser.actionTableToString(); + tests.test(countMatches(actionTable, 'S'), 66); + tests.test(countMatches(actionTable, 'R'), 91); + tests.test(countMatches(actionTable, "acc"), 1); + } + } + + public static void testParser(Tests tests) throws FileNotFoundException, IOException { + { + Parser parser = new Parser("data/Simple.cfg"); + + String s = "xxx"; + try { + List<Action> actions = parser.parseFromString(s); + tests.test(countMatches(actions.toString(), 'S'), 3); // 3 shift actions + tests.test(countMatches(actions.toString(), 'R'), 3); // 3 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + s + ": " + e.getMessage()); + } + + s = "xxxxx"; + try { + List<Action> actions = parser.parseFromString(s); + tests.test(countMatches(actions.toString(), 'S'), 5); // 5 shift actions + tests.test(countMatches(actions.toString(), 'R'), 5); // 5 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + s + ": " + e.getMessage()); + } + + s = ""; + try { + List<Action> actions = parser.parseFromString(s); + tests.addFailure("Should have failed to parse"); + } catch (ParserException e) { + } + } + + { + Parser parser = new Parser("data/Paren.cfg"); + + String fn = "data/paren0.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 2); // 2 shift actions + tests.test(countMatches(actions.toString(), 'R'), 2); // 2 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/paren1.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 20); // 20 shift actions + tests.test(countMatches(actions.toString(), 'R'), 20); // 20 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/paren2.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + fn = "data/paren3.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + fn = "data/paren4.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 10); // 10 shift actions + tests.test(countMatches(actions.toString(), 'R'), 10); // 10 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + } + { + Parser parser = new Parser("data/Expr.cfg"); + + String fn = "data/expr1.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 7); // 7 shift actions + tests.test(countMatches(actions.toString(), 'R'), 10); // 10 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/expr2.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + fn = "data/expr3.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 1); // 1 shift actions + tests.test(countMatches(actions.toString(), 'R'), 3); // 3 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/expr4.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 15); // 15 shift actions + tests.test(countMatches(actions.toString(), 'R'), 22); // 22 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/expr5.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + fn = "data/expr6.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + } + + { + // TODO: To pass these tests you must implement data/Tiny.cfg + // according to the grammar given in data/tinyGrammar.pdf. + // See notes in data/Tiny.cfg (be sure to read all of them!). + Parser parser = new Parser("data/Tiny.cfg"); + + String fn = "data/tiny1.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 11); // 11 shift actions + tests.test(countMatches(actions.toString(), 'R'), 25); // 25 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/tiny2.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.test(countMatches(actions.toString(), 'S'), 27); // 27 shift actions + tests.test(countMatches(actions.toString(), 'R'), 55); // 55 reduce actions + } catch (ParserException e) { + tests.addFailure("Failed to parse " + fn + ": " + e.getMessage()); + } + + fn = "data/tiny3.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + fn = "data/tiny4.dat"; + try { + List<Action> actions = parser.parseFromFile(fn); + tests.addFailure("Should have failed to parse " + fn); + } catch (ParserException e) { + } + + } + } + + private static int countMatches(String s, char c) { + return (int) s.chars().filter(ch -> ch == c).count(); + } + + private static int countMatches(String s, String c) { + return s.split(c, -1).length - 1; + } +}
\ No newline at end of file diff --git a/Homework/cs5300/project-three/parser/Parser.java b/Homework/cs5300/project-three/parser/Parser.java new file mode 100644 index 0000000..67d723a --- /dev/null +++ b/Homework/cs5300/project-three/parser/Parser.java @@ -0,0 +1,341 @@ +/* + * Look for TODO comments in this file for suggestions on how to implement + * your parser. + */ +package parser; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +import javax.lang.model.util.ElementScanner6; + +import lexer.ExprLexer; +import lexer.ParenLexer; +import lexer.SimpleLexer; +import lexer.TinyLexer; +import org.antlr.v4.runtime.*; + +/** + * + */ +public class Parser { + + final Grammar grammar; + + /** + * All states in the parser. + */ + private final States states; + + /** + * Action table for bottom-up parsing. Accessed as + * actionTable.get(state).get(terminal). You may replace + * the Integer with a State class if you choose. + */ + private final HashMap<Integer, HashMap<String, Action>> actionTable; + /** + * Goto table for bottom-up parsing. Accessed as + * gotoTable.get(state).get(nonterminal). + * You may replace the Integers with State classes if you choose. + */ + private final HashMap<Integer, HashMap<String, Integer>> gotoTable; + + public Parser(String grammarFilename) throws IOException { + actionTable = new HashMap<>(); + gotoTable = new HashMap<>(); + + grammar = new Grammar(grammarFilename); + + states = new States(); + + initializeStates(); + constructActionTable(); + constructGotoTable(); + } + + private int findState(State state) { + for (State f_state : states.getStateList()) + if (f_state.equals(state)) + return f_state.getName(); + return -1; + } + + private void constructActionTable() { + for (State state : states.getStateList()) { + int name = state.getName(); + actionTable.computeIfAbsent(name, k -> new HashMap<String, Action>()); + + for (Item item : state.getItems()) { + String nextSymbol = item.getNextSymbol(); + + if (nextSymbol != null && grammar.isTerminal(nextSymbol)) + actionTable.get(name).put(nextSymbol, Action.createShift(findState(Parser.GOTO(state, nextSymbol, grammar)))); + else if (nextSymbol == null) { + Rule rule = item.getRule(); + if (rule.equals(grammar.startRule) && item.getLookahead() == Util.EOF) + actionTable.get(name).put(Util.EOF, Action.createAccept()); + else + actionTable.get(name).put(item.getLookahead(), Action.createReduce(item.getRule())); + } + } + } + } + + private void constructGotoTable() { + for (State state : states.getStateList()) { + int name = state.getName(); + gotoTable.computeIfAbsent(name, k -> new HashMap<String, Integer>()); + for (String nonTerminal : grammar.nonterminals) { + int goTo = findState(Parser.GOTO(state, nonTerminal, grammar)); + if (goTo != -1) + gotoTable.get(name).put(nonTerminal, goTo); + } + } + } + + private void initializeStates() { + int stateId = 0; + + State closure = Parser.computeClosure(new Item(grammar.startRule, 0, Util.EOF), grammar); + closure.setName(stateId++); + states.addState(closure); + + boolean added = true; + while (added) { + added = false; + + ArrayList<State> stateList = new ArrayList<>(states.getStateList()); + for (State state : stateList) + for (String symbol : grammar.symbols) { + State goTo = Parser.GOTO(state, symbol, grammar); + if (goTo.getItems().size() > 0 && states.addState(goTo)) { + goTo.setName(stateId++); + added = true; + } + } + } + } + + public States getStates() { + return states; + } + + static public State computeClosure(Item I, Grammar grammar) { + State closure = new State(I); + + boolean added = true; + while (added) { + added = false; + + List<Item> items = new ArrayList<Item>(closure.getItems()); + + for (Item item : items) { + String symbol = item.getNextSymbol(); + + if (grammar.isNonterminal(symbol)) { + List<Rule> symbolRules = grammar.rules.stream().filter(rule -> rule.getLhs().equals(symbol)) + .collect(Collectors.toList()); + + ArrayList<String> firstOperand = new ArrayList<>(); + if (item.getNextNextSymbol() != null) { + List<String> rhs = item.getRule().getRhs(); + firstOperand.addAll(rhs.subList(item.getDot() + 1, rhs.size())); + } + firstOperand.add(item.getLookahead()); + + HashSet<String> firstSet = new HashSet<>(); + for (String firstSymbol : firstOperand) { + if (grammar.first.containsKey(firstSymbol)) { + firstSet.addAll(grammar.first.get(firstSymbol)); + if (!grammar.first.get(firstSymbol).contains(Util.EPSILON)) + break; + } else if (firstSymbol.equals(Util.EOF)) { + firstSet.add(firstSymbol); + } + } + + for (Rule rule : symbolRules) + for (String terminal : firstSet) + added = closure.addItem(new Item(rule, 0, terminal)) || added; + } + } + } + + return closure; + } + + // This returns a new state that represents the transition from + // the given state on the symbol X. + static public State GOTO(State state, String X, Grammar grammar) { + State gotoState = new State(); + + for (Item item : state.getItems()) + if (item.getNextSymbol() != null && item.getNextSymbol().equals(X)) + gotoState.addItem(new Item(item.getRule(), item.getDot() + 1, item.getLookahead())); + + List<Item> gotoItems = new ArrayList<>(gotoState.getItems()); + for (Item gotoItem : gotoItems) + for (Item closureItem : Parser.computeClosure(gotoItem, grammar).getItems()) + gotoState.addItem(closureItem); + + return gotoState; + } + + // You will want to use StringBuilder. Another useful method will be + // String.format: for + // printing a value in the table, use + // String.format("%8s", value) + // How much whitespace you have shouldn't matter with regard to the tests, but + // it will + // help you debug if you can format it nicely. + public String actionTableToString() { + StringBuilder builder = new StringBuilder(); + + List<String> terminalsPlusEof = new ArrayList<String>(grammar.terminals); + terminalsPlusEof.add(Util.EOF); + + builder.append("state"); + for (String terminal : terminalsPlusEof) + builder.append(String.format("%8s", terminal)); + builder.append("\n"); + + for (State state : states.getStateList()) { + int stateName = state.getName(); + HashMap<String, Action> actionRow = actionTable.get(stateName); + builder.append(String.format("%5s", stateName)); + + for (String terminal : terminalsPlusEof) { + Action action = actionRow.get(terminal); + builder.append(String.format("%8s", action != null ? action.toString() : "")); + } + builder.append("\n"); + } + return builder.toString(); + } + + // You will want to use StringBuilder. Another useful method will be + // String.format: for + // printing a value in the table, use + // String.format("%8s", value) + // How much whitespace you have shouldn't matter with regard to the tests, but + // it will + // help you debug if you can format it nicely. + public String gotoTableToString() { + StringBuilder builder = new StringBuilder(); + + builder.append("state"); + for (String nonTerminal : grammar.nonterminals) + builder.append(String.format("%8s", nonTerminal)); + builder.append("\n"); + + for (State state : states.getStateList()) { + int stateName = state.getName(); + HashMap<String, Integer> gotoTableRow = gotoTable.get(stateName); + builder.append(String.format("%5s", stateName)); + + for (String nonTerminal : grammar.nonterminals) { + if (gotoTableRow.containsKey(nonTerminal)) + builder.append(String.format("%8s", gotoTableRow.get(nonTerminal))); + else + builder.append(String.format("%8s", "")); + } + + builder.append("\n"); + } + return builder.toString(); + } + + // You should return a list of the actions taken. + public List<Action> parse(Lexer scanner) throws ParserException { + // tokens is the output from the scanner. It is the list of tokens + // scanned from the input file. + // To get the token type: v.getSymbolicName(t.getType()) + // To get the token lexeme: t.getText() + ArrayList<? extends Token> tokens = new ArrayList<>(scanner.getAllTokens()); + Vocabulary v = scanner.getVocabulary(); + + Stack<String> input = new Stack<>(); + Collections.reverse(tokens); + input.add(Util.EOF); + for (Token t : tokens) { + input.push(v.getSymbolicName(t.getType())); + } + Collections.reverse(tokens); + + List<Action> actions = new ArrayList<>(); + + Stack<State> stack = new Stack<>(); + stack.push(states.getState(0)); + + String currentInput = input.pop(); + while (true) { + State s = stack.peek(); + Action action = actionTable.get(s.getName()).get(currentInput); + + if (action == null) + throw ParserException.create(tokens, input.size()); + + actions.add(action); + + if (action.isShift()) { + stack.push(states.getState(action.getState())); + currentInput = input.pop(); + } else if (action.isReduce()) { + Rule rule = action.getRule(); + for (int i = 0; i < rule.getRhs().size(); i++) + stack.pop(); + + s = stack.peek(); + stack.push(states.getState(gotoTable.get(s.getName()).get(rule.getLhs()))); + } else if (action.isAccept()) { + break; + } else { + throw ParserException.create(tokens, input.size()); + } + } + + return actions; + } + + // ------------------------------------------------------------------- + // Convenience functions + // ------------------------------------------------------------------- + + public List<Action> parseFromFile(String filename) throws IOException, ParserException { + // System.out.println("\nReading input file " + filename + "\n"); + final CharStream charStream = CharStreams.fromFileName(filename); + Lexer scanner = scanFile(charStream); + return parse(scanner); + } + + public List<Action> parseFromString(String program) throws ParserException { + Lexer scanner = scanFile(CharStreams.fromString(program)); + return parse(scanner); + } + + private Lexer scanFile(CharStream charStream) { + // We use ANTLR's scanner (lexer) to produce the tokens. + Lexer scanner = null; + switch (grammar.grammarName) { + case "Simple": + scanner = new SimpleLexer(charStream); + break; + case "Paren": + scanner = new ParenLexer(charStream); + break; + case "Expr": + scanner = new ExprLexer(charStream); + break; + case "Tiny": + scanner = new TinyLexer(charStream); + break; + default: + System.out.println("Unknown scanner"); + break; + } + + return scanner; + } + +} diff --git a/Homework/cs5300/project-three/parser/ParserException.java b/Homework/cs5300/project-three/parser/ParserException.java new file mode 100644 index 0000000..1c32080 --- /dev/null +++ b/Homework/cs5300/project-three/parser/ParserException.java @@ -0,0 +1,81 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser; + +import java.util.ArrayList; +import org.antlr.v4.runtime.Token; + +/** + * + * @author edwajohn + */ +public class ParserException extends Exception { + + /** + * Create a ParserException. This is a convenience factory method to easily + * format your output to meet project specification. + * + * @param words the words being parsed + * @param i the index to the current word being parsed + * @return + */ + public static ParserException create(final ArrayList<? extends Token> words, final int i) { + String msg; + if (i < words.size()) { + msg = "Error parsing token \"" + words.get(i).getText() + "\" in \n "; + int preLen = 4; + for (int j = 0; j < i; ++j) { + Token token = words.get(j); + msg = msg + token.getText() + " "; + preLen += token.getText().length() + 1; + } + final int curLen = words.get(i).getText().length(); + msg = msg + words.get(i).getText(); + if (i < words.size() - 1) { + msg = msg + " "; + } + for (int j = i + 1; j < words.size(); ++j) { + Token token = words.get(j); + msg = msg + token.getText(); + if (j < words.size() - 1) { + msg = msg + " "; + } + } + + msg = msg + "\n"; + for (int j = 0; j < preLen; ++j) { + msg += " "; + } + for (int j = 0; j < curLen; ++j) { + msg += "^"; + } + } else { + msg = "Unexpectedly reached end of file in\n"; + for (int j = 0; j < words.size(); ++j) { + Token token = words.get(j); + msg = msg + token.getText(); + if (j < words.size() - 1) { + msg = msg + " "; + } + } + + } + return new ParserException(msg); + } + + /** + * Constructor. You will most likely NOT use this constructor directly, but + * rather use the convenience factory method create(). You may choose to use + * this constructor, however, if your data structures are set up differently + * than what the factory method expects. + * + * @param msg + */ + public ParserException(String msg) { + super(msg); + } + +} diff --git a/Homework/cs5300/project-three/parser/Rule.java b/Homework/cs5300/project-three/parser/Rule.java new file mode 100644 index 0000000..b251e84 --- /dev/null +++ b/Homework/cs5300/project-three/parser/Rule.java @@ -0,0 +1,93 @@ +/* + * Do not modify this file. + */ +package parser; + +import java.util.ArrayList; +import java.util.Objects; + +/** + * + */ +public class Rule { + + private int name; + private final String lhs; + private ArrayList<String> rhs; + + public Rule(String lhs) { + this.name = 0; + this.lhs = lhs; + rhs = new ArrayList<>(); + } + + public void setName(int name) { + this.name = name; + } + + public int getName() { + return this.name; + } + + @Override + public String toString() { + String ret = "R" + name + " " + lhs + " -> "; + for (String symbol : getRhs()) { + ret = ret + symbol + " "; + } + return ret; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 37 * hash + Objects.hashCode(this.lhs); + hash = 37 * hash + this.name; + for (int i = 0; i < this.rhs.size(); ++i) { + hash = 37 * hash + Objects.hashCode(this.rhs.get(i)); + } + return hash; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final Rule other = (Rule) obj; + if (!this.lhs.equals(other.lhs)) { + return false; + } + if (this.name != other.name) { + return false; + } + if (this.rhs.size() != other.rhs.size()) { + return false; + } + for (int i = 0; i < this.rhs.size(); ++i) { + if (!this.rhs.get(i).equals(other.rhs.get(i))) { + return false; + } + } + return true; + } + + public String getLhs() { + return lhs; + } + + public ArrayList<String> getRhs() { + if (rhs.isEmpty()) { + rhs.add("EPSILON"); + } + return rhs; + } + + public void addRhs(String symbol) { + rhs.add(symbol); + } + +} diff --git a/Homework/cs5300/project-three/parser/State.java b/Homework/cs5300/project-three/parser/State.java new file mode 100644 index 0000000..90c9c33 --- /dev/null +++ b/Homework/cs5300/project-three/parser/State.java @@ -0,0 +1,94 @@ +package parser; + +import java.util.*; + +public class State implements Comparable<State> { + private Set<Item> itemSet; + private List<Item> items; + private int name; + + public State(int name) { + this.itemSet = new HashSet<>(); + this.items = new ArrayList<>(); + this.name = name; + } + + public State() { + this(0); + } + + public State(Item item) { + this(); + addItem(item); + } + + public void setName(int name) { + this.name = name; + } + + public int getName() { + return this.name; + } + + public List<Item> getItems() { + return this.items; + } + + public int size() { + return items.size(); + } + + /* + * true if the item was added, false if it was already in the set + */ + public boolean addItem(Item item) { + if (!itemSet.contains(item)) { + itemSet.add(item); + items.add(item); + return true; + } + return false; + } + + @Override + public int hashCode() { + int hash = 7; + ArrayList<Item> sortedList = new ArrayList<>(items); + sortedList.sort(Comparator.comparingInt(Item::hashCode)); + for (Item item : sortedList) { + hash = 37 * hash + Objects.hashCode(item); + } + return hash; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final State other = (State) obj; + if (items.size() != other.items.size()) { + return false; + } + for (Item item : items) { + if (!other.itemSet.contains(item)) { + return false; + } + } + return true; + } + + @Override + public String toString() { + return this.name + ": " + items.toString(); + } + + @Override + public int compareTo(State o) { + return Integer.valueOf(this.name).compareTo(Integer.valueOf(o.name)); + } + +} diff --git a/Homework/cs5300/project-three/parser/States.java b/Homework/cs5300/project-three/parser/States.java new file mode 100644 index 0000000..7002f53 --- /dev/null +++ b/Homework/cs5300/project-three/parser/States.java @@ -0,0 +1,44 @@ +package parser; + +import java.util.*; + +public class States { + private Set<State> stateSet; + private List<State> states; + private int id = 0; + + public States() { + this.stateSet = new HashSet<>(); + this.states = new ArrayList<>(); + } + + public int size() { + return this.states.size(); + } + + public boolean addState(State state) { + if (!this.stateSet.contains(state)) { + this.stateSet.add(state); + this.states.add(state); + return true; + } + return false; + } + + public List<State> getStateList() { + return states; + } + + public State getState(int name) { + if (name >= states.size()) { + return null; + } + return states.get(name); + } + + @Override + public String toString() { + return states.toString(); + } + +} diff --git a/Homework/cs5300/project-three/parser/Tests.java b/Homework/cs5300/project-three/parser/Tests.java new file mode 100644 index 0000000..73732af --- /dev/null +++ b/Homework/cs5300/project-three/parser/Tests.java @@ -0,0 +1,69 @@ +package parser; + +/** + * + * @author edwajohn + */ +public class Tests { + + private int N = 0; + private int failures = 0; + + public void test(boolean b) { + N++; + if (!b) { + System.err.println("Failed test " + N); + failures++; + } + } + + public void test(boolean b, String message) { + test(b); + if (!b) { + System.err.println(message); + } + } + + public void test(Object a, Object target) { + N++; + if (a == null && target == null) { + return; + } + if (a == null || target == null || !a.equals(target)) { + System.err.println("Failed test " + N); + System.err.println(" " + a + " not equal to target " + target); + failures++; + } + } + + public void test(Object a, String target) { + N++; + if (a == null && target == null) { + return; + } + if (a == null || target == null || !a.toString().equals(target)) { + System.err.println("Failed test " + N); + System.err.println(" " + a + " not equal to target " + target); + failures++; + } + } + + public void addFailure(String msg) { + N++; + System.err.println("Failed test " + N); + System.err.println(msg); + failures++; + } + + public int getN() { + return N; + } + + public int getSuccesses() { + return N - failures; + } + + public int getFailures() { + return failures; + } +} diff --git a/Homework/cs5300/project-three/parser/Util.java b/Homework/cs5300/project-three/parser/Util.java new file mode 100644 index 0000000..fe7d9ab --- /dev/null +++ b/Homework/cs5300/project-three/parser/Util.java @@ -0,0 +1,132 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +/** + * + * @author edwajohn + */ +public class Util { + + public static final String EPSILON = "EPSILON"; + public static final String EOF = "$"; + + public static HashMap<String, HashSet<String>> computeFirst(ArrayList<String> symbols, HashSet<String> terminals, ArrayList<Rule> rules) { + HashMap<String, HashSet<String>> first = new HashMap<>(); + for (String symbol : symbols) { + first.put(symbol, new HashSet<>()); + } + for (String terminal : terminals) { + first.get(terminal).add(terminal); + } + // Algorithm 3.7 + boolean changed = true; + while (changed) { + changed = false; + for (Rule rule : rules) { + ArrayList<String> rhsSymbols = rule.getRhs(); + HashSet<String> rhs = new HashSet<>(); + final String B0 = rhsSymbols.get(0); + rhs.addAll(noEpsilon(first.get(B0))); + boolean stop = !first.get(B0).contains(EPSILON); + for (int i = 0; !stop && i < rhsSymbols.size() - 1; ++i) { + final String B = rhsSymbols.get(i + 1); + rhs.addAll(noEpsilon(first.get(B0))); + stop = !first.get(B).contains(EPSILON); + } + if (!stop) { + rhs.add(EPSILON); + } + final String A = rule.getLhs(); + if (!first.get(A).containsAll(rhs)) { + first.get(A).addAll(rhs); + changed = true; + } + } + } +// System.out.println("\n"); +// for (String symbol : symbols) { +// final String A = symbol; +// System.out.println("FIRST(" + A + ") = " + first.get(A)); +// } + return first; + } + + public static HashMap<String, HashSet<String>> computeFollow( + ArrayList<String> symbols, HashSet<String> terminals, ArrayList<Rule> rules, + HashMap<String, HashSet<String>> first) { + HashMap<String, HashSet<String>> follow = new HashMap<>(); + for (String symbol : symbols) { + follow.put(symbol, new HashSet<>()); + } + follow.get(rules.get(0).getLhs()).add(EOF); + boolean changed = true; + while (changed) { + changed = false; + for (Rule rule : rules) { + HashSet<String> trailer = new HashSet<>(follow.get(rule.getLhs())); + final ArrayList<String> rhsSymbols = rule.getRhs(); + for (int i = rhsSymbols.size() - 1; i >= 0; --i) { + final String Bi = rhsSymbols.get(i); + if (!terminals.contains(Bi)) { + if (!follow.get(Bi).containsAll(trailer)) { + follow.get(Bi).addAll(trailer); + changed = true; + } + if (first.get(Bi).contains(EPSILON)) { + trailer.addAll(noEpsilon(first.get(Bi))); + } else { + trailer = first.get(Bi); + } + } else { + trailer = first.get(Bi); + } + } + } + } +// System.out.println("\n"); +// for (String symbol : symbols) { +// if (!terminals.contains(symbol)) { +// System.out.println("FOLLOW(" + symbol + ") = " + follow.get(symbol)); +// } +// } + return follow; + } + + public static HashMap<Rule, HashSet<String>> computeFirstPlus( + ArrayList<String> symbols, HashSet<String> terminals, ArrayList<Rule> rules, + HashMap<String, HashSet<String>> first, HashMap<String, HashSet<String>> follow) { + HashMap<Rule, HashSet<String>> firstPlus = new HashMap<>(); + for (Rule rule : rules) { + final String A = rule.getLhs(); + final String B = rule.getRhs().get(0); + HashSet<String> firstPlusA = new HashSet<>(); + if (!first.get(B).contains(EPSILON)) { + firstPlusA.addAll(first.get(B)); + } else { + firstPlusA.addAll(first.get(B)); + firstPlusA.addAll(follow.get(A)); + } + firstPlus.put(rule, firstPlusA); + } +// System.out.println("\n"); +// for (Rule rule : rules) { +// System.out.println("FIRST+(" + rule + ") = " + firstPlus.get(rule)); +// } + return firstPlus; + } + + private static HashSet<String> noEpsilon(final HashSet<String> set) { + HashSet<String> s = new HashSet<>(set); + s.remove(EPSILON); + return s; + } +} diff --git a/Homework/cs5300/project-three/parser/util/SymbolComparator.java b/Homework/cs5300/project-three/parser/util/SymbolComparator.java new file mode 100644 index 0000000..e131c79 --- /dev/null +++ b/Homework/cs5300/project-three/parser/util/SymbolComparator.java @@ -0,0 +1,38 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package parser.util; + +import java.util.Comparator; +import java.util.HashSet; + +/** + * + * @author edwajohn + */ +public class SymbolComparator implements Comparator<String> { + + private final HashSet<String> terminals; + + public SymbolComparator(HashSet<String> terminals) { + this.terminals = terminals; + } + + @Override + public int compare(String o1, String o2) { + // Slightly less efficient code for clarity. + if (terminals.contains(o1) && terminals.contains(o2)) { + return o1.compareTo(o2); + } + if (terminals.contains(o1)) { + return 1; + } + if (terminals.contains(o2)) { + return -1; + } + return o1.compareTo(o2); + } + +} |
