Merge branch 'Reza' into 'master' (603db61f) · Commits · EC504 Spring 2024 Group Projects / Group6

.gitignore

+6 −2

Original line number	Diff line number	Diff line
		@@ -5,6 +5,10 @@ CheckerCorrector/*.jar
		CheckerCorrector/test.*
		CheckerCorrector/*.txt
		CheckerCorrector/*db
		!CheckerCorrector/manifest*
		CheckerCorrector/corrector
		CheckerCorrector/checker
		Crawler/target/classes/ScratchCrawler.class
		Crawler/target/classes/crawledData.txt
		*.java
		!CheckerCorrector/manifest*
		!CheckerCorrector/*/.java

CheckerCorrector/Checker.java

+37 −1

Original line number	Diff line number	Diff line
		@@ -3,7 +3,9 @@ import java.security.NoSuchAlgorithmException;
		import java.sql.SQLException;
		import java.util.HashMap;
		import java.util.List;
		import javax.swing.*;

		import GUI.*;
		import HashTableMaker.HashTableMaker;
		import DirectedGraph.BasicGraph;
		import DirectedGraph.DirectedGraph;
		@@ -21,7 +23,41 @@ public class Checker {
		DBinterface dbInterface = new DBinterface();
		DirectedGraph<State> graph = basicGraphClass.getGraph();
		JsonMaker jsonMaker = JsonMaker.create();
		if(argPars.isUpdateHashTable()){

		if(argPars.isValidateUpdates()){
		SwingUtilities.invokeLater(new Runnable() {
		@Override
		public void run() {
		new WordRoleUpdater();
		}
		});
		}else if(argPars.isUpdateToken()){
		dbInterface.readDataFromDatabase();
		if(argPars.isCheckFile()){
		SentenceExtractor extractor = SentenceExtractor.of(argPars.getFileName());
		List<String> extractedSentences = extractor.getSentences();
		int i = 0;
		int n = extractedSentences.size();
		int cntUpdate = 0;
		for (String sentence : extractedSentences) {
		i++;
		//dbInterface.updateTokenInDatabase(sentence.toLowerCase(), graph);
		PhraseExtractor extractorPhrase = PhraseExtractor.fromSentence(sentence, 3, 5);
		List<String> phrases = extractorPhrase.getPhrases();
		for (String phrase : phrases) {
		cntUpdate += dbInterface.updateTokenInDatabase(phrase.toLowerCase(), graph);
		}
		ProgressBar.printProgress(((double)i)/((double)n));
		}
		System.out.println("\n-------------------------------------------\n"+ "Number of update: "+ cntUpdate);
		}else if(argPars.isCheckSentence()){
		//dbInterface.updateTokenInDatabase(argPars.getSentence().toLowerCase(), graph);
		for (String phrase : PhraseExtractor.fromSentence(argPars.getSentence(),3, 5).getPhrases()) {
		dbInterface.updateTokenInDatabase(phrase.toLowerCase(), graph);
		}
		}
		dbInterface.updateDatabase();
		}else if(argPars.isUpdateHashTable()){
		if(argPars.isCheckFile()){
		SentenceExtractor extractor = SentenceExtractor.of(argPars.getFileName());
		List<String> extractedSentences = extractor.getSentences();

CheckerCorrector/DBinterface/DBinterface.java

+156 −0

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@ package DBinterface;

		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.HashMap;
		import java.util.List;

		import DirectedGraph.DirectedGraph;
		@@ -10,6 +11,7 @@ import java.sql.*;
		import StateMachine.*;
		import TypoCorrector.TypoCorrector;
		import util.TwoListStruct;
		import util.StringProcessor;


		public class DBinterface {
		@@ -215,4 +217,158 @@ public class DBinterface {
		}
		return new String();
		}

		public int updateTokenInDatabase(String sentence, DirectedGraph<State> graph){
		//System.out.print(sentence + " \|");
		sentence = StringProcessor.processString(sentence);

		if(sentence.equals(""))
		return 0;
		StateMachine SM = new StateMachine();
		sentence = sentence.replaceAll("\\p{Punct}", " $0");
		//System.out.println(sentence);
		String[] tokens = sentence.split("\\s+");
		String[] tokensCopy = tokens.clone();
		List<Boolean> missFlag = new ArrayList();
		List<String> tokenList = new ArrayList<>(Arrays.asList(tokensCopy));
		//String url = "jdbc:sqlite:./SQLite/mydatabase.db";
		//String urlinsert = "jdbc:sqlite:./SQLite/newdatabase.db";
		String dicFileName = "./SQLite/smallDic.txt";
		//String url = "jdbc:sqlite:./SQLite/newdatabase.db";
		TypoCorrector typoChecker = TypoCorrector.of(dicFileName);
		int initialConf = 0;
		int cntMiss = 0;
		// Lookup each token in the database and categorize it
		for (int i = 0; i < tokens.length; i++) {
		String token = tokens[i];
		String role;
		// System.out.print("\nBefor token: "+tokens[i]+"\| ");
		if (wordRolesMap.containsKey(token)) {
		role = wordRolesMap.get(token);
		//////System.out.print("first try: " + token + " -> " + role);
		// System.out.println("role: " + role);
		tokens[i] = role;
		missFlag.add(false);
		}else{
		String tokenCorrected = new String();
		tokenCorrected = typoChecker.closestWord(token);
		if(!tokenCorrected.equals(token))
		initialConf += 5;
		// ////System.out.print("Corrected token: " + token + " -> " + tokenCorrected);

		if (wordRolesMap.containsKey(tokenCorrected)) {
		tokenList.set(i,tokenCorrected);
		role = wordRolesMap.get(tokenCorrected);
		// ////System.out.print("\| Second try: "+ token + " -> " + role);
		// System.out.println("role: " + role);
		tokens[i] = role;
		missFlag.add(false);
		}else{
		missFlag.add(true);
		cntMiss ++;
		}
		}
		//System.out.print("After token: "+tokens[i]+"\| ");
		//////System.out.println();
		}

		//System.out.println("\nMISS: "+cntMiss);
		if(cntMiss<3 && cntMiss>0 && !missFlag.get(missFlag.size()-1)){
		List<State> actions = new ArrayList<>();

		for(String token: tokens){
		actions.add(State.fromString(token));
		}
		// Define the initial state
		State initialState = State.START;

		// Check if the sequence of actions follows the state machine
		List<State> suggested = SM.updateDB(graph, actions, initialState);
		// System.out.println(actions);
		// System.out.println(suggested);
		// System.out.println("---------------------------");

		for(int i=0; i<suggested.size(); i++){
		if(!suggested.get(i).toString().equals(tokens[i])){
		if(!missFlag.get(i))
		return 0;
		}

		}
		int cntUpdate = 0;
		for(int i=0; i<tokens.length; i++){
		if(missFlag.get(i)){
		if(!suggested.get(i).toString().equals(tokens[i]) && !suggested.get(i).toString().equals("nan") && State.validSuggestedState(suggested.get(i))){
		System.out.println("\n"+sentence);
		System.out.println(actions);
		System.out.println(suggested);
		System.out.println(missFlag);
		System.out.println(tokens[i] + " -> " + suggested.get(i));
		System.out.println("--------------------------------");
		wordRolesMap.put(tokens[i], suggested.get(i).toString());
		cntUpdate++;
		}
		}
		}
		return cntUpdate;

		}
		return 0;
		}


		private HashMap<String, String> wordRolesMap;

		// Method to read data from SQLite database into HashMap
		public void readDataFromDatabase() {
		wordRolesMap = new HashMap<>();
		String dbUrl = "jdbc:sqlite:./SQLite/newdatabase.db";
		try (Connection connection = DriverManager.getConnection(dbUrl)) {
		Statement statement = connection.createStatement();
		ResultSet resultSet = statement.executeQuery("SELECT * FROM word_roles");

		while (resultSet.next()) {
		String word = resultSet.getString("word");
		String role = resultSet.getString("role");
		wordRolesMap.put(word, role);
		}

		} catch (SQLException e) {
		e.printStackTrace();
		}
		}

		// Method to update SQLite database with updated HashMap
		public void updateDatabase() {
		String dbUrl = "jdbc:sqlite:./SQLite/newdatabase.db";
		try (Connection connection = DriverManager.getConnection(dbUrl)) {
		// Clear existing data in the table
		Statement clearStatement = connection.createStatement();
		clearStatement.executeUpdate("DELETE FROM word_roles");

		// Insert updated data from HashMap into the table
		PreparedStatement insertStatement = connection.prepareStatement("INSERT INTO word_roles (word, role) VALUES (?, ?)");
		for (String word : wordRolesMap.keySet()) {
		String role = wordRolesMap.get(word);
		insertStatement.setString(1, word);
		insertStatement.setString(2, role);
		insertStatement.addBatch();
		}
		insertStatement.executeBatch();

		} catch (SQLException e) {
		e.printStackTrace();
		}
		}

		// Getter method for the HashMap
		public HashMap<String, String> getWordRolesMap() {
		return wordRolesMap;
		}

		// Setter method for the HashMap
		public void setWordRolesMap(HashMap<String, String> wordRolesMap) {
		this.wordRolesMap = wordRolesMap;
		}

		}

CheckerCorrector/DirectedGraph/BasicGraph.java

+190 −5

Original line number	Diff line number	Diff line
		package DirectedGraph;
		import java.util.HashMap;
		import java.sql.*;

		import StateMachine.State;

		public class BasicGraph {
		@@ -19,52 +22,234 @@ public class BasicGraph {
		cur = cur.next();
		graph.addEdge(State.first(), cur);
		}
		createTable();


		graph.addEdge(State.START, State.PRONOUN);
		edgeList.put(State.START.toString(), State.PRONOUN.toString());

		graph.addEdge(State.PRONOUN, State.VERB);
		edgeList.put(State.PRONOUN.toString(), State.VERB.toString());

		graph.addEdge(State.VERB, State.ADVERB);
		edgeList.put(State.VERB.toString(), State.ADVERB.toString());

		graph.addEdge(State.ADVERB, State.ADJECTIVE);
		edgeList.put(State.ADVERB.toString(), State.ADJECTIVE.toString());

		graph.addEdge(State.VERB, State.ARTICLE);
		edgeList.put(State.VERB.toString(), State.ARTICLE.toString());

		graph.addEdge(State.ARTICLE, State.ADVERB);
		edgeList.put(State.ARTICLE.toString(), State.ADVERB.toString());

		graph.addEdge(State.ARTICLE, State.ADJECTIVE);
		edgeList.put(State.ARTICLE.toString(), State.ADJECTIVE.toString());

		graph.addEdge(State.ARTICLE, State.NOUN);
		edgeList.put(State.ARTICLE.toString(), State.NOUN.toString());

		graph.addEdge(State.ADJECTIVE, State.DOT);
		edgeList.put(State.ADJECTIVE.toString(), State.DOT.toString());

		graph.addEdge(State.ADJECTIVE, State.NOUN);
		edgeList.put(State.ADJECTIVE.toString(), State.NOUN.toString());

		graph.addEdge(State.ADJECTIVE, State.COMMA);
		edgeList.put(State.ADJECTIVE.toString(), State.COMMA.toString());

		graph.addEdge(State.NOUN, State.DOT);
		edgeList.put(State.NOUN.toString(), State.DOT.toString());

		graph.addEdge(State.NOUN, State.COMMA);
		edgeList.put(State.NOUN.toString(), State.COMMA.toString());

		graph.addEdge(State.DOT, State.END);
		edgeList.put(State.DOT.toString(), State.END.toString());

		//graph.addEdge(State.COMMA, State.PRONOUN);
		graph.addEdge(State.COMMA, State.CONJ);
		edgeList.put(State.COMMA.toString(), State.CONJ.toString());

		graph.addEdge(State.CONJ, State.PRONOUN);
		graph.addEdge(State.PRONOUN, State.CAN);
		graph.addEdge(State.CAN, State.VERB);
		edgeList.put(State.CONJ.toString(), State.PRONOUN.toString());

		graph.addEdge(State.PRONOUN, State.MODAL);
		edgeList.put(State.PRONOUN.toString(), State.MODAL.toString());

		graph.addEdge(State.NOUN, State.MODAL);
		edgeList.put(State.NOUN.toString(), State.MODAL.toString());

		graph.addEdge(State.MODAL, State.VERB);
		edgeList.put(State.MODAL.toString(), State.VERB.toString());

		graph.addEdge(State.IF, State.PRONOUN);
		edgeList.put(State.IF.toString(), State.PRONOUN.toString());

		graph.addEdge(State.THAT, State.PRONOUN);
		edgeList.put(State.THAT.toString(), State.PRONOUN.toString());

		graph.addEdge(State.IF, State.NOUN);
		edgeList.put(State.IF.toString(), State.NOUN.toString());

		graph.addEdge(State.THAT, State.NOUN);
		edgeList.put(State.THAT.toString(), State.NOUN.toString());

		graph.addEdge(State.NOUN, State.VERB);
		graph.addEdge(State.NOUN, State.CAN);
		edgeList.put(State.NOUN.toString(), State.VERB.toString());

		graph.addEdge(State.NOUN, State.MODAL);
		edgeList.put(State.NOUN.toString(), State.MODAL.toString());

		//graph.addEdge(State.VERB, State.NOT);
		graph.addEdge(State.PRONOUN, State.DOES);
		edgeList.put(State.PRONOUN.toString(), State.DOES.toString());

		graph.addEdge(State.NOUN, State.DOES);
		edgeList.put(State.NOUN.toString(), State.DOES.toString());

		graph.addEdge(State.DOES, State.NOT);
		edgeList.put(State.DOES.toString(), State.NOT.toString());

		graph.addEdge(State.NOT, State.VERB);
		edgeList.put(State.NOT.toString(), State.VERB.toString());

		graph.addEdge(State.NOT, State.ADVERB);
		edgeList.put(State.NOT.toString(), State.ADVERB.toString());

		graph.addEdge(State.NOT, State.ADJECTIVE);
		edgeList.put(State.NOT.toString(), State.ADJECTIVE.toString());

		graph.addEdge(State.NOT, State.ARTICLE);
		graph.addEdge(State.OF, State.NOUN);
		graph.addEdge(State.NOUN, State.OF);
		edgeList.put(State.NOT.toString(), State.ARTICLE.toString());

		graph.addEdge(State.NOT, State.DOT);
		edgeList.put(State.NOT.toString(), State.DOT.toString());

		graph.addEdge(State.PREPOS, State.PRONOUN);
		edgeList.put(State.PREPOS.toString(), State.PRONOUN.toString());

		graph.addEdge(State.NOUN, State.IS);
		edgeList.put(State.NOUN.toString(), State.IS.toString());

		graph.addEdge(State.PRONOUN, State.IS);
		edgeList.put(State.PRONOUN.toString(), State.IS.toString());

		graph.addEdge(State.IS, State.ADJECTIVE);
		edgeList.put(State.IS.toString(), State.ADJECTIVE.toString());

		graph.addEdge(State.IS, State.ADVERB);
		edgeList.put(State.IS.toString(), State.ADVERB.toString());

		graph.addEdge(State.IS, State.ARTICLE);
		edgeList.put(State.IS.toString(), State.ARTICLE.toString());

		graph.addEdge(State.IS, State.NOT);
		edgeList.put(State.IS.toString(), State.NOT.toString());

		graph.addEdge(State.THAT, State.IF);
		edgeList.put(State.THAT.toString(), State.IF.toString());

		graph.addEdge(State.VERB, State.PREPOS);
		edgeList.put(State.VERB.toString(), State.PREPOS.toString());

		graph.addEdge(State.IS, State.PREPOS);
		edgeList.put(State.IS.toString(), State.PREPOS.toString());

		graph.addEdge(State.PREPOS, State.NOUN);
		edgeList.put(State.PREPOS.toString(), State.NOUN.toString());

		graph.addEdge(State.PREPOS, State.ARTICLE);
		edgeList.put(State.PREPOS.toString(), State.ARTICLE.toString());

		graph.addEdge(State.PREPOS, State.ADJECTIVE);
		edgeList.put(State.PREPOS.toString(), State.ADJECTIVE.toString());

		graph.addEdge(State.ADVERB, State.VERB);
		edgeList.put(State.ADVERB.toString(), State.VERB.toString());

		graph.addEdge(State.NOUN, State.NOUN);
		edgeList.put(State.NOUN.toString(), State.NOUN.toString());

		graph.addEdge(State.ADJECTIVE, State.ADJECTIVE);
		edgeList.put(State.ADJECTIVE.toString(), State.ADJECTIVE.toString());
		}
		public DirectedGraph<State> getGraph() {
		return graph;
		}


		private HashMap<String, String> edgeList;

		private static final String URL = "jdbc:sqlite:./SQLite/graphdatabase_name";

		// Method to create the table
		private void createTable() {
		edgeList = new HashMap<>();
		// SQL statement to create the table
		String sqlCreateTable = "CREATE TABLE nodeOne_nodeTwo ("
		+ "nodeOne VARCHAR(50) PRIMARY KEY,"
		+ "nodeTwo VARCHAR(50)"
		+ ")";

		try (Connection conn = DriverManager.getConnection(URL);
		Statement stmt = conn.createStatement()) {
		// Execute the SQL statement to create the table
		stmt.execute(sqlCreateTable);
		} catch (SQLException e) {
		e.printStackTrace();
		}
		}

		// Method to read data from SQLite database into HashMap
		public void readDataFromDatabase() {
		edgeList = new HashMap<>();
		String dbUrl = "jdbc:sqlite:./SQLite/graphdatabase.db";
		try (Connection connection = DriverManager.getConnection(dbUrl)) {
		Statement statement = connection.createStatement();
		ResultSet resultSet = statement.executeQuery("SELECT * FROM nodeOne_nodeTwo");

		while (resultSet.next()) {
		String word = resultSet.getString("nodeOne");
		String role = resultSet.getString("nodeTwo");
		edgeList.put(word, role);
		}

		} catch (SQLException e) {
		e.printStackTrace();
		}
		}

		// Method to update SQLite database with updated HashMap
		public void updateDatabase() {
		String dbUrl = "jdbc:sqlite:./SQLite/newdatabase.db";
		try (Connection connection = DriverManager.getConnection(dbUrl)) {
		// Clear existing data in the table
		Statement clearStatement = connection.createStatement();
		clearStatement.executeUpdate("DELETE FROM nodeOne_nodeTwo");

		// Insert updated data from HashMap into the table
		PreparedStatement insertStatement = connection.prepareStatement("INSERT INTO word_roles (nodeOne, nodeTwo) VALUES (?, ?)");
		for (String word : edgeList.keySet()) {
		String role = edgeList.get(word);
		insertStatement.setString(1, word);
		insertStatement.setString(2, role);
		insertStatement.addBatch();
		}
		insertStatement.executeBatch();

		} catch (SQLException e) {
		e.printStackTrace();
		}
		}

		// Getter method for the HashMap
		public HashMap<String, String> getEdgeList() {
		return edgeList;
		}

		// Setter method for the HashMap
		public void setWordRolesMap(HashMap<String, String> edgeList) {
		this.edgeList = edgeList;
		}
		}

CheckerCorrector/DirectedGraph/DFS.java

+8 −6

Original line number	Diff line number	Diff line
		@@ -19,14 +19,14 @@ public class DFS {
		return new DFS();
		}

		public Set<String> dfs(DirectedGraph<State> graph, State currentState, int maxDepth){
		public Set<String> dfs(DirectedGraph<State> graph, State currentState, State end, int maxDepth, int minDepth){
		List<State> path = new ArrayList<>();
		path.add(currentState);
		dfsRecurssion(graph, currentState, 0, maxDepth, path);
		dfsRecurssion(graph, currentState, end, 0, maxDepth, path, minDepth);
		return allPaths;
		}
		private void dfsRecurssion(DirectedGraph<State> graph, State currentState, int depth, int maxDepth, List<State> path) {
		if ((currentState == State.DOT)) {
		private void dfsRecurssion(DirectedGraph<State> graph, State currentState, State end, int depth, int maxDepth, List<State> path, int minDepth) {
		if ((currentState == end)) {
		ListToString lTS = ListToString.of();
		StringBuilder sb = new StringBuilder();
		int cnt=0;
		@@ -34,8 +34,10 @@ public class DFS {
		lTS.addString(p);
		cnt++;
		}
		if(cnt>(maxDepth-4))

		if(cnt>(maxDepth-minDepth)){
		allPaths.add(lTS.getString());
		}
		return;
		}else if(depth >= maxDepth){
		return;
		@@ -43,7 +45,7 @@ public class DFS {
		List<State> transitions = graph.getAdjacentNodes(currentState);
		for (State nextState : transitions) {
		path.add(nextState);
		dfsRecurssion(graph, nextState, depth + 1, maxDepth, path);
		dfsRecurssion(graph, nextState, end, depth + 1, maxDepth, path, minDepth);
		path.remove(path.size() - 1);
		}
		}