nGramScore (9ce016fd) · Commits · EC504 Spring 2024 Group Projects / Group6

Checker.java

+59 −17

Original line number	Diff line number	Diff line
		import java.io.IOException;
		import java.security.NoSuchAlgorithmException;
		import java.sql.SQLException;
		import java.util.HashMap;
		import java.util.List;

		import HashTableMaker.HashTableMaker;
		import DirectedGraph.BasicGraph;
		import DirectedGraph.DirectedGraph;
		import HashTableMaker.HashTableMaker;
		import StateMachine.*;
		import DBinterface.DBinterface;
		import util.*;
		@@ -16,27 +21,64 @@ public class Checker {
		DBinterface dbInterface = new DBinterface();
		DirectedGraph<State> graph = basicGraphClass.getGraph();
		JsonMaker jsonMaker = JsonMaker.create();

		if(argPars.isUpdateHashTable()){
		if(argPars.isCheckFile()){
		SentenceExtractor extractor = SentenceExtractor.of(argPars.getFileName());
		List<String> extractedSentences = extractor.getSentences();


		try {
		HashTableMaker manager = new HashTableMaker();
		for (String sentence : extractedSentences) {
		manager.updateDatabase(sentence.toLowerCase());
		PhraseExtractor extractorPhrase = PhraseExtractor.fromSentence(sentence);
		List<String> phrases = extractorPhrase.getPhrases();
		for (String phrase : phrases) {
		manager.updateDatabase(phrase.toLowerCase());
		}
		}
		manager.closeConnection();
		} catch (SQLException \| NoSuchAlgorithmException e) {
		e.printStackTrace();
		}
		}else if(argPars.isCheckSentence()){
		try {
		HashTableMaker manager = new HashTableMaker();
		manager.updateDatabase(argPars.getSentence().toLowerCase());
		for (String phrase : PhraseExtractor.fromSentence(argPars.getSentence()).getPhrases()) {
		manager.updateDatabase(phrase.toLowerCase());
		}
		manager.closeConnection();
		} catch (SQLException \| NoSuchAlgorithmException e) {
		e.printStackTrace();
		}
		}
		}else if(argPars.isCheckFile()){
		SentenceExtractor extractor = SentenceExtractor.of(argPars.getFileName());
		List<String> extractedSentences = extractor.getSentences();
		try {
		HashTableMaker manager = new HashTableMaker();
		for (String sentence : extractedSentences) {
		System.out.println("Sentence: " + sentence);
		jsonMaker.addSentence(sentence.toLowerCase(), dbInterface.checkTokenInDatabase(sentence.toLowerCase(), graph));

		System.out.println("*********************************************************");
		PhraseExtractor extractorPhrase = PhraseExtractor.fromSentence(sentence);
		List<String> phrases = extractorPhrase.getPhrases();
		int score = 0;
		jsonMaker.addSentence(sentence.toLowerCase(), (dbInterface.checkTokenInDatabase(sentence.toLowerCase(), graph)+manager.nGram(sentence, 3))/2);
		for (String phrase : phrases) {
		System.out.println("Phrase: "+ phrase);
		jsonMaker.addPhrase(phrase.toLowerCase(), dbInterface.checkTokenInDatabase(phrase.toLowerCase(), graph));
		jsonMaker.addPhrase(phrase.toLowerCase(), (dbInterface.checkTokenInDatabase(phrase.toLowerCase(), graph)+manager.nGram(phrase.toLowerCase(),3))/2);
		System.out.println("------------------------------------------------------------");

		}

		jsonMaker.toJson("data.json");
		System.out.println("##########################################################");

		}
		} catch (SQLException e) {
		e.printStackTrace();
		}
		}else if(argPars.isCheckSentence()){

		System.out.println("Sentence: " + argPars.getSentence());

HashTableMaker/HashTableMaker.java

0 → 100644

+154 −0

Original line number	Diff line number	Diff line
		package HashTableMaker;

		import java.security.MessageDigest;
		import java.security.NoSuchAlgorithmException;
		import java.sql.*;
		import java.util.HashMap;
		import java.util.List;

		import util.PhraseExtractor;

		public class HashTableMaker {
		private Connection connection;

		public HashTableMaker() throws SQLException {
		connection = DriverManager.getConnection("jdbc:sqlite:SQLite/hash_database.db");
		createTableIfNotExists();
		}

		private void createTableIfNotExists() throws SQLException {
		try (Statement statement = connection.createStatement()) {
		statement.executeUpdate("CREATE TABLE IF NOT EXISTS hashes (hash TEXT PRIMARY KEY, count INTEGER)");
		}
		}


		public HashMap<String, Integer> loadHashedSentences() throws SQLException {
		HashMap<String, Integer> hashedSentencesMap = new HashMap<>();

		String query = "SELECT hash, count FROM hashes";
		try (Statement statement = connection.createStatement();
		ResultSet resultSet = statement.executeQuery(query)) {
		while (resultSet.next()) {
		String hash = resultSet.getString("hash");
		int count = resultSet.getInt("count");
		hashedSentencesMap.put(hash, count);
		}
		}

		return hashedSentencesMap;
		}

		public void updateDatabase(String phrase) throws SQLException, NoSuchAlgorithmException {
		String hash = generateHash(phrase);

		PreparedStatement selectStatement = connection.prepareStatement("SELECT count FROM hashes WHERE hash = ?");
		selectStatement.setString(1, hash);
		ResultSet resultSet = selectStatement.executeQuery();

		if (resultSet.next()) {
		int count = resultSet.getInt("count");
		PreparedStatement updateStatement = connection.prepareStatement("UPDATE hashes SET count = ? WHERE hash = ?");
		updateStatement.setInt(1, count + 1);
		updateStatement.setString(2, hash);
		updateStatement.executeUpdate();
		} else {
		PreparedStatement insertStatement = connection.prepareStatement("INSERT INTO hashes (hash, count) VALUES (?, ?)");
		insertStatement.setString(1, hash);
		insertStatement.setInt(2, 1);
		insertStatement.executeUpdate();
		}
		}

		private String generateHash(String phrase) throws NoSuchAlgorithmException {
		MessageDigest digest = MessageDigest.getInstance("SHA-256");
		byte[] hashBytes = digest.digest(phrase.getBytes());

		StringBuilder hexString = new StringBuilder();
		for (byte b : hashBytes) {
		String hex = Integer.toHexString(0xff & b);
		if (hex.length() == 1) hexString.append('0');
		hexString.append(hex);
		}

		return hexString.toString();
		}

		public void closeConnection() throws SQLException {
		if (connection != null) {
		connection.close();
		}
		}

		public int getTotalCount() throws SQLException {
		int totalCount = 0;

		try (Statement statement = connection.createStatement();
		ResultSet resultSet = statement.executeQuery("SELECT SUM(count) AS total FROM hashes")) {
		if (resultSet.next()) {
		totalCount = resultSet.getInt("total");
		}
		}

		return totalCount;
		}

		public int getCountForSentence(String sentence) throws SQLException, NoSuchAlgorithmException {
		String hash = generateHash(sentence);

		PreparedStatement selectStatement = connection.prepareStatement("SELECT count FROM hashes WHERE hash = ?");
		selectStatement.setString(1, hash);
		ResultSet resultSet = selectStatement.executeQuery();

		if (resultSet.next()) {
		return resultSet.getInt("count");
		} else {
		return 0;
		}
		}

		public int nGram(String sentence, int n){
		String[] words = sentence.split("\\s+");
		if(words.length<n)
		return 0;
		PhraseExtractor extractorPhraseN1 = PhraseExtractor.fromSentence(sentence, n-2, n-1);
		PhraseExtractor extractorPhraseN = PhraseExtractor.fromSentence(sentence, n-1, n);
		List<String> phrasesN = extractorPhraseN.getPhrases();
		List<String> phrasesN1 = extractorPhraseN1.getPhrases();
		int cntCurN = 0;
		int cntCurN1 = 0;
		try{
		for(String phrase: phrasesN){
		cntCurN += getCountForSentence(phrase);
		}
		for(String phrase: phrasesN1){
		cntCurN1 += getCountForSentence(phrase);
		}
		} catch (SQLException \| NoSuchAlgorithmException e) {
		e.printStackTrace();
		}
		if(cntCurN1==0)
		return 0;
		double p = ((double)cntCurN)/((double)cntCurN1);
		return 100 - (int)p;
		}
		public static void main(String[] args) {
		try {
		HashTableMaker manager = new HashTableMaker();

		// Loading hashed sentences from the database into a HashMap
		HashMap<String, Integer> hashedSentences = manager.loadHashedSentences();
		System.out.println("Loaded Hashed Sentences:");
		System.out.println(hashedSentences);

		// Updating the database with a new phrase
		manager.updateDatabase("This is a test phrase.");
		System.out.println("Updated Hashed Sentences:");
		System.out.println(manager.loadHashedSentences());

		manager.closeConnection();
		} catch (SQLException \| NoSuchAlgorithmException e) {
		e.printStackTrace();
		}
		}
		}

SQLite/hash_database.db

0 → 100644

+1.57 MiB

File added.

No diff preview for this file type.

View file

all_path.txt

+0 −115

Original line number	Diff line number	Diff line
		b.d.e.f.k.b.d.c.e.k.
		b.d.e.f.k.b.d.e.k.b.
		b.d.c.e.f.k.i.b.d.g.
		b.d.c.e.k.b.d.c.e.f.
		b.d.e.f.k.b.d.c.g.e.
		b.d.g.e.k.b.d.e.f.k.
		b.d.c.e.k.i.b.d.e.h.
		b.d.e.k.b.d.e.k.i.b.
		b.d.e.k.i.b.d.c.e.h.
		b.d.c.g.e.k.b.d.c.e.
		b.d.e.k.b.d.c.g.e.k.
		b.d.g.e.k.i.b.d.e.f.
		b.d.c.e.f.k.b.d.e.f.
		b.d.e.f.k.b.d.g.e.f.
		b.d.g.e.k.b.d.c.g.e.
		b.d.c.g.e.f.k.b.d.c.
		b.d.e.k.b.d.e.f.h.j.
		b.d.e.k.b.d.e.f.k.i.
		b.d.g.e.k.b.d.c.e.k.
		b.d.c.g.e.k.b.d.e.h.
		b.d.g.e.f.k.i.b.d.g.
		b.d.g.e.k.b.d.g.e.h.
		b.d.c.e.k.i.b.d.c.e.
		b.d.c.e.f.k.b.d.c.e.
		b.d.c.e.f.k.i.b.d.e.
		b.d.e.f.k.b.d.e.f.k.
		b.d.e.k.i.b.d.g.e.h.
		b.d.c.e.k.b.d.c.e.h.
		b.d.e.f.k.b.d.g.e.h.
		b.d.g.e.f.k.b.d.e.f.
		b.d.g.e.k.b.d.e.k.b.
		b.d.c.g.e.k.i.b.d.c.
		b.d.c.e.k.b.d.g.e.k.
		b.d.e.k.i.b.d.e.f.h.
		b.d.e.f.k.i.b.d.c.e.
		b.d.e.k.i.b.d.e.h.j.
		b.d.e.k.i.b.d.e.k.i.
		b.d.e.k.i.b.d.e.k.b.
		b.d.e.f.k.i.b.d.e.f.
		b.d.c.g.e.k.b.d.e.f.
		b.d.g.e.f.k.i.b.d.e.
		b.d.g.e.k.b.d.e.f.h.
		b.d.g.e.k.b.d.e.k.i.
		b.d.g.e.k.b.d.e.h.j.
		b.d.e.k.b.d.e.k.b.d.
		b.d.g.e.k.i.b.d.g.e.
		b.d.c.g.e.f.k.i.b.d.
		b.d.g.e.k.b.d.g.e.f.
		b.d.g.e.f.k.b.d.c.e.
		b.d.g.e.k.i.b.d.e.k.
		b.d.c.e.k.i.b.d.c.g.
		b.d.g.e.k.b.d.c.e.h.
		b.d.c.e.k.b.d.e.f.h.
		b.d.c.e.f.k.i.b.d.c.
		b.d.c.e.k.b.d.e.h.j.
		b.d.c.e.k.b.d.e.k.i.
		b.d.e.k.i.b.d.g.e.f.
		b.d.e.f.k.i.b.d.c.g.
		b.d.c.g.e.k.b.d.g.e.
		b.d.c.g.e.k.i.b.d.e.
		b.d.e.k.b.d.c.e.f.k.
		b.d.e.k.b.d.g.e.f.k.
		b.d.c.g.e.k.b.d.e.k.
		b.d.g.e.f.k.b.d.e.h.
		b.d.g.e.k.b.d.g.e.k.
		b.d.c.g.e.f.k.b.d.g.
		b.d.c.e.k.b.d.e.k.b.
		b.d.e.f.k.b.d.c.e.f.
		b.d.e.k.i.b.d.c.e.k.
		b.d.g.e.f.k.i.b.d.c.
		b.d.c.e.k.i.b.d.e.k.
		b.d.e.f.k.i.b.d.e.h.
		b.d.e.f.k.b.d.g.e.k.
		b.d.e.k.b.d.c.g.e.h.
		b.d.c.e.f.k.b.d.e.k.
		b.d.g.e.k.i.b.d.c.g.
		b.d.c.e.f.k.b.d.g.e.
		b.d.e.k.i.b.d.c.g.e.
		b.d.e.k.i.b.d.e.f.k.
		b.d.g.e.f.k.b.d.c.g.
		b.d.c.e.k.b.d.g.e.f.
		b.d.c.e.k.i.b.d.g.e.
		b.d.g.e.k.b.d.c.e.f.
		b.d.e.k.b.d.e.f.k.b.
		b.d.e.f.k.i.b.d.e.k.
		b.d.c.g.e.k.b.d.c.g.
		b.d.c.e.f.k.b.d.e.h.
		b.d.e.k.b.d.g.e.k.b.
		b.d.e.k.b.d.c.e.k.b.
		b.d.e.f.k.i.b.d.g.e.
		b.d.e.k.i.b.d.c.e.f.
		b.d.g.e.k.i.b.d.e.h.
		b.d.c.g.e.k.i.b.d.g.
		b.d.c.e.k.i.b.d.e.f.
		b.d.c.g.e.f.k.b.d.e.
		b.d.e.k.i.b.d.g.e.k.
		b.d.e.f.k.b.d.c.e.h.
		b.d.e.f.k.b.d.e.f.h.
		b.d.c.e.k.b.d.c.e.k.
		b.d.c.e.k.b.d.e.f.k.
		b.d.c.e.k.b.d.c.g.e.
		b.d.e.f.k.b.d.e.h.j.
		b.d.e.f.k.b.d.e.k.i.
		b.d.e.k.b.d.c.e.h.j.
		b.d.e.k.b.d.c.e.k.i.
		b.d.g.e.k.i.b.d.c.e.
		b.d.e.k.b.d.c.g.e.f.
		b.d.e.k.b.d.g.e.h.j.
		b.d.g.e.f.k.b.d.e.k.
		b.d.e.k.b.d.g.e.k.i.
		b.d.c.e.k.b.d.g.e.h.
		b.d.c.e.f.k.b.d.c.g.
		b.d.g.e.f.k.b.d.g.e.
		b.d.e.k.b.d.g.e.f.h.
		b.d.e.k.b.d.c.e.f.h.

corrected.txt

+1 −1

Original line number	Diff line number	Diff line
		it is a very good book, but it is the very small book. it is the very bad. i am sophisticated. this is so strange, it is good choice.
		No newline at end of file
		sentences found a.
		No newline at end of file