loading bar (94eb9892) · Commits · EC504 Spring 2024 Group Projects / Group7

src/main/java/edu/bu/LanguageCorrection/MainApp.java

+37 −10

Original line number	Diff line number	Diff line
		@@ -36,6 +36,7 @@ public class MainApp extends JFrame {
		private final JButton runButton;
		private final JButton changeLanguageButton;
		private final JComboBox<String> moduleSelector;
		private JProgressBar progressBar = new JProgressBar(0, 100);

		private String languageFile = "English.ser";

		@@ -51,9 +52,7 @@ public class MainApp extends JFrame {
		String[] languages = {"English", "German", "Portuguese", "Italian"};
		String selectedLanguage = (String) JOptionPane.showInputDialog(this, "Metadata file not found. Please choose a language to build off of. \n(If you want to build from scratch just click Cancel)", "Language Selection", JOptionPane.PLAIN_MESSAGE, null, languages, languages[0]);
		if (selectedLanguage != null) {
		languageFile = selectedLanguage + ".ser";
		crawler webCrawler = new crawler(languageFile);
		webCrawler.build(selectedLanguage);
		runBuilder(selectedLanguage);
		}
		}

		@@ -70,10 +69,10 @@ public class MainApp extends JFrame {

		switch (selectedModule) {
		case "Web Crawler":
		runCrawler(input);
		runCrawler(input,10);
		break;
		case "Reddit Crawler":
		runCrawler("https://www.reddit.com/r/"+input);
		runCrawler("https://www.reddit.com/r/"+input, 1);
		break;
		case "File Checker":
		runChecker(input,true);
		@@ -101,8 +100,7 @@ public class MainApp extends JFrame {
		// Check if metadata file exists
		String extraText = "";
		if (!Files.exists(Paths.get(languageFile))) {
		crawler webCrawler = new crawler(languageFile);
		webCrawler.build(selectedLanguage);
		runBuilder(selectedLanguage);
		extraText = "New metadata file created.\n";
		}

		@@ -115,6 +113,7 @@ public class MainApp extends JFrame {
		northPanel.add(urlField, BorderLayout.CENTER);
		northPanel.add(runButton, BorderLayout.EAST);
		northPanel.add(changeLanguageButton, BorderLayout.NORTH);
		northPanel.add(progressBar, BorderLayout.SOUTH);

		// Result area
		resultArea = new JTextArea();
		@@ -125,12 +124,29 @@ public class MainApp extends JFrame {
		add(scrollPane, BorderLayout.CENTER);
		}

		private void runCrawler(String input) {
		private void runBuilder(String selectedLanguage) {
		languageFile = selectedLanguage + ".ser";
		StringBuilder outputBuilder = new StringBuilder();
		crawler webCrawler = new crawler(languageFile, output -> {
		outputBuilder.append(output);
		SwingUtilities.invokeLater(() -> resultArea.setText(outputBuilder.toString()));
		}, progressBar);
		Thread builderThread = new Thread(() -> {
		webCrawler.build(selectedLanguage);
		});
		progressBar.setString("Building "+selectedLanguage+" metadata file...");
		progressBar.setValue(0);
		builderThread.start();
		}

		private void runCrawler(String input, Integer limit) {
		progressBar.setValue(0);
		progressBar.setString(null);
		StringBuilder outputBuilder = new StringBuilder();
		crawler webCrawler = new crawler(languageFile, output -> {
		outputBuilder.append(output);
		SwingUtilities.invokeLater(() -> resultArea.setText(outputBuilder.toString()));
		}, progressBar);
		if (input.startsWith("http")) {
		// Input is a single URL
		webCrawler.add_to_queue(input);
		@@ -146,7 +162,7 @@ public class MainApp extends JFrame {
		}
		// Create a new thread for running the crawler
		Thread crawlerThread = new Thread(() -> {
		webCrawler.crawl(5); // Adjust the limit (DEFAULT SET TO 1)
		webCrawler.crawl(limit); // Adjust the limit (DEFAULT SET TO 1)
		});
		// Start the crawler thread
		crawlerThread.start();
		@@ -297,11 +313,17 @@ public class MainApp extends JFrame {
		content = input;
		Corrector corrector = new Corrector(languageFile);
		String[] sentences = TextProcessor.extractSentences(content).toArray(new String[0]);
		progressBar.setValue(0);
		progressBar.setStringPainted(true);
		progressBar.setString(null);
		progressBar.setMaximum(sentences.length*2);

		StringBuilder result = new StringBuilder();
		for (String sentence : sentences) {
		progressBar.setValue(progressBar.getValue() + 1);
		sentence = sentence.replaceAll("[^a-zA-Z0-9\\s]", "");
		String[] corrected = corrector.correct(sentence);
		progressBar.setValue(progressBar.getValue() + 2);
		if (corrected.length == 0) {
		result.append(sentence + " \| No corrections found for this sentence.\n\n");
		continue;
		@@ -344,11 +366,16 @@ public class MainApp extends JFrame {
		}
		String selected = (String) JOptionPane.showInputDialog(this, "Select the best correction for the following sentence:\n " + sentence, "Correction Selection", JOptionPane.PLAIN_MESSAGE, null, options, options[0]);
		if (selected != null) {
		progressBar.setValue(0);
		progressBar.setStringPainted(true);
		progressBar.setString(null);
		// Insert the selected correction into the trie for future reference
		List<String> phrases = TextProcessor.extractPhrases(selected, 2, 3);
		List<String> phrases = TextProcessor.extractPhrases(selected, 1, 3);
		progressBar.setMaximum(phrases.size());
		for (String phrase : phrases) {
		String[] words = phrase.split(" ");
		node.insert(words);
		progressBar.setValue(progressBar.getValue() + 1);
		}
		changeMade = true;
		} else {

src/main/java/edu/bu/LanguageCorrection/crawler.java

+29 −3

Original line number	Diff line number	Diff line
		@@ -203,7 +203,7 @@ public class crawler {
		while ((line = reader.readLine()) != null) {
		doc.append(line);
		}
		System.out.println("File read successfully.");
		// System.out.println("File read successfully.");
		outputCallback.accept("\nFile read successfully.");
		}
		return doc;
		@@ -264,6 +264,10 @@ public class crawler {
		return;
		}
		// System.out.println(chunks.size());
		if (progressBar != null) {
		progressBar.setMaximum(1024);
		progressBar.setStringPainted(true);
		}

		for (String chunk : chunks) {
		// chunkCount++;
		@@ -274,9 +278,12 @@ public class crawler {
		extractWordUsage(chunk, wordUsage);
		uncompressedData = wordUsage.serialize();
		compressedData = compress(uncompressedData);
		if (progressBar != null) {
		progressBar.setValue(compressedData.length - compression_size);
		}
		if ((compressedData.length - compression_size > 1024)) {
		System.out.println("Previous compressed data size: " + compression_size
		+ " bytes. Current compressed data size: " + compressedData.length + " bytes. Delta: "
		System.out.println("Previous size: " + compression_size
		+ " bytes. Current size: " + compressedData.length + " bytes. Delta: "
		+ (compressedData.length - compression_size) + " bytes.");
		outputCallback.accept("\nPrevious compressed data size: " + compression_size
		+ " bytes. Current compressed data size: " + compressedData.length + " bytes. Delta: "
		@@ -286,14 +293,33 @@ public class crawler {
		sizeLimitExceeded = true;
		uncompressedData = previousUncompressedData; // Revert to the previous uncompressed data
		compressedData = compress(uncompressedData); // Recompress the reverted state
		if (progressBar != null) {
		progressBar.setValue(1024);
		}
		// Output # of links found in page
		System.out.println("# of additional links found: " + foundLinks + "\n");
		outputCallback.accept("\n# of additional links found: " + foundLinks + "\n");
		break; // Stop processing further chunks
		}
		}
		} else {
		if (progressBar != null) {
		progressBar.setMaximum(3);
		progressBar.setStringPainted(true);
		progressBar.setValue(0);
		}
		extractWordUsage(web_data.text().replaceAll("\\p{Punct}", ""), wordUsage);
		// System.out.println("Ngrams built successfully. for size:"+MAXNGRAM);
		if (progressBar != null)
		progressBar.setValue(1);
		uncompressedData = wordUsage.serialize();
		if (progressBar != null)
		progressBar.setValue(2);
		compressedData = compress(uncompressedData);
		if (progressBar != null) {
		progressBar.setValue(3);
		progressBar.setString("Done building inital trie");
		}
		build_off_corpus = false; // if there are urls to read it should still be able to read them
		}