Commit 7f931b11 authored by Alexander Ross Melnick's avatar Alexander Ross Melnick
Browse files

Merge branch 'Alex' into 'master' 

Added more Dutch text

See merge request ec504/ec504_projects/group6!32
parents a26ea8d7 ba901afd

Crawler/src/main/java/Debug.java

0 → 100644
+8 −0
Original line number Diff line number Diff line 
// Allows for quick and easy enabling/disabling of debug messages



public class Debug {

    // Set to true to enable debug messages. Set to false to disable debug messages

    public static boolean DEBUG = false; 

    public static boolean DEBUG_RobotsTXT = false;

    public static boolean DEBUG_Dutch = false;

}

Crawler/src/main/java/RobotsTXT.class

(1.19 KiB)

File changed.

No diff preview for this file type.

View original file
View changed file

Crawler/src/main/java/RobotsTXT.java

0 → 100644
+44 −0
Original line number Diff line number Diff line 
import java.util.ArrayList;



// This class represents the robots.txt file for a given domain

public class RobotsTXT {

    private String domain;

    private ArrayList<String> disallowedPaths; // Excluded paths

    private ArrayList<String> allowedPaths; // Exceptions to the excluded paths

    private int crawlDelay; // in seconds



    public RobotsTXT(String domain) {

        this.domain = domain;

        disallowedPaths = new ArrayList<String>();

        allowedPaths = new ArrayList<String>();

        crawlDelay = 1; // Default crawl delay is 1 second

    }



    public void setDomain(String domain) {

        this.domain = domain;

    }

    public String getDomain() {

        return domain;

    }



    public void addDisallowedPath(String path) {

        disallowedPaths.add(path);

    }

    public ArrayList<String> getDisallowedPaths() {

        return disallowedPaths;

    }



    public void addAllowedPath(String path) {

        allowedPaths.add(path);

    }

    public ArrayList<String> getAllowedPaths() {

        return allowedPaths;

    }



    public void setCrawlDelay(int delay) {

        crawlDelay = delay;

    }

    public int getCrawlDelay() {

        return crawlDelay;

    }

}

Crawler/src/main/java/ScratchCrawler.java

+1 −19
Original line number Diff line number Diff line
@@ -353,23 +353,6 @@ public class ScratchCrawler { 
        return domain; // Return the domain

    }



    // Unused in this version of the crawler

    // public void crawl(String seed) {

    //     pagesToVisit.add(seed); // Add the seed page to pagesToVisit



    //     while (pagesVisited.size() < MAX_PAGES && !pagesToVisit.isEmpty()) { // While the number of visited pages is less than MAX_PAGES

    //         String nextPage = getNextPage(); // Get the next page

    //         try {

    //             Thread.sleep(waitTime); // Wait to be polite

    //             getPage(nextPage); // Get the page

    //         } catch (InterruptedException e) {

    //             System.out.println("Error waiting between crawling pages.");

    //             e.printStackTrace();

    //         } 

    //     }



    //     System.out.println("Crawling complete."); // Print message

    // }

    public void crawl() {

        if (crawlingDutch) {

            crawlDutchDict(); // Crawl the Dutch translation website
@@ -531,7 +514,6 @@ public class ScratchCrawler { 
                System.out.println("Processing rate in bytes per second: " + processingRateSize);

            }





            writer.close(); // Close the writer

            reader.close(); // Close the reader
@@ -623,7 +605,7 @@ public class ScratchCrawler { 
                case "--dutchSeed":

                    //  Extend your system to a language in which none of the team members have fluency

                    // Adds a Dutch website as a seed URL

                    crawler.pagesToVisit.add("https://www.rijksmuseum.nl/");

                    crawler.pagesToVisit.add("https://www.telegraaf.nl/");

                    startCrawl = true;

                    break;

                case "--turkish":