added web crawler code

Author: Vishal M Yadav

pom.xml

@@ -20,6 +20,11 @@
             <version>RELEASE</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.16.1</version>
+        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>

src/main/java/com/gatomalvado/done/multithreadedwebcrawler/HtmlParser.java

@@ -0,0 +1,9 @@
+package com.gatomalvado.done.multithreadedwebcrawler;
+
+import java.util.List;
+
+public interface HtmlParser {
+
+    List<String> parseHtml(String inputUrl);
+
+}

src/main/java/com/gatomalvado/done/multithreadedwebcrawler/Main.java

@@ -0,0 +1,18 @@
+package com.gatomalvado.done.multithreadedwebcrawler;
+
+import java.util.concurrent.ExecutionException;
+
+public class Main {
+
+    public static void main(String[] args) throws ExecutionException, InterruptedException {
+        System.out.println("Hello Multithreaded Web Crawler!");
+        String inputUrl = "https://en.wikipedia.org/wiki/As_It_Was";
+
+        MultithreadedWebCrawler crawler = new MultithreadedWebCrawler(new SingleThreadedHtmlParser(), 10);
+        crawler.startCrawl(inputUrl);
+        crawler.showParsedUrls();
+    }
+
+
+
+}

src/main/java/com/gatomalvado/done/multithreadedwebcrawler/MultithreadedWebCrawler.java

@@ -0,0 +1,72 @@
+package com.gatomalvado.done.multithreadedwebcrawler;
+
+import java.util.ArrayDeque;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class MultithreadedWebCrawler  {
+
+    private final HtmlParser htmlParser;
+    private final Map<String, Boolean> map;
+    private final ExecutorService executorService;
+    private final int limit;
+
+    public MultithreadedWebCrawler(HtmlParser htmlParser, int limit) {
+        this.htmlParser = htmlParser;
+        this.map = new ConcurrentHashMap<>();
+        this.limit = limit;
+        this.executorService = Executors.newFixedThreadPool(12);
+    }
+
+    public void startCrawl(String inputUrl) throws ExecutionException, InterruptedException {
+        Future<List<String>> extractedUrls = executorService.submit(crawl(inputUrl));
+        Deque<Future<List<String>>> queue = new ArrayDeque<>();
+        queue.add(extractedUrls);
+        while (!queue.isEmpty()) {
+            if(map.size() >= limit) {
+                break;
+            }
+            Thread.sleep(3000);
+
+            Future<List<String>> extractedUrlsFuture = queue.removeFirst();
+            List<String> parsedUrls = extractedUrlsFuture.get();
+            for(String parsedUrl : parsedUrls) {
+                if (!map.containsKey(parsedUrl)) {
+                    Callable<List<String>> callable = crawl(parsedUrl);
+                    queue.add(executorService.submit(callable));
+                }
+            }
+        }
+        executorService.shutdown();
+        // Wait for existing tasks to complete
+        executorService.awaitTermination(1, TimeUnit.SECONDS);
+    }
+
+    public void showParsedUrls() {
+        for(String key : map.keySet()) {
+            System.out.println(key);
+        }
+    }
+
+    private Callable<List<String>> crawl(String url) {
+        return () -> {
+            if(!map.containsKey(url)) {
+                List<String> parsedUrls = htmlParser.parseHtml(url);
+                map.put(url, true);
+                return parsedUrls.stream().filter((u) -> !map.containsKey(u)).collect(Collectors.toUnmodifiableList());
+            }
+            return Collections.emptyList();
+        };
+    }
+
+}

src/main/java/com/gatomalvado/done/multithreadedwebcrawler/SingleThreadedHtmlParser.java

@@ -0,0 +1,53 @@
+package com.gatomalvado.done.multithreadedwebcrawler;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+public class SingleThreadedHtmlParser implements HtmlParser {
+
+    @Override
+    public List<String> parseHtml(String inputUrl) {
+        String rawHtml = readUrl(inputUrl);
+        return getUrlsFromWebsite(rawHtml);
+    }
+
+    private List<String> getUrlsFromWebsite(String rawHtml) {
+        List<String> urls = new LinkedList<>();
+        Document doc = Jsoup.parse(rawHtml);
+        Elements elements = doc.select("a[href]");
+
+        for(Element element : elements) {
+            String link = element.attr("abs:href");
+            if(!link.isEmpty()) {
+                urls.add(link);
+            }
+        }
+
+        return urls;
+    }
+
+    private String readUrl(String webLink) {
+        String rawHtml = "";
+        try {
+            URL url = new URL(webLink);
+            BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()));
+            String inputLine = "";
+            while ((inputLine = reader.readLine()) != null) {
+                rawHtml += inputLine;
+            }
+            reader.close();
+        } catch (Exception e) {
+            System.out.println("Error reading url: " + webLink);
+        }
+
+        return rawHtml;
+    }
+}

src/main/java/com/gatomalvado/done/webcrawler/Main.java

@@ -0,0 +1,12 @@
+package com.gatomalvado.done.webcrawler;
+
+public class Main {
+
+    public static void main(String[] args) throws InterruptedException {
+        System.out.println("Hello Simple Webcrawler!");
+        WebCrawler webCrawler = new WebCrawler();
+        webCrawler.crawl("https://en.wikipedia.org/wiki/As_It_Was");
+        System.out.println(webCrawler.getDiscoveredWebsites());
+    }
+
+}

src/main/java/com/gatomalvado/done/webcrawler/WebCrawler.java

@@ -0,0 +1,96 @@
+package com.gatomalvado.done.webcrawler;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import lombok.Getter;
+
+public class WebCrawler {
+
+    private Queue<String> queue;
+
+    @Getter
+    private Set<String> discoveredWebsites;
+
+    private int websitesProcessed;
+
+    public WebCrawler() {
+        this.queue = new LinkedList<>();
+        this.discoveredWebsites = new HashSet<>();
+        this.websitesProcessed = 0;
+    }
+
+    public void crawl(String seedUrl) throws InterruptedException {
+        this.queue.offer(seedUrl);
+        Thread.sleep(3000);
+        while (!this.queue.isEmpty()) {
+            String currentUrl = this.queue.poll();
+            String rawHtml = readUrl(currentUrl);
+            if("".equals(rawHtml)) {
+                continue;
+            }
+
+            List<String> urlsParsed = getUrlsFromWebsite(rawHtml);
+
+            for (String websiteUrl : urlsParsed) {
+                if(!discoveredWebsites.contains(websiteUrl)) {
+                    // System.out.println("Website found with URL: " + websiteUrl);
+                    queue.add(websiteUrl);
+                }
+            }
+
+            this.discoveredWebsites.add(currentUrl);
+            this.websitesProcessed++;
+
+            if(this.websitesProcessed == 10000) {
+                break;
+            }
+        }
+    }
+
+    private List<String> getUrlsFromWebsite(String rawHtml) {
+        List<String> urls = new LinkedList<>();
+        Document doc = Jsoup.parse(rawHtml);
+        Elements elements = doc.select("a[href]");
+
+        for(Element element : elements) {
+            String link = element.attr("abs:href");
+            if(!link.isEmpty()) {
+                urls.add(link);
+            }
+        }
+
+        return urls;
+    }
+
+    private String readUrl(String webLink) {
+        String rawHtml = "";
+        try {
+            URL url = new URL(webLink);
+            BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()));
+            String inputLine = "";
+            while ((inputLine = reader.readLine()) != null) {
+                rawHtml += inputLine;
+            }
+            reader.close();
+        } catch (Exception e) {
+            System.out.println("Error reading url: " + webLink);
+        }
+
+        return rawHtml;
+    }
+
+}