From 75534424bb152ecb4ffa5d0b9f27fc2f3a3d6a77 Mon Sep 17 00:00:00 2001 From: Florent Youinou Date: Sat, 9 Dec 2017 17:33:48 +0100 Subject: [PATCH] fix last erros --- .../main/java/com/epita/domain/Document.java | 6 ++++-- domain/src/main/java/com/epita/domain/Term.java | 8 +++++--- src/main/java/com/epita/guereza/CrawlerApp.java | 6 ++++-- .../java/com/epita/guereza/EventStoreApp.java | 3 ++- src/main/java/com/epita/guereza/UrlStore.java | 17 ++++++++--------- .../com/epita/guereza/WrapperStringArray.java | 14 ++++++++++++++ 6 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/epita/guereza/WrapperStringArray.java diff --git a/domain/src/main/java/com/epita/domain/Document.java b/domain/src/main/java/com/epita/domain/Document.java index 462f1dc..2e3d335 100644 --- a/domain/src/main/java/com/epita/domain/Document.java +++ b/domain/src/main/java/com/epita/domain/Document.java @@ -3,8 +3,10 @@ import java.util.HashMap; public class Document { - public final String url; - public final HashMap terms; + public String url; + public HashMap terms; + + public Document() {} public Document(final String url, final HashMap terms) { this.url = url; diff --git a/domain/src/main/java/com/epita/domain/Term.java b/domain/src/main/java/com/epita/domain/Term.java index f70e020..3c1b14f 100644 --- a/domain/src/main/java/com/epita/domain/Term.java +++ b/domain/src/main/java/com/epita/domain/Term.java @@ -3,9 +3,11 @@ import java.util.List; public class Term { - public final String token; - public final List position; - public final double frequency; + public String token; + public List position; + public double frequency; + + public Term() {} public Term(final String token, final List position, final double frequency) { this.token = token; diff --git a/src/main/java/com/epita/guereza/CrawlerApp.java b/src/main/java/com/epita/guereza/CrawlerApp.java index 679fad8..5c0b2da 100644 --- a/src/main/java/com/epita/guereza/CrawlerApp.java +++ b/src/main/java/com/epita/guereza/CrawlerApp.java @@ -32,8 +32,10 @@ private void requestNextUrl() { } private void storeUrls(final String[] urls) { - LOGGER.info("Store {} urls", urls.length); - sendMessage("/store/crawler", urls); + if (urls != null) { + LOGGER.info("Store {} urls", urls.length); + sendMessage("/store/crawler", new WrapperStringArray(urls)); + } } diff --git a/src/main/java/com/epita/guereza/EventStoreApp.java b/src/main/java/com/epita/guereza/EventStoreApp.java index b7ad04a..f9431af 100644 --- a/src/main/java/com/epita/guereza/EventStoreApp.java +++ b/src/main/java/com/epita/guereza/EventStoreApp.java @@ -22,6 +22,7 @@ protected EventStoreApp(EventBusClient eventBus, EventStore eventStore) { private void extractThen(final EventBusClient.Message msg, final Consumer consumer) { try { + LOGGER.info("Message Type: {}", msg.getMessageType()); Class c = ClassLoader.getSystemClassLoader().loadClass(msg.getMessageType()); Object o = new ObjectMapper().readValue(msg.getContent(), c); if (o != null) { @@ -43,7 +44,7 @@ public void run() { eventStore.dispatch(ev); })); eventBus.subscribe("/store/crawler", msg -> extractThen(msg, o -> { - Event ev = new Event<>("ADD_URLS", (String[]) o); + Event ev = new Event<>("ADD_URLS", (WrapperStringArray) o); eventStore.dispatch(ev); })); eventBus.subscribe("/store/indexer", msg -> extractThen(msg, o -> { diff --git a/src/main/java/com/epita/guereza/UrlStore.java b/src/main/java/com/epita/guereza/UrlStore.java index 7431dc1..ed8c22e 100644 --- a/src/main/java/com/epita/guereza/UrlStore.java +++ b/src/main/java/com/epita/guereza/UrlStore.java @@ -9,10 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Queue; -import java.util.Set; +import java.util.*; public class UrlStore implements Reducer { private static final Logger LOGGER = LoggerFactory.getLogger(CrawlerService.class); @@ -26,7 +23,9 @@ public class UrlStore implements Reducer { public UrlStore(final EventBusClient eventBus) { this.eventBus = eventBus; - store(new String[] {STARTING_URL}); + final List initialList = new ArrayList(); + initialList.add(STARTING_URL); + store(initialList); } @SuppressWarnings("unchecked") @@ -34,7 +33,7 @@ public UrlStore(final EventBusClient eventBus) { public void reduce(final Event event) { switch (event.type) { case "ADD_URLS": - addUrls((Event) event); + addUrls((Event) event); break; case "CRAWLER_REQUEST_URL": crawlerRequestUrl((Event) event); @@ -45,7 +44,7 @@ public void reduce(final Event event) { } } - private void store(String[] urls) { + private void store(List urls) { for (String url : urls) { if (url == null || url.isEmpty()) continue; @@ -58,8 +57,8 @@ private void store(String[] urls) { } } - private void addUrls(Event event) { - store(event.obj); + private void addUrls(Event event) { + store(event.obj.content); LOGGER.info("added URLs to the repo"); } diff --git a/src/main/java/com/epita/guereza/WrapperStringArray.java b/src/main/java/com/epita/guereza/WrapperStringArray.java new file mode 100644 index 0000000..edff3d1 --- /dev/null +++ b/src/main/java/com/epita/guereza/WrapperStringArray.java @@ -0,0 +1,14 @@ +package com.epita.guereza; + +import java.util.Arrays; +import java.util.List; + +public class WrapperStringArray { + public List content; + + public WrapperStringArray() {} + + public WrapperStringArray(String[] content) { + this.content = Arrays.asList(content); + } +}