diff --git a/bot/pom.xml b/bot/pom.xml index f7ae0b6..07300f4 100644 --- a/bot/pom.xml +++ b/bot/pom.xml @@ -157,10 +157,16 @@ jcache - edu.java - retry - 0.1 - compile + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer + micrometer-tracing-bridge-brave diff --git a/bot/src/main/java/edu/java/bot/BotApplication.java b/bot/src/main/java/edu/java/bot/BotApplication.java index 510b644..b028bbf 100644 --- a/bot/src/main/java/edu/java/bot/BotApplication.java +++ b/bot/src/main/java/edu/java/bot/BotApplication.java @@ -1,7 +1,7 @@ package edu.java.bot; -import edu.java.RetryQueryConfiguration; import edu.java.bot.configuration.ApplicationConfig; +import edu.java.bot.configuration.RetryQueryConfiguration; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; diff --git a/bot/src/main/java/edu/java/bot/bot/UpdateTrackingBot.java b/bot/src/main/java/edu/java/bot/bot/UpdateTrackingBot.java index a2f2d99..6758932 100644 --- a/bot/src/main/java/edu/java/bot/bot/UpdateTrackingBot.java +++ b/bot/src/main/java/edu/java/bot/bot/UpdateTrackingBot.java @@ -10,27 +10,27 @@ import com.pengrad.telegrambot.response.BaseResponse; import edu.java.bot.commands.Command; import edu.java.bot.processors.MessageProcessor; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; import jakarta.annotation.PostConstruct; import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; @Component +@RequiredArgsConstructor public class UpdateTrackingBot implements Bot { private final TelegramBot telegramBot; private final MessageProcessor userMessagesProcessor; - - @Autowired - public UpdateTrackingBot(MessageProcessor userMessagesProcessor, TelegramBot telegramBot) { - this.userMessagesProcessor = userMessagesProcessor; - this.telegramBot = telegramBot; - } + private final MeterRegistry meterRegistry; + private Counter userMessagesCounter; @Override public int process(List updates) { updates.forEach(update -> { SendMessage sendMessage = userMessagesProcessor.process(update); + userMessagesCounter.increment(); execute(sendMessage); }); return UpdatesListener.CONFIRMED_UPDATES_ALL; @@ -55,4 +55,11 @@ public , R extends BaseResponse> void execute(BaseRe public void close() { telegramBot.shutdown(); } + + @PostConstruct + public void initMetrics() { + userMessagesCounter = Counter.builder("user_messages") + .description("Count of processed user messages") + .register(meterRegistry); + } } diff --git a/bot/src/main/java/edu/java/bot/configuration/RetryQueryConfiguration.java b/bot/src/main/java/edu/java/bot/configuration/RetryQueryConfiguration.java new file mode 100644 index 0000000..8060591 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/configuration/RetryQueryConfiguration.java @@ -0,0 +1,22 @@ +package edu.java.bot.configuration; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.jetbrains.annotations.NotNull; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "retry-query", ignoreUnknownFields = false) +public record RetryQueryConfiguration(Map targets) { + public record RetryElement( + @NotNull String type, + int maxAttempts, + double factor, + Duration minDelay, + Duration maxDelay, + List codes + ) { + } +} diff --git a/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java b/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java index 992b2d5..e59caf4 100644 --- a/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java +++ b/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java @@ -1,8 +1,7 @@ package edu.java.bot.configuration; -import edu.java.RetryFactory; -import edu.java.RetryQueryConfiguration; import edu.java.bot.client.scrapper.ScrapperClient; +import edu.java.bot.util.retry.RetryFactory; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -11,7 +10,6 @@ import org.springframework.web.reactive.function.client.support.WebClientAdapter; import org.springframework.web.service.invoker.HttpServiceProxyFactory; import reactor.core.publisher.Mono; -import reactor.util.retry.Retry; @Configuration @Log4j2 @@ -22,11 +20,10 @@ public class ScrapperClientConfiguration { @Bean public ScrapperClient scrapperClient(RetryQueryConfiguration retryQueryConfiguration) { - Retry retry = RetryFactory.createRetry(retryQueryConfiguration, "scrapper"); WebClient webClient = WebClient.builder() .defaultStatusHandler(httpStatusCode -> true, clientResponse -> Mono.empty()) .defaultHeader("Content-Type", "application/json") - .filter(RetryFactory.createFilter(retry)) + .filter(RetryFactory.createFilter(retryQueryConfiguration, "scrapper")) .baseUrl(scrapperUrl).build(); HttpServiceProxyFactory httpServiceProxyFactory = HttpServiceProxyFactory diff --git a/bot/src/main/java/edu/java/bot/util/retry/ErrorFilterPredicate.java b/bot/src/main/java/edu/java/bot/util/retry/ErrorFilterPredicate.java new file mode 100644 index 0000000..f1cd850 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/ErrorFilterPredicate.java @@ -0,0 +1,21 @@ +package edu.java.bot.util.retry; + +import java.util.List; +import java.util.function.Predicate; +import org.springframework.web.reactive.function.client.WebClientResponseException; + +public class ErrorFilterPredicate implements Predicate { + private final List retryCodes; + + public ErrorFilterPredicate(List retryCodes) { + this.retryCodes = retryCodes; + } + + @Override + public boolean test(Throwable throwable) { + if (throwable instanceof WebClientResponseException e) { + return retryCodes.contains(e.getStatusCode().value()); + } + return true; + } +} diff --git a/bot/src/main/java/edu/java/bot/util/retry/LinearRetryBackoffSpec.java b/bot/src/main/java/edu/java/bot/util/retry/LinearRetryBackoffSpec.java new file mode 100644 index 0000000..39941d9 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/LinearRetryBackoffSpec.java @@ -0,0 +1,93 @@ +package edu.java.bot.util.retry; + +import java.time.Duration; +import java.util.function.Predicate; +import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; +import org.reactivestreams.Publisher; +import org.springframework.retry.ExhaustedRetryException; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; +import reactor.util.retry.Retry; + + +@RequiredArgsConstructor +public class LinearRetryBackoffSpec extends Retry { + private static final Duration MAX_BACKOFF = Duration.ofMillis(Long.MAX_VALUE); + private final Duration minBackoff; + private final Duration maxBackoff; + private final double factor; + private final int maxAttempts; + private final Predicate errorFilter; + private final Supplier schedulerSupplier; + + public LinearRetryBackoffSpec factor(double factor) { + return new LinearRetryBackoffSpec( + this.minBackoff, + this.maxBackoff, + factor, + this.maxAttempts, + this.errorFilter, + this.schedulerSupplier + ); + } + + public LinearRetryBackoffSpec filter(Predicate errorFilter) { + return new LinearRetryBackoffSpec( + this.minBackoff, + this.maxBackoff, + this.factor, + this.maxAttempts, + errorFilter, + this.schedulerSupplier + ); + } + + public static LinearRetryBackoffSpec linear(int maxAttempts, Duration minDelay) { + return new LinearRetryBackoffSpec( + minDelay, + MAX_BACKOFF, + 1.0, + maxAttempts, + e -> true, + Schedulers::parallel + ); + } + + @Override + public Publisher generateCompanion(Flux retrySignals) { + return Flux.deferContextual(cv -> + retrySignals.contextWrite(cv) + .concatMap(retryWhenState -> { + RetrySignal copy = retryWhenState.copy(); + Throwable currentFailure = copy.failure(); + long iteration = copy.totalRetries(); + if (currentFailure == null) { + return Mono.error(new IllegalStateException( + "Retry.RetrySignal#failure() not expected to be null")); + } + if (!errorFilter.test(currentFailure)) { + return Mono.error(currentFailure); + } + if (iteration >= maxAttempts) { + return Mono.error(new ExhaustedRetryException("Retry exhausted: " + this)); + } + + Duration nextBackoff; + try { + nextBackoff = minBackoff.multipliedBy((long) (iteration * factor)); + if (nextBackoff.compareTo(maxBackoff) > 0) { + nextBackoff = maxBackoff; + } + } catch (ArithmeticException overflow) { + nextBackoff = maxBackoff; + } + + return Mono.delay(nextBackoff, schedulerSupplier.get()).contextWrite(cv); + }) + .onErrorStop() + ); + } +} diff --git a/bot/src/main/java/edu/java/bot/util/retry/RetryFactory.java b/bot/src/main/java/edu/java/bot/util/retry/RetryFactory.java new file mode 100644 index 0000000..ce1a0d2 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/RetryFactory.java @@ -0,0 +1,43 @@ +package edu.java.bot.util.retry; + +import edu.java.bot.configuration.RetryQueryConfiguration; +import edu.java.bot.util.retry.builders.ExponentialRetryBuilder; +import edu.java.bot.util.retry.builders.FixedRetryBuilder; +import edu.java.bot.util.retry.builders.LinearRetryBuilder; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import lombok.experimental.UtilityClass; +import org.springframework.web.reactive.function.client.ExchangeFilterFunction; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; + +@UtilityClass +public class RetryFactory { + + private static final Map> RETRY_BUILDERS = + new HashMap<>(); + + static { + RETRY_BUILDERS.put("fixed", new FixedRetryBuilder()); + RETRY_BUILDERS.put("linear", new LinearRetryBuilder()); + RETRY_BUILDERS.put("exponential", new ExponentialRetryBuilder()); + } + + public static ExchangeFilterFunction createFilter(RetryQueryConfiguration config, String target) { + return (response, next) -> next.exchange(response) + .flatMap(clientResponse -> { + if (clientResponse.statusCode().isError() + && config.targets().get(target).codes().contains(clientResponse.statusCode().value())) { + return clientResponse.createError(); + } else { + return Mono.just(clientResponse); + } + }).retryWhen(createRetry(config, target)); + } + + public static Retry createRetry(RetryQueryConfiguration config, String target) { + RetryQueryConfiguration.RetryElement element = config.targets().get(target); + return RETRY_BUILDERS.get(element.type()).apply(element); + } +} diff --git a/bot/src/main/java/edu/java/bot/util/retry/builders/ExponentialRetryBuilder.java b/bot/src/main/java/edu/java/bot/util/retry/builders/ExponentialRetryBuilder.java new file mode 100644 index 0000000..632b4f0 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/builders/ExponentialRetryBuilder.java @@ -0,0 +1,16 @@ +package edu.java.bot.util.retry.builders; + +import edu.java.bot.configuration.RetryQueryConfiguration; +import edu.java.bot.util.retry.ErrorFilterPredicate; +import java.util.function.Function; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class ExponentialRetryBuilder implements Function { + @Override + public Retry apply(RetryQueryConfiguration.RetryElement retryElement) { + return RetryBackoffSpec.backoff(retryElement.maxAttempts(), retryElement.minDelay()) + .maxBackoff(retryElement.maxDelay()) + .filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/bot/src/main/java/edu/java/bot/util/retry/builders/FixedRetryBuilder.java b/bot/src/main/java/edu/java/bot/util/retry/builders/FixedRetryBuilder.java new file mode 100644 index 0000000..5266a47 --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/builders/FixedRetryBuilder.java @@ -0,0 +1,15 @@ +package edu.java.bot.util.retry.builders; + +import edu.java.bot.configuration.RetryQueryConfiguration; +import edu.java.bot.util.retry.ErrorFilterPredicate; +import java.util.function.Function; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class FixedRetryBuilder implements Function { + @Override + public Retry apply(RetryQueryConfiguration.RetryElement retryElement) { + return RetryBackoffSpec.fixedDelay(retryElement.maxAttempts(), retryElement.minDelay()) + .filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/bot/src/main/java/edu/java/bot/util/retry/builders/LinearRetryBuilder.java b/bot/src/main/java/edu/java/bot/util/retry/builders/LinearRetryBuilder.java new file mode 100644 index 0000000..1df20cc --- /dev/null +++ b/bot/src/main/java/edu/java/bot/util/retry/builders/LinearRetryBuilder.java @@ -0,0 +1,15 @@ +package edu.java.bot.util.retry.builders; + +import edu.java.bot.configuration.RetryQueryConfiguration; +import edu.java.bot.util.retry.ErrorFilterPredicate; +import edu.java.bot.util.retry.LinearRetryBackoffSpec; +import java.util.function.Function; +import reactor.util.retry.Retry; + +public class LinearRetryBuilder implements Function { + @Override + public Retry apply(RetryQueryConfiguration.RetryElement retryElement) { + return LinearRetryBackoffSpec.linear(retryElement.maxAttempts(), retryElement.minDelay()) + .factor(retryElement.factor()).filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/bot/src/main/resources/application.yml b/bot/src/main/resources/application.yml index d7b4e0a..dc7716c 100644 --- a/bot/src/main/resources/application.yml +++ b/bot/src/main/resources/application.yml @@ -47,13 +47,13 @@ scrapper: url: http://localhost:8080 retry-query: - retries: - - target: scrapper + targets: + scrapper: type: exponential max-attempts: 5 min-delay: 1s max-delay: 10s - codes: 500 + codes: 429 bucket4j: enabled: true @@ -72,3 +72,17 @@ bucket4j: rate-limiter: whitelist: ${WHITELISTED_IPS:localhost} + +management: + metrics: + tags: + application: ${spring.application.name} + server: + port: 8001 + endpoints: + web: + exposure: + include: health, info, prometheus + path-mapping: + prometheus: /metrics + base-path: '/' diff --git a/bot/src/test/java/edu/java/bot/bot/UpdateTrackingBotTest.java b/bot/src/test/java/edu/java/bot/bot/UpdateTrackingBotTest.java index 54c1550..40ad0a1 100644 --- a/bot/src/test/java/edu/java/bot/bot/UpdateTrackingBotTest.java +++ b/bot/src/test/java/edu/java/bot/bot/UpdateTrackingBotTest.java @@ -3,6 +3,7 @@ import com.pengrad.telegrambot.TelegramBot; import com.pengrad.telegrambot.request.SendMessage; import edu.java.bot.processors.BotMessageProcessor; +import io.micrometer.core.instrument.MeterRegistry; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; @@ -11,8 +12,9 @@ public class UpdateTrackingBotTest { @Test public void startTest() { TelegramBot bot = Mockito.mock(TelegramBot.class); + MeterRegistry meterRegistry = Mockito.mock(MeterRegistry.class); UpdateTrackingBot updateTrackingBot = - new UpdateTrackingBot(Mockito.mock(BotMessageProcessor.class), bot); + new UpdateTrackingBot(bot, Mockito.mock(BotMessageProcessor.class), meterRegistry); updateTrackingBot.start(); Mockito.verify(bot, Mockito.times(1)).setUpdatesListener(Mockito.eq(updateTrackingBot)); } @@ -20,8 +22,9 @@ public void startTest() { @Test public void closeTest() { TelegramBot bot = Mockito.mock(TelegramBot.class); + MeterRegistry meterRegistry = Mockito.mock(MeterRegistry.class); UpdateTrackingBot updateTrackingBot = - new UpdateTrackingBot(Mockito.mock(BotMessageProcessor.class), bot); + new UpdateTrackingBot(bot, Mockito.mock(BotMessageProcessor.class), meterRegistry); updateTrackingBot.start(); updateTrackingBot.close(); Mockito.verify(bot, Mockito.times(1)).shutdown(); @@ -29,14 +32,18 @@ public void closeTest() { @Test public void executeNullTest() { - UpdateTrackingBot updateTrackingBot = new UpdateTrackingBot(Mockito.mock(BotMessageProcessor.class), null); + MeterRegistry meterRegistry = Mockito.mock(MeterRegistry.class); + UpdateTrackingBot updateTrackingBot = + new UpdateTrackingBot(null, Mockito.mock(BotMessageProcessor.class), meterRegistry); assertThatThrownBy(updateTrackingBot::start).isInstanceOf(RuntimeException.class); } @Test public void executeTest() { TelegramBot bot = Mockito.mock(TelegramBot.class); - UpdateTrackingBot updateTrackingBot = new UpdateTrackingBot(Mockito.mock(BotMessageProcessor.class), bot); + MeterRegistry meterRegistry = Mockito.mock(MeterRegistry.class); + UpdateTrackingBot updateTrackingBot = + new UpdateTrackingBot(bot, Mockito.mock(BotMessageProcessor.class), meterRegistry); SendMessage sendMessage = new SendMessage(1, "text"); updateTrackingBot.execute(sendMessage); diff --git a/bot/src/test/java/edu/java/bot/kafka/KafkaUpdatesListenerTest.java b/bot/src/test/java/edu/java/bot/kafka/KafkaUpdatesListenerTest.java index 733b6cf..2f06135 100644 --- a/bot/src/test/java/edu/java/bot/kafka/KafkaUpdatesListenerTest.java +++ b/bot/src/test/java/edu/java/bot/kafka/KafkaUpdatesListenerTest.java @@ -1,10 +1,12 @@ package edu.java.bot.kafka; - import edu.java.bot.configuration.ApplicationConfig; import edu.java.bot.dto.request.LinkUpdate; import edu.java.bot.service.LinkUpdatesSenderService; import edu.java.bot.util.URLCreator; +import java.time.Duration; +import java.util.List; +import java.util.Map; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -14,11 +16,6 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.kafka.core.KafkaTemplate; - -import java.time.Duration; -import java.util.List; -import java.util.Map; - import static org.awaitility.Awaitility.await; @SpringBootTest diff --git a/compose.yml b/compose.yml index 5cf9984..63773a7 100644 --- a/compose.yml +++ b/compose.yml @@ -1,35 +1,33 @@ services: - zookeeper: + zoo1: image: confluentinc/cp-zookeeper:7.6.0 - hostname: zookeeper - container_name: zookeeper + hostname: zoo1 + container_name: zoo1 ports: - "2181:2181" environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 volumes: - - zookeeper:/var/lib/zookeeper/data - - zookeeper:/var/lib/zookeeper/log + - zoo:/var/lib/zookeeper/data kafka1: image: confluentinc/cp-kafka:7.6.0 hostname: kafka1 container_name: kafka1 ports: - - "29092:29092" - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092, PLAINTEXT_HOST://localhost:29092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092,PLAINTEXT_HOST://localhost:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: zoo1:2181 KAFKA_BROKER_ID: 1 - BOOTSTRAP_SERVERS: kafka1:9092 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 depends_on: - - zookeeper + - zoo1 volumes: - kafka:/var/lib/kafka/data @@ -62,10 +60,34 @@ services: networks: - backend + prometheus: + image: prom/prometheus + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana-oss + container_name: grafana + ports: + - 3000:3000 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=grafana + - GF_SECURITY_ADMIN_PASSWORD=grafana + volumes: + - grafana:/var/lib/grafana + volumes: postgresql: { } kafka: { } - zookeeper: { } + zoo: { } + grafana: {} networks: backend: { } diff --git a/pom.xml b/pom.xml index 531ee0d..7c7d0b3 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,6 @@ bot scrapper scrapper-jooq - retry diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..2f29609 --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 10s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['host.docker.internal:9090'] + - job_name: 'bot' + static_configs: + - targets: ['host.docker.internal:8001'] + - job_name: 'scrapper' + static_configs: + - targets: ['host.docker.internal:8000'] diff --git a/retry/pom.xml b/retry/pom.xml deleted file mode 100644 index 9f21383..0000000 --- a/retry/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - 4.0.0 - - edu.java - root - ${revision} - - - retry - ${revision} - - - 21 - 21 - UTF-8 - - - - io.projectreactor - reactor-core - - - org.springframework - spring-webflux - - - org.springframework - spring-context - - - org.springframework.boot - spring-boot - - - org.springframework.retry - spring-retry - - - org.projectlombok - lombok - - - - diff --git a/scrapper/pom.xml b/scrapper/pom.xml index 53fac52..99e4628 100644 --- a/scrapper/pom.xml +++ b/scrapper/pom.xml @@ -195,10 +195,16 @@ jcache - edu.java - retry - 0.1 - compile + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer + micrometer-tracing-bridge-brave diff --git a/scrapper/src/main/java/edu/java/ScrapperApplication.java b/scrapper/src/main/java/edu/java/ScrapperApplication.java index b77624a..915813b 100644 --- a/scrapper/src/main/java/edu/java/ScrapperApplication.java +++ b/scrapper/src/main/java/edu/java/ScrapperApplication.java @@ -1,6 +1,7 @@ package edu.java; import edu.java.configuration.ApplicationConfig; +import edu.java.configuration.RetryQueryConfiguration; import edu.java.configuration.supplier.GithubConfig; import edu.java.configuration.supplier.StackOverflowConfig; import org.springframework.boot.SpringApplication; diff --git a/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java b/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java index 83dbc42..bc2aefc 100644 --- a/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java +++ b/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java @@ -1,8 +1,7 @@ package edu.java.configuration; -import edu.java.RetryFactory; -import edu.java.RetryQueryConfiguration; import edu.java.client.bot.BotClient; +import edu.java.util.retry.RetryFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,7 +9,6 @@ import org.springframework.web.reactive.function.client.support.WebClientAdapter; import org.springframework.web.service.invoker.HttpServiceProxyFactory; import reactor.core.publisher.Mono; -import reactor.util.retry.Retry; @Configuration public class BotClientConfiguration { @@ -20,11 +18,10 @@ public class BotClientConfiguration { @Bean public BotClient botClient(RetryQueryConfiguration retryQueryConfiguration) { - Retry retry = RetryFactory.createRetry(retryQueryConfiguration, "bot"); WebClient webClient = WebClient.builder() .defaultStatusHandler(httpStatusCode -> true, clientResponse -> Mono.empty()) .defaultHeader("Content-Type", "application/json") - .filter(RetryFactory.createFilter(retry)) + .filter(RetryFactory.createFilter(retryQueryConfiguration, "bot")) .baseUrl(botUrl).build(); HttpServiceProxyFactory httpServiceProxyFactory = HttpServiceProxyFactory diff --git a/retry/src/main/java/edu/java/RetryQueryConfiguration.java b/scrapper/src/main/java/edu/java/configuration/RetryQueryConfiguration.java similarity index 80% rename from retry/src/main/java/edu/java/RetryQueryConfiguration.java rename to scrapper/src/main/java/edu/java/configuration/RetryQueryConfiguration.java index f45606c..e965c86 100644 --- a/retry/src/main/java/edu/java/RetryQueryConfiguration.java +++ b/scrapper/src/main/java/edu/java/configuration/RetryQueryConfiguration.java @@ -1,16 +1,16 @@ -package edu.java; +package edu.java.configuration; import java.time.Duration; import java.util.List; +import java.util.Map; import org.jetbrains.annotations.NotNull; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; @Validated @ConfigurationProperties(prefix = "retry-query", ignoreUnknownFields = false) -public record RetryQueryConfiguration(List retries) { +public record RetryQueryConfiguration(Map targets) { public record RetryElement( - @NotNull String target, @NotNull String type, int maxAttempts, double factor, diff --git a/scrapper/src/main/java/edu/java/exception/ApplicationExceptionHandler.java b/scrapper/src/main/java/edu/java/exception/ApplicationExceptionHandler.java index 4b2ddcd..38daa4e 100644 --- a/scrapper/src/main/java/edu/java/exception/ApplicationExceptionHandler.java +++ b/scrapper/src/main/java/edu/java/exception/ApplicationExceptionHandler.java @@ -2,9 +2,14 @@ import edu.java.dto.response.ApiErrorResponse; import java.util.Arrays; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.TypeMismatchException; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatusCode; import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.ServletRequestBindingException; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.context.request.WebRequest; @@ -13,6 +18,62 @@ @RestControllerAdvice public class ApplicationExceptionHandler extends ResponseEntityExceptionHandler { + @Override + protected ResponseEntity handleMethodArgumentNotValid( + @NotNull MethodArgumentNotValidException ex, + @NotNull HttpHeaders headers, + @NotNull HttpStatusCode status, + @NotNull WebRequest request + ) { + return handleIncorrectRequest(ex, status); + } + + @Override + protected ResponseEntity handleTypeMismatch( + @NotNull TypeMismatchException ex, + @NotNull HttpHeaders headers, + @NotNull HttpStatusCode status, + @NotNull WebRequest request + ) { + return handleIncorrectRequest(ex, status); + } + + @Override + protected ResponseEntity handleServletRequestBindingException( + @NotNull ServletRequestBindingException ex, + @NotNull HttpHeaders headers, + @NotNull HttpStatusCode status, + @NotNull WebRequest request + ) { + return handleIncorrectRequest(ex, status); + } + + @Override + protected ResponseEntity handleHttpMessageNotReadable( + HttpMessageNotReadableException ex, + HttpHeaders headers, + HttpStatusCode status, + WebRequest request + ) { + return handleIncorrectRequest(ex, status); + } + + private ResponseEntity handleIncorrectRequest( + Exception ex, + HttpStatusCode status + ) { + return new ResponseEntity<>( + new ApiErrorResponse( + "Request with wrong arguments", + String.valueOf(status.value()), + ex.getClass().getSimpleName(), + ex.getMessage(), + Arrays.stream(ex.getStackTrace()).map(StackTraceElement::toString).toList() + ), + status + ); + } + @ExceptionHandler(ScrapperException.class) public ResponseEntity handleScrapperException(ScrapperException ex) { return new ResponseEntity<>( @@ -26,29 +87,4 @@ public ResponseEntity handleScrapperException(ScrapperExceptio ex.getStatus() ); } - - @Override - protected ResponseEntity handleExceptionInternal( - Exception ex, - Object body, - HttpHeaders headers, - HttpStatusCode statusCode, - WebRequest request - ) { - if (body == null) { - return new ResponseEntity<>( - new ApiErrorResponse( - "Internal Server Error", - String.valueOf(statusCode.value()), - ex.getClass().getName(), - ex.getMessage(), - null - ), - headers, - statusCode - ); - } - return new ResponseEntity<>(body, headers, statusCode); - } } - diff --git a/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java index 064a2b2..eec3218 100644 --- a/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java @@ -1,10 +1,10 @@ package edu.java.supplier.api; -import edu.java.RetryFactory; +import edu.java.configuration.RetryQueryConfiguration; +import edu.java.util.retry.RetryFactory; import java.time.OffsetDateTime; import org.springframework.http.MediaType; import org.springframework.web.reactive.function.client.WebClient; -import reactor.util.retry.Retry; public abstract class WebClientInfoSupplier implements InfoSupplier { private final WebClient webClient; @@ -17,10 +17,10 @@ public WebClientInfoSupplier(WebClient webClient) { this.webClient = webClient; } - public WebClientInfoSupplier(String baseUrl, Retry retry) { + public WebClientInfoSupplier(String baseUrl, RetryQueryConfiguration retryQueryConfiguration, String target) { this(WebClient.builder() .baseUrl(baseUrl) - .filter(RetryFactory.createFilter(retry)) + .filter(RetryFactory.createFilter(retryQueryConfiguration, target)) .build() ); } diff --git a/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java index 6ac1a62..b739a3f 100644 --- a/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java @@ -1,13 +1,13 @@ package edu.java.supplier.github; -import edu.java.RetryFactory; -import edu.java.RetryQueryConfiguration; import edu.java.configuration.ApplicationConfig; +import edu.java.configuration.RetryQueryConfiguration; import edu.java.configuration.supplier.GithubConfig; import edu.java.supplier.api.LinkInfo; import edu.java.supplier.api.LinkUpdateEvent; import edu.java.supplier.api.WebClientInfoSupplier; import edu.java.supplier.github.data.GithubEventsCollector; +import edu.java.util.retry.RetryFactory; import java.net.URL; import java.time.OffsetDateTime; import java.util.List; @@ -43,7 +43,7 @@ public GithubInfoSupplier( headers.set("Authorization", "Bearer " + applicationConfig.githubToken()); } }) - .filter(RetryFactory.createFilter(RetryFactory.createRetry(retryQueryConfiguration, TYPE_SUPPLIER))) + .filter(RetryFactory.createFilter(retryQueryConfiguration, TYPE_SUPPLIER)) .build() ); repositoryPattern = Pattern.compile(githubConfig.patterns().repository()); diff --git a/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java index cd1ac8b..b307755 100644 --- a/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java @@ -2,8 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import edu.java.RetryFactory; -import edu.java.RetryQueryConfiguration; +import edu.java.configuration.RetryQueryConfiguration; import edu.java.configuration.supplier.StackOverflowConfig; import edu.java.supplier.api.EventResolver; import edu.java.supplier.api.LinkInfo; @@ -43,7 +42,7 @@ public StackOverflowInfoSupplier( ObjectMapper mapper, RetryQueryConfiguration retryQueryConfiguration ) { - super(config.url(), RetryFactory.createRetry(retryQueryConfiguration, TYPE_SUPPLIER)); + super(config.url(), retryQueryConfiguration, TYPE_SUPPLIER); questionsPattern = Pattern.compile(config.patterns().questions()); this.mapper = mapper; eventResolver = new StackOverflowEventResolver(); diff --git a/retry/src/main/java/edu/java/ErrorFilterPredicate.java b/scrapper/src/main/java/edu/java/util/retry/ErrorFilterPredicate.java similarity index 95% rename from retry/src/main/java/edu/java/ErrorFilterPredicate.java rename to scrapper/src/main/java/edu/java/util/retry/ErrorFilterPredicate.java index 7ee702c..f4ea595 100644 --- a/retry/src/main/java/edu/java/ErrorFilterPredicate.java +++ b/scrapper/src/main/java/edu/java/util/retry/ErrorFilterPredicate.java @@ -1,4 +1,4 @@ -package edu.java; +package edu.java.util.retry; import java.util.List; import java.util.function.Predicate; diff --git a/retry/src/main/java/edu/java/LinearRetryBackoffSpec.java b/scrapper/src/main/java/edu/java/util/retry/LinearRetryBackoffSpec.java similarity index 99% rename from retry/src/main/java/edu/java/LinearRetryBackoffSpec.java rename to scrapper/src/main/java/edu/java/util/retry/LinearRetryBackoffSpec.java index cf41479..bc6d4f0 100644 --- a/retry/src/main/java/edu/java/LinearRetryBackoffSpec.java +++ b/scrapper/src/main/java/edu/java/util/retry/LinearRetryBackoffSpec.java @@ -1,4 +1,4 @@ -package edu.java; +package edu.java.util.retry; import java.time.Duration; import java.util.function.Predicate; diff --git a/retry/src/main/java/edu/java/RetryFactory.java b/scrapper/src/main/java/edu/java/util/retry/RetryFactory.java similarity index 58% rename from retry/src/main/java/edu/java/RetryFactory.java rename to scrapper/src/main/java/edu/java/util/retry/RetryFactory.java index 5089dcc..8a12c5e 100644 --- a/retry/src/main/java/edu/java/RetryFactory.java +++ b/scrapper/src/main/java/edu/java/util/retry/RetryFactory.java @@ -1,8 +1,9 @@ -package edu.java; +package edu.java.util.retry; -import edu.java.builders.ExponentialRetryBuilder; -import edu.java.builders.FixedRetryBuilder; -import edu.java.builders.LinearRetryBuilder; +import edu.java.configuration.RetryQueryConfiguration; +import edu.java.util.retry.builders.ExponentialRetryBuilder; +import edu.java.util.retry.builders.FixedRetryBuilder; +import edu.java.util.retry.builders.LinearRetryBuilder; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -23,20 +24,20 @@ public class RetryFactory { RETRY_BUILDERS.put("exponential", new ExponentialRetryBuilder()); } - public static ExchangeFilterFunction createFilter(Retry retry) { + public static ExchangeFilterFunction createFilter(RetryQueryConfiguration config, String target) { return (response, next) -> next.exchange(response) .flatMap(clientResponse -> { - if (clientResponse.statusCode().isError()) { + if (clientResponse.statusCode().isError() + && config.targets().get(target).codes().contains(clientResponse.statusCode().value())) { return clientResponse.createError(); } else { return Mono.just(clientResponse); } - }).retryWhen(retry); + }).retryWhen(createRetry(config, target)); } public static Retry createRetry(RetryQueryConfiguration config, String target) { - return config.retries().stream().filter(element -> element.target().equals(target)).findFirst() - .map(element -> RETRY_BUILDERS.get(element.type()).apply(element)) - .orElseThrow(() -> new IllegalStateException("Unknown target " + target)); + RetryQueryConfiguration.RetryElement element = config.targets().get(target); + return RETRY_BUILDERS.get(element.type()).apply(element); } } diff --git a/retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java b/scrapper/src/main/java/edu/java/util/retry/builders/ExponentialRetryBuilder.java similarity index 78% rename from retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java rename to scrapper/src/main/java/edu/java/util/retry/builders/ExponentialRetryBuilder.java index 5aff1f7..1ee6524 100644 --- a/retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java +++ b/scrapper/src/main/java/edu/java/util/retry/builders/ExponentialRetryBuilder.java @@ -1,7 +1,7 @@ -package edu.java.builders; +package edu.java.util.retry.builders; -import edu.java.ErrorFilterPredicate; -import edu.java.RetryQueryConfiguration; +import edu.java.configuration.RetryQueryConfiguration; +import edu.java.util.retry.ErrorFilterPredicate; import java.util.function.Function; import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; diff --git a/retry/src/main/java/edu/java/builders/FixedRetryBuilder.java b/scrapper/src/main/java/edu/java/util/retry/builders/FixedRetryBuilder.java similarity index 77% rename from retry/src/main/java/edu/java/builders/FixedRetryBuilder.java rename to scrapper/src/main/java/edu/java/util/retry/builders/FixedRetryBuilder.java index d9e6724..36ec0b8 100644 --- a/retry/src/main/java/edu/java/builders/FixedRetryBuilder.java +++ b/scrapper/src/main/java/edu/java/util/retry/builders/FixedRetryBuilder.java @@ -1,7 +1,7 @@ -package edu.java.builders; +package edu.java.util.retry.builders; -import edu.java.ErrorFilterPredicate; -import edu.java.RetryQueryConfiguration; +import edu.java.configuration.RetryQueryConfiguration; +import edu.java.util.retry.ErrorFilterPredicate; import java.util.function.Function; import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; diff --git a/retry/src/main/java/edu/java/builders/LinearRetryBuilder.java b/scrapper/src/main/java/edu/java/util/retry/builders/LinearRetryBuilder.java similarity index 70% rename from retry/src/main/java/edu/java/builders/LinearRetryBuilder.java rename to scrapper/src/main/java/edu/java/util/retry/builders/LinearRetryBuilder.java index 7b95b37..56088b0 100644 --- a/retry/src/main/java/edu/java/builders/LinearRetryBuilder.java +++ b/scrapper/src/main/java/edu/java/util/retry/builders/LinearRetryBuilder.java @@ -1,8 +1,8 @@ -package edu.java.builders; +package edu.java.util.retry.builders; -import edu.java.ErrorFilterPredicate; -import edu.java.LinearRetryBackoffSpec; -import edu.java.RetryQueryConfiguration; +import edu.java.configuration.RetryQueryConfiguration; +import edu.java.util.retry.ErrorFilterPredicate; +import edu.java.util.retry.LinearRetryBackoffSpec; import java.util.function.Function; import reactor.util.retry.Retry; diff --git a/scrapper/src/main/resources/application.yml b/scrapper/src/main/resources/application.yml index 705c064..75269b9 100644 --- a/scrapper/src/main/resources/application.yml +++ b/scrapper/src/main/resources/application.yml @@ -65,24 +65,24 @@ bot: url: http://localhost:8090 retry-query: - retries: - - target: github + targets: + github: type: exponential max-attempts: 5 min-delay: 1s max-delay: 10s - codes: 500 - - target: stackoverflow + codes: 429 + stackoverflow: type: fixed max-attempts: 5 min-delay: 1s max-delay: 10s - codes: 500 - - target: bot + codes: 429 + bot: type: linear max-attempts: 5 min-delay: 5s - codes: 500 + codes: 429 bucket4j: enabled: true @@ -101,3 +101,17 @@ bucket4j: rate-limiter: whitelist: ${WHITELISTED_IPS:localhost} + +management: + metrics: + tags: + application: ${spring.application.name} + server: + port: 8000 + endpoints: + web: + exposure: + include: health, info, prometheus + path-mapping: + prometheus: /metrics + base-path: '/' diff --git a/scrapper/src/test/java/edu/java/scrapper/controller/LinkControllerTest.java b/scrapper/src/test/java/edu/java/scrapper/controller/LinkControllerTest.java index c00c48c..6fa98f4 100644 --- a/scrapper/src/test/java/edu/java/scrapper/controller/LinkControllerTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/controller/LinkControllerTest.java @@ -80,7 +80,7 @@ public void getTrackedLinksMissingHeaderTgChatIdTest() { ApiErrorResponse error = objectMapper.readValue(result.getResponse().getContentAsString(), ApiErrorResponse.class); Assertions.assertThat(error).extracting("code", "exceptionName") - .contains("400", "org.springframework.web.bind.MissingRequestHeaderException"); + .contains("400", "MissingRequestHeaderException"); } @Test diff --git a/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java b/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java index 9326d4f..5c4e277 100644 --- a/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java @@ -1,8 +1,8 @@ package edu.java.scrapper.supplier.github; import com.github.tomakehurst.wiremock.WireMockServer; -import edu.java.RetryQueryConfiguration; import edu.java.configuration.ApplicationConfig; +import edu.java.configuration.RetryQueryConfiguration; import edu.java.configuration.supplier.GithubConfig; import edu.java.configuration.supplier.GithubPatternConfig; import edu.java.supplier.api.LinkInfo; @@ -10,6 +10,7 @@ import java.net.URI; import java.time.Duration; import java.util.List; +import java.util.Map; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -33,8 +34,7 @@ public class GithubInfoSupplierTest { ); private static final RetryQueryConfiguration RETRY_QUERY_CONFIGURATION = new RetryQueryConfiguration( - List.of(new RetryQueryConfiguration.RetryElement( - "github", + Map.of("github", new RetryQueryConfiguration.RetryElement( "fixed", 1, 1, diff --git a/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java b/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java index c39cfe0..f2c7362 100644 --- a/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; -import edu.java.RetryQueryConfiguration; +import edu.java.configuration.RetryQueryConfiguration; import edu.java.configuration.supplier.StackOverflowConfig; import edu.java.configuration.supplier.StackOverflowPatternConfig; import edu.java.supplier.api.LinkInfo; @@ -10,6 +10,7 @@ import java.net.URI; import java.time.Duration; import java.util.List; +import java.util.Map; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -24,8 +25,7 @@ public class StackOverflowInfoSupplierTest { private static WireMockServer server; private static final RetryQueryConfiguration RETRY_QUERY_CONFIGURATION = new RetryQueryConfiguration( - List.of(new RetryQueryConfiguration.RetryElement( - "stackoverflow", + Map.of("stackoverflow", new RetryQueryConfiguration.RetryElement( "fixed", 1, 1, diff --git a/scrapper/src/test/java/edu/java/scrapper/updates/KafkaLinkUpdateSenderTest.java b/scrapper/src/test/java/edu/java/scrapper/updates/KafkaLinkUpdateSenderTest.java index d2fc82b..5c374cb 100644 --- a/scrapper/src/test/java/edu/java/scrapper/updates/KafkaLinkUpdateSenderTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/updates/KafkaLinkUpdateSenderTest.java @@ -6,15 +6,15 @@ import edu.java.updates.sender.KafkaLinkUpdateSender; import edu.java.updates.sender.LinkUpdateSender; import edu.java.util.URLCreator; +import java.time.Duration; +import java.util.List; +import java.util.Map; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.kafka.core.KafkaTemplate; -import java.time.Duration; -import java.util.List; -import java.util.Map; import static org.awaitility.Awaitility.await; @SpringBootTest