Javadoc site crawler

.github/workflows/javadoc-crawler.yml

@@ -0,0 +1,26 @@
+name: Javadoc.io site crawler (daily)
+
+on:
+  schedule:
+    - cron: "30 1 * * *" # daily at 1:30 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  crawl:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Set up gradle
+        uses: gradle/actions/setup-gradle@06832c7b30a0129d7fb559bcc6e43d26f6374244 # v4.3.1
+
+      - name: Run crawler
+        run: ./gradlew :javadoc-crawler:crawl

javadoc-crawler/README.md

@@ -0,0 +1,20 @@
+# Javadoc Crawler
+
+## Context
+
+The javadocs.io website lazy loads content only when the artifacts have been accessed, which can
+lead to inaccuracies and confusion when someone loads the
+https://www.javadoc.io/doc/io.opentelemetry page, since the published `Latest version` will only be
+accurate if someone has accessed the page for the actual latest version.
+
+This module provides a simple scraper that pulls the list of all `io.opentelemetry` artifacts from
+maven central and then visits each corresponding page on the javadoc.io website in order to trigger
+loading them into the site's system.
+
+See https://github.com/open-telemetry/opentelemetry-java/issues/7294 for more information.
+
+## How to run
+
+```bash
+./gradlew :javadoc-crawler:crawl
+```

javadoc-crawler/build.gradle.kts

@@ -0,0 +1,34 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+dependencies {
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+  testImplementation("org.assertj:assertj-core:3.27.3")
+}
+
+description = "OpenTelemetry Javadoc Crawler"
+otelJava.moduleName.set("io.opentelemetry.javadocs")
+
+tasks {
+  withType<JavaCompile>().configureEach {
+    sourceCompatibility = "17"
+    targetCompatibility = "17"
+    options.release.set(17)
+  }
+
+  // only test on java 17+
+  val testJavaVersion: String? by project
+  if (testJavaVersion != null && Integer.valueOf(testJavaVersion) < 17) {
+    test {
+      enabled = false
+    }
+  }
+
+  val crawl by registering(JavaExec::class) {
+    dependsOn(classes)
+
+    mainClass.set("io.opentelemetry.javadocs.JavaDocsCrawler")
+    classpath(sourceSets["main"].runtimeClasspath)
+  }
+}

javadoc-crawler/src/main/java/io/opentelemetry/javadocs/Artifact.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javadocs;
+
+public class Artifact {
+  private final String name;
+  private final String version;
+
+  public Artifact(String name, String version) {
+    this.name = name;
+    this.version = version;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+}

javadoc-crawler/src/main/java/io/opentelemetry/javadocs/JavaDocsCrawler.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javadocs;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * The javadoc.io site relies on someone accessing the page for an artifact version in order to
+ * update the contents of the site. This will query Maven Central for all artifacts under
+ * io.opentelemetry in order to identify the latest versions. Then it will crawl the associated
+ * pages on the javadoc.io site to trigger updates.
+ */
+public final class JavaDocsCrawler {
+  private static final String GROUP = "io.opentelemetry";
+  private static final String MAVEN_CENTRAL_BASE_URL =
+      "https://search.maven.org/solrsearch/select?q=g:";
+  private static final String JAVA_DOCS_BASE_URL = "https://javadoc.io/doc/";
+  private static final int PAGE_SIZE = 20;
+  private static final int THROTTLE_MS = 500;
+
+  // visible for testing
+  static final String JAVA_DOC_DOWNLOADED_TEXT = "Javadoc is being downloaded";
+
+  private static final Logger logger = Logger.getLogger(JavaDocsCrawler.class.getName());
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  public static void main(String[] args) throws Exception {
+    HttpClient client = HttpClient.newHttpClient();
+    List<Artifact> artifacts = getArtifacts(client);
+    if (artifacts.isEmpty()) {
+      logger.log(Level.SEVERE, "No artifacts found");
+      return;
+    }
+    logger.info(String.format(Locale.ROOT, "Found %d artifacts", artifacts.size()));
+
+    List<String> updated = crawlJavaDocs(client, artifacts);
+    if (updated.isEmpty()) {
+      logger.info("No updates were needed");
+      return;
+    }
+
+    logger.info("Artifacts that triggered updates:\n" + String.join("\n", updated));
+  }
+
+  static List<Artifact> getArtifacts(HttpClient client) throws IOException, InterruptedException {
+    int start = 0;
+    Integer numFound;
+    List<Artifact> result = new ArrayList<>();
+
+    do {
+      if (start != 0) {
+        Thread.sleep(THROTTLE_MS); // try not to DDoS the site, it gets knocked over easily
+      }
+
+      Map<?, ?> map = queryMavenCentral(client, start);
+
+      numFound =
+          Optional.ofNullable(map)
+              .map(mavenResult -> (Map<?, ?>) mavenResult.get("response"))
+              .map(response -> (Integer) response.get("numFound"))
+              .orElse(null);
+
+      List<Artifact> artifacts = convertToArtifacts(map);
+      result.addAll(artifacts);
+
+      start += PAGE_SIZE;
+    } while (numFound != null && start < numFound);
+
+    return result;
+  }
+
+  private static List<Artifact> convertToArtifacts(Map<?, ?> map) {
+    return Optional.ofNullable(map)
+        .map(mavenResults -> (Map<?, ?>) mavenResults.get("response"))
+        .map(response -> (List<?>) response.get("docs"))
+        .map(
+            docs -> {
+              List<Artifact> artifacts = new ArrayList<>();
+              for (Object doc : docs) {
+                Map<?, ?> docMap = (Map<?, ?>) doc;
+                String artifact = (String) docMap.get("a");
+                String version = (String) docMap.get("latestVersion");
+                if (artifact != null && version != null) {
+                  artifacts.add(new Artifact(artifact, version));
+                }
+              }
+              return artifacts;
+            })
+        .orElseGet(ArrayList::new);
+  }
+
+  private static Map<?, ?> queryMavenCentral(HttpClient client, int start)
+      throws IOException, InterruptedException {
+    URI uri =
+        URI.create(
+            String.format(
+                Locale.ROOT,
+                "%s%s&rows=%d&start=%d&wt=json",
+                MAVEN_CENTRAL_BASE_URL,
+                GROUP,
+                PAGE_SIZE,
+                start));
+
+    HttpRequest request = HttpRequest.newBuilder(uri).GET().build();
+
+    HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+    if (response.statusCode() != 200) {
+      logger.log(
+          Level.SEVERE,
+          "Unexpected response code: " + response.statusCode() + ": " + response.body());
+      throw new IOException("Unable to pull Maven central artifacts list");
+    }
+    return objectMapper.readValue(response.body(), Map.class);
+  }
+
+  static List<String> crawlJavaDocs(HttpClient client, List<Artifact> artifacts)
+      throws IOException, InterruptedException {
+    List<String> updatedArtifacts = new ArrayList<>();
+
+    for (Artifact artifact : artifacts) {
+      String[] parts = artifact.getName().split("-");
+      StringBuilder path = new StringBuilder();
+      path.append(JAVA_DOCS_BASE_URL)
+          .append(GROUP)
+          .append("/")
+          .append(artifact.getName())
+          .append("/")
+          .append(artifact.getVersion())
+          .append("/")
+          .append(String.join("/", parts))
+          .append("/package-summary.html");
+
+      HttpRequest crawlRequest = HttpRequest.newBuilder(URI.create(path.toString())).GET().build();
+      HttpResponse<String> crawlResponse =
+          client.send(crawlRequest, HttpResponse.BodyHandlers.ofString());
+
+      // gets a status code 303 when version exists and the site redirects it to use /latest/
+      if (crawlResponse.statusCode() != 200 && crawlResponse.statusCode() != 303) {
+        logger.log(
+            Level.WARNING,
+            String.format(
+                Locale.ROOT,
+                "Crawl failed for %s with status code %d at URL %s\nResponse: %s",
+                artifact.getName(),
+                crawlResponse.statusCode(),
+                path,
+                crawlResponse.body()));
+        continue;
+      }
+
+      if (crawlResponse.body().contains(JAVA_DOC_DOWNLOADED_TEXT)) {
+        updatedArtifacts.add(artifact.getName());
+      }
+
+      Thread.sleep(THROTTLE_MS); // some light throttling
+    }
+    return updatedArtifacts;
+  }
+
+  private JavaDocsCrawler() {}
+}

javadoc-crawler/src/test/java/io/opentelemetry/javadocs/JavaDocsCrawlerTest.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javadocs;
+
+import static io.opentelemetry.javadocs.JavaDocsCrawler.JAVA_DOC_DOWNLOADED_TEXT;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class JavaDocsCrawlerTest {
+  @Mock HttpClient mockClient;
+  @Mock HttpResponse<Object> mockMavenCentralRequest1;
+  @Mock HttpResponse<Object> mockMavenCentralRequest2;
+  @Mock HttpResponse<Object> mockJavaDocResponse;
+
+  @Test
+  void testGetArtifactsHandlesPagination() throws IOException, InterruptedException {
+    String page1Response =
+        """
+            {
+              "response": {
+                "numFound": 40,
+                "docs": [
+                  {"a": "artifact1", "latestVersion": "1.0"},
+                  {"a": "artifact2", "latestVersion": "1.1"}
+                ]
+              }
+            }
+        """;
+    String page2Response =
+        """
+            {
+              "response": {
+                "numFound": 40,
+                "docs": [
+                  {"a": "artifact3", "latestVersion": "2.0"}
+                ]
+              }
+            }
+        """;
+
+    when(mockMavenCentralRequest1.body()).thenReturn(page1Response);
+    when(mockMavenCentralRequest1.statusCode()).thenReturn(200);
+    when(mockMavenCentralRequest2.body()).thenReturn(page2Response);
+    when(mockMavenCentralRequest2.statusCode()).thenReturn(200);
+
+    when(mockClient.send(any(), any()))
+        .thenReturn(mockMavenCentralRequest1)
+        .thenReturn(mockMavenCentralRequest2);
+
+    List<Artifact> artifacts = JavaDocsCrawler.getArtifacts(mockClient);
+
+    // 2 calls for the pagination
+    verify(mockClient, times(2)).send(any(), any());
+    assertThat(artifacts.size()).isEqualTo(3);
+  }
+
+  @Test
+  void testCrawler() throws IOException, InterruptedException {
+    List<Artifact> artifacts = new ArrayList<>();
+    artifacts.add(new Artifact("opentelemetry-context", "1.49.0"));
+    ArgumentCaptor<HttpRequest> requestCaptor = ArgumentCaptor.forClass(HttpRequest.class);
+
+    when(mockJavaDocResponse.body()).thenReturn(JAVA_DOC_DOWNLOADED_TEXT);
+    when(mockJavaDocResponse.statusCode()).thenReturn(200);
+
+    when(mockClient.send(any(), any())).thenReturn(mockJavaDocResponse);
+
+    List<String> updated = JavaDocsCrawler.crawlJavaDocs(mockClient, artifacts);
+
+    verify(mockClient, times(1)).send(requestCaptor.capture(), any());
+
+    assertThat(requestCaptor.getValue().uri().toString())
+        .isEqualTo(
+            "https://javadoc.io/doc/io.opentelemetry/opentelemetry-context/1.49.0/opentelemetry/context/package-summary.html");
+    assertThat(updated).containsExactly("opentelemetry-context");
+  }
+}

settings.gradle.kts

@@ -51,6 +51,7 @@ include(":integration-tests:otlp")
 include(":integration-tests:tracecontext")
 include(":integration-tests:graal")
 include(":integration-tests:graal-incubating")
+include(":javadoc-crawler")
 include(":opencensus-shim")
 include(":opentracing-shim")
 include(":perf-harness")