[Backport 2.19-dev] Publish internal modules separately for downstream reuse (#4723)

* Publish internal modules separately for downstream reuse (#4484)

Co-authored-by: Louis Chu <[email protected]>
Co-authored-by: Chen Dai <[email protected]>
Co-authored-by: Mebsina <[email protected]>
Signed-off-by: Chen Dai <[email protected]>

* Add test branch and downgrade JDK in maven publish workflow

Signed-off-by: Chen Dai <[email protected]>

* Add 2.19-dev branch to workflow and bump Calcite version in api module

Signed-off-by: Chen Dai <[email protected]>

---------

Signed-off-by: Chen Dai <[email protected]>
Co-authored-by: Simeon Widdis <[email protected]>
Co-authored-by: Louis Chu <[email protected]>
Co-authored-by: Mebsina <[email protected]>

Author: 4 people

.github/workflows/maven-publish-modules.yml

@@ -0,0 +1,48 @@
+name: Publish unified query modules to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+      - 2.19-dev
+
+env:
+  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
+
+jobs:
+  publish-unified-query-modules:
+    strategy:
+      fail-fast: false
+    if: github.repository == 'opensearch-project/sql'
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 11
+      - uses: actions/checkout@v3
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
+        with:
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          ./gradlew publishUnifiedQueryPublicationToSnapshotsRepository

api/README.md

@@ -0,0 +1,75 @@
+# Unified Query API
+
+This module provides a high-level integration layer for the Calcite-based query engine, enabling external systems such as Apache Spark or command-line tools to parse and analyze queries without exposing low-level internals.
+
+## Overview
+
+The `UnifiedQueryPlanner` serves as the primary entry point for external consumers. It accepts PPL (Piped Processing Language) queries and returns Calcite `RelNode` logical plans as intermediate representation.
+
+## Usage
+
+Use the declarative, fluent builder API to initialize the `UnifiedQueryPlanner`.
+
+```java
+UnifiedQueryPlanner planner = UnifiedQueryPlanner.builder()
+    .language(QueryType.PPL)
+    .catalog("opensearch", schema)
+    .defaultNamespace("opensearch")
+    .cacheMetadata(true)
+    .build();
+
+RelNode plan = planner.plan("source = opensearch.test");
+```
+
+## Development & Testing
+
+A set of unit tests is provided to validate planner behavior.
+
+To run tests:
+
+```
+./gradlew :api:test
+```
+
+## Integration Guide
+
+This guide walks through how to integrate unified query planner into your application.
+
+### Step 1: Add Dependency
+
+The module is currently published as a snapshot to the AWS Sonatype Snapshots repository. To include it as a dependency in your project, add the following to your `pom.xml` or `build.gradle`:
+
+```xml
+<dependency>
+  <groupId>org.opensearch.query</groupId>
+  <artifactId>unified-query-api</artifactId>
+  <version>YOUR_VERSION_HERE</version>
+</dependency>
+```
+
+### Step 2: Implement a Calcite Schema
+
+You must implement the Calcite `Schema` interface and register them using the fluent `catalog()` method on the builder.
+
+```java
+public class MySchema extends AbstractSchema {
+  @Override
+  protected Map<String, Table> getTableMap() {
+    return Map.of(
+      "test_table",
+      new AbstractTable() {
+        @Override
+        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+          return typeFactory.createStructType(
+            List.of(typeFactory.createSqlType(SqlTypeName.INTEGER)),
+            List.of("id"));
+        }
+      });
+  }
+}
+```
+
+## Future Work
+
+- Expand support to SQL language.
+- Extend planner to generate optimized physical plans using Calcite's optimization frameworks.

api/build.gradle

@@ -0,0 +1,71 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+plugins {
+    id 'java-library'
+    id 'jacoco'
+    id 'com.diffplug.spotless'
+}
+
+dependencies {
+    api project(':ppl')
+
+    testImplementation group: 'junit', name: 'junit', version: '4.13.2'
+    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
+    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0'
+}
+
+spotless {
+    java {
+        target fileTree('.') {
+            include '**/*.java'
+            exclude '**/build/**', '**/build-*/**', 'src/main/gen/**'
+        }
+        importOrder()
+        removeUnusedImports()
+        trimTrailingWhitespace()
+        endWithNewline()
+        googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format')
+    }
+}
+
+test {
+    testLogging {
+        events "passed", "skipped", "failed"
+        exceptionFormat "full"
+    }
+}
+
+jacocoTestReport {
+    reports {
+        html.required = true
+        xml.required = true
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it,
+                    exclude: ['**/antlr/parser/**'])
+        }))
+    }
+}
+test.finalizedBy(project.tasks.jacocoTestReport)
+jacocoTestCoverageVerification {
+    violationRules {
+        rule {
+            limit {
+                minimum = 0.9
+            }
+
+        }
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it,
+                    exclude: ['**/antlr/parser/**'])
+        }))
+    }
+}
+check.dependsOn jacocoTestCoverageVerification

api/src/main/java/org/opensearch/sql/api/EmptyDataSourceService.java

@@ -0,0 +1,51 @@
+package org.opensearch.sql.api;
+
+import java.util.Map;
+import java.util.Set;
+import org.opensearch.sql.datasource.DataSourceService;
+import org.opensearch.sql.datasource.RequestContext;
+import org.opensearch.sql.datasource.model.DataSource;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+
+/** A DataSourceService that assumes no access to data sources */
+public class EmptyDataSourceService implements DataSourceService {
+  public EmptyDataSourceService() {}
+
+  @Override
+  public DataSource getDataSource(String dataSourceName) {
+    return null;
+  }
+
+  @Override
+  public Set<DataSourceMetadata> getDataSourceMetadata(boolean isDefaultDataSourceRequired) {
+    return Set.of();
+  }
+
+  @Override
+  public DataSourceMetadata getDataSourceMetadata(String name) {
+    return null;
+  }
+
+  @Override
+  public void createDataSource(DataSourceMetadata metadata) {}
+
+  @Override
+  public void updateDataSource(DataSourceMetadata dataSourceMetadata) {}
+
+  @Override
+  public void patchDataSource(Map<String, Object> dataSourceData) {}
+
+  @Override
+  public void deleteDataSource(String dataSourceName) {}
+
+  @Override
+  public Boolean dataSourceExists(String dataSourceName) {
+    return false;
+  }
+
+  @Override
+  public DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata(
+      String dataSourceName, RequestContext context) {
+    return null;
+  }
+}