Update README.md

.buildkite/pipeline.yml

@@ -0,0 +1,15 @@
+agents:
+  provider: "gcp"
+  machineType: "n1-standard-8"
+
+defaultTimeoutInMinutes: 45
+
+steps:
+  - label: ":safety_vest: Connectors Tests"
+    commands:
+      - ".buildkite/scripts/run_command.sh tests"
+    artifact_paths:
+      - "coverage/index.html"
+  - label: ":wrench: Linter"
+    commands:
+      - ".buildkite/scripts/run_command.sh linter"

.buildkite/scripts/run_ci_step.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+export PATH="$PATH:/root/.rbenv/bin:/root/.rbenv/plugins/ruby-build/bin:/ci/.rbenv/shims"
+
+RUBY_VERSION=$(cat .ruby-version)
+echo "---- installing Ruby version $RUBY_VERSION"
+rbenv install $RUBY_VERSION
+rbenv global $RUBY_VERSION
+
+case $1 in
+
+  tests)
+    echo "---- running unit tests"
+    make install test
+    ;;
+
+  linter)
+    echo "---- running linter"
+    make install lint
+    ;;
+
+  packaging)
+    echo "---- running packaging"
+    git config --global --add safe.directory '*'
+    git config --global --add safe.directory /ci
+    curl -L -o yq https://github.com/mikefarah/yq/releases/download/v4.21.1/yq_linux_amd64
+    chmod +x yq
+    YQ=`realpath yq` make install build_service build_service_gem
+    gem install .gems/connectors_service-8.*
+    ;;
+
+  *)
+    echo "Usage: run_command {tests|linter|packaging}"
+    exit 2
+    ;;
+esac

.buildkite/scripts/run_command.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+COMMAND_TO_RUN=${1:-}
+
+if [[ "${COMMAND_TO_RUN:-}" == "" ]]; then
+    echo "Usage: run_command.sh {tests|linter|docker|packaging}"
+    exit 2
+fi
+
+function realpath {
+  echo "$(cd "$(dirname "$1")"; pwd)"/"$(basename "$1")";
+}
+
+SCRIPT_WORKING_DIR=$(realpath "$(dirname "$0")")
+BUILDKITE_DIR=$(realpath "$(dirname "$SCRIPT_WORKING_DIR")")
+PROJECT_ROOT=$(realpath "$(dirname "$BUILDKITE_DIR")")
+
+if [[ "${COMMAND_TO_RUN:-}" == "docker" ]]; then
+  echo "running docker build"
+  make build-docker
+else
+  DOCKER_IMAGE="docker.elastic.co/ci-agent-images/enterprise-search/rbenv-buildkite-agent:latest"
+  SCRIPT_CMD="/ci/.buildkite/scripts/run_ci_step.sh"
+
+  docker run --interactive --rm             \
+              --sig-proxy=true --init      \
+              --user "root"                \
+              --volume "$PROJECT_ROOT:/ci" \
+              --workdir /ci                \
+              --env HOME=/ci               \
+              --env CI                     \
+              --env GIT_REVISION=${BUILDKITE_COMMIT-}        \
+              --env BUILD_ID=${BUILDKITE_BUILD_NUMBER-}      \
+              --entrypoint "${SCRIPT_CMD}" \
+              $DOCKER_IMAGE                \
+              $COMMAND_TO_RUN
+fi

Makefile

@@ -14,6 +14,9 @@ lint: config/connectors.yml
 autocorrect: config/connectors.yml
 	bundle _$(shell cat .bundler-version)_ exec rubocop lib spec -a
 
+autocorrect_all: config/connectors.yml
+	bundle _$(shell cat .bundler-version)_ exec rubocop lib spec -A
+
 api_key: config/connectors.yml
 	${YQ} e ".http.api_key = \"$(shell uuidgen | tr -d '-')\"" -i config/connectors.yml
 
@@ -41,14 +44,16 @@ tag:
 
 build_gem:
 	mkdir -p .gems
-	gem build connectors_sdk.gemspec
-	gem build connectors_stubs.gemspec
+	bundle _$(shell cat .bundler-version)_ exec gem build connectors_sdk.gemspec
+	bundle _$(shell cat .bundler-version)_ exec gem build connectors_stubs.gemspec
 	rm -f .gems/*
 	mv *.gem .gems/
 	echo "DO NOT FORGET TO UPDATE ENT-SEARCH"
 
 push_gem:
-	gem push .gems/*
+	gem push .gems/connectors_sdk*
+	gem push .gems/connectors_stubs*
+
 
 install:
 	rbenv install -s

README.md

@@ -1,8 +1,18 @@
-# Elastic Enterprise Search Connectors 
+# Elastic Workplace Search Connectors 
 
-![logo](logo-enterprise-search.png)
+> [!IMPORTANT]
+> _**Enterprise Search will be discontinued in 9.0.**_
+>
+> Starting with Elastic version 9.0, we're deprecating the standalone Enterprise Search product with its included features and functionalities (including [Workplace Search](https://www.elastic.co/guide/en/workplace-search/8.x/index.html) and [App Search](https://www.elastic.co/guide/en/app-search/8.x/index.html)). They remain supported in their current form in version 8.x and will only receive security upgrades and fixes. Workplace Search Connector Packages will continue to be supported in their current form throughout 8.x versions, according to our EOL policy: https://www.elastic.co/support/eol.
+> We recommend transitioning to our actively developed [Elastic Stack](https://www.elastic.co/elastic-stack) tools for your search use cases. However, if you're still using any Enterprise Search products, we recommend using the latest stable release.
+>
+> Here are some useful links with more information:
+> * Enterprise Search FAQ: https://www.elastic.co/resources/enterprise-search/enterprise-search-faq
+> * Migrating to 9.x from Enterprise Search 8.x versions: https://www.elastic.co/guide/en/enterprise-search/current/upgrading-to-9-x.html
+___
 
-The home of Elastic Enterprise Connector Packages. Use connector packages to
+
+The home of Workplace Search Connector Packages. Use connector packages to
 customize connectors such as Workplace Search Content Sources for advanced use
 cases.
 
@@ -97,7 +107,7 @@ This repository is always growing! At the moment, the connectors currently avail
 
 Don't see the connector you're looking for? If it is in the list of [Workplace Search Content Sources](https://www.elastic.co/guide/en/workplace-search/current/workplace-search-content-sources.html), it is probably on our roadmap to move here! See our [Getting Support](./docs/SUPPORT.md) guide for ways to reach out.
 
-Not seeing the connector you want there, either? We encourage community contirubions! See our [Contributors Guide](./docs/CONTRIBUTING.md).
+Not seeing the connector you want there, either? We encourage community contributions! See our [Contributors Guide](./docs/CONTRIBUTING.md).
 
 ### Sinatra Console
 run `make console`

VERSION

@@ -1 +1 @@
-8.4.0.0
+8.3.4.0

connectors_sdk.gemspec

@@ -3,38 +3,38 @@ require_relative 'lib/connectors_app/config'
 Gem::Specification.new do |s|
   s.name        = 'connectors_sdk'
   s.version     = ConnectorsApp::Config['version']
-  s.homepage    = "https://github.com/elastic/connectors"
+  s.homepage    = 'https://github.com/elastic/connectors'
   s.summary     = 'Gem containing apis used by Enterprise Search and implementations of Connectors'
   s.description = ''
   s.authors     = ['Elastic']
   s.email       = '[email protected]'
   s.metadata    = {
-    "revision" => ConnectorsApp::Config['revision'],
-    "repository" => ConnectorsApp::Config['repository']
+    'revision' => ConnectorsApp::Config['revision'],
+    'repository' => ConnectorsApp::Config['repository']
   }
-  s.files       = Dir.glob("lib/connectors_sdk/**/*", File::FNM_DOTMATCH) +
-    Dir.glob("lib/connectors_shared/**/*", File::FNM_DOTMATCH) +
-    [
-      'LICENSE',
-      'NOTICE.txt',
-      'lib/connectors_sdk.rb',
-      'lib/connectors_shared.rb'
-    ]
+  s.files       = Dir.glob('lib/connectors_sdk/**/*', File::FNM_DOTMATCH) +
+                  Dir.glob('lib/connectors_shared/**/*', File::FNM_DOTMATCH) +
+                  [
+                    'LICENSE',
+                    'NOTICE.txt',
+                    'lib/connectors_sdk.rb',
+                    'lib/connectors_shared.rb'
+                  ]
   s.license     = 'Elastic-2.0'
 
   # TODO: figure out how to pin versions without harming ent-search repo
   s.add_dependency 'activesupport'
   s.add_dependency 'bson'
   s.add_dependency 'mime-types'
-  s.add_dependency 'tzinfo-data'
   s.add_dependency 'nokogiri'
+  s.add_dependency 'tzinfo-data'
 
   # Dependencies for the HTTP service
-  s.add_dependency 'forwardable'
   s.add_dependency 'faraday'
   s.add_dependency 'faraday_middleware'
-  s.add_dependency 'httpclient'
+  s.add_dependency 'forwardable'
   s.add_dependency 'hashie'
+  s.add_dependency 'httpclient'
 
   # Dependencies for oauth
   s.add_dependency 'signet'

connectors_stubs.gemspec

@@ -3,20 +3,20 @@ require_relative 'lib/connectors_app/config'
 Gem::Specification.new do |s|
   s.name        = 'connectors_stubs'
   s.version     = ConnectorsApp::Config['version']
-  s.homepage    = "https://github.com/elastic/connectors"
+  s.homepage    = 'https://github.com/elastic/connectors'
   s.summary     = 'Gem containing utilities used by implementations of Connectors when run external to Enterprise Search'
   s.description = ''
   s.authors     = ['Elastic']
   s.email       = '[email protected]'
   s.metadata    = {
-    "revision" => ConnectorsApp::Config['revision'],
-    "repository" => ConnectorsApp::Config['repository']
+    'revision' => ConnectorsApp::Config['revision'],
+    'repository' => ConnectorsApp::Config['repository']
   }
-  s.files       = Dir.glob("lib/stubs/**/*", File::FNM_DOTMATCH) +
-    [
-      'LICENSE',
-      'NOTICE.txt'
-    ]
+  s.files       = Dir.glob('lib/stubs/**/*', File::FNM_DOTMATCH) +
+                  [
+                    'LICENSE',
+                    'NOTICE.txt'
+                  ]
   s.license     = 'Elastic-2.0'
 
   s.add_dependency 'activesupport'

lib/connectors_async/job_runner.rb

@@ -30,15 +30,17 @@ def start_job(job:, connector_class:, secret_storage:, params:)
         init_thread
 
         connector = connector_class.new
-        content_source_id = params[:content_source_id]
-        cursors = params[:cursors] ||= {}
-        cursors[:modified_since] = params.delete(:modified_since) if params[:modified_since]
+        extractor_params = params.dup
+        cursors = extractor_params[:cursors] ||= {}
+        cursors[:modified_since] = extractor_params.delete(:modified_since) if extractor_params[:modified_since]
+        extractor_params[:cursors] = cursors
+        extractor_params[:secret_storage] = secret_storage
 
         log_with_thread_id(:info, "Running the job #{job.id}")
 
         job.update_status(ConnectorsShared::JobStatus::RUNNING)
 
-        new_cursors = connector.extract({ :content_source_id => content_source_id, :cursors => cursors, :secret_storage => secret_storage }) do |doc|
+        new_cursors = connector.extract(extractor_params) do |doc|
           with_throttling(job) do
             job.store(doc)
           end

lib/connectors_sdk/confluence_cloud/connector.rb

@@ -6,6 +6,7 @@
 
 # frozen_string_literal: true
 
+require 'base64'
 require 'connectors_sdk/atlassian/config'
 require 'connectors_sdk/confluence_cloud/extractor'
 require 'connectors_sdk/confluence_cloud/authorization'
@@ -29,23 +30,19 @@ def display_name
         'Confluence Cloud'
       end
 
-      def connection_requires_redirect
-        true
-      end
-
       def configurable_fields
         [
           {
             'key' => 'base_url',
-            'label' => 'Base URL'
+            'label' => 'Confluence Cloud Base URL'
           },
           {
-            'key' => 'client_id',
-            'label' => 'Client ID'
+            'key' => 'confluence_user_email',
+            'label' => 'Confluence user email'
           },
           {
-            'key' => 'client_secret',
-            'label' => 'Client Secret'
+            'key' => 'confluence_api_token',
+            'label' => 'Confluence user REST API Token'
           },
         ]
       end
@@ -61,15 +58,22 @@ def authorization
       end
 
       def client(params)
-        ConnectorsSdk::ConfluenceCloud::CustomClient.new(:base_url => base_url(params[:cloud_id]), :access_token => params[:access_token])
+        ConnectorsSdk::ConfluenceCloud::CustomClient.new(
+          :base_url => extract_base_url(params),
+          :basic_auth_token => extract_basic_auth_token(params)
+        )
       end
 
       def custom_client_error
         ConnectorsSdk::Atlassian::CustomClient::ClientError
       end
 
       def config(params)
-        ConnectorsSdk::Atlassian::Config.new(:base_url => "#{params[:external_connector_base_url]}/wiki", :cursors => params.fetch(:cursors, {}) || {})
+        ConnectorsSdk::Atlassian::Config.new(
+          :base_url => extract_base_url(params),
+          :cursors => params.fetch(:cursors, {}) || {},
+          :index_permissions => params[:index_permissions] || false
+        )
       end
 
       def health_check(params)
@@ -79,6 +83,28 @@ def health_check(params)
       def base_url(cloud_id)
         "https://api.atlassian.com/ex/confluence/#{cloud_id}"
       end
+
+      def is_basic_auth(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        login.present? && api_token.present?
+      end
+
+      def extract_basic_auth_token(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        nil unless login.present? && api_token.present?
+        Base64.strict_encode64("#{login}:#{api_token}")
+      end
+
+      def extract_base_url(params)
+        # From Confluence API documentation:
+        # Requests that use OAuth 2.0 (3LO) are made via api.atlassian.com (not https://your-domain.atlassian.net).
+        if is_basic_auth(params)
+          return params[:base_url].end_with?('/wiki') ? params[:base_url] : "#{params[:base_url]}/wiki"
+        end
+        base_url(params[:cloud_id])
+      end
     end
   end
 end

lib/connectors_sdk/confluence_cloud/extractor.rb

@@ -52,7 +52,8 @@ def yield_permissions(source_user_id)
       def download(item)
         content = item[:content]
         parent_id = content.dig('container', 'id')
-        client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
+        base_url = client.base_url.end_with?('/wiki') ? client.base_url : "#{client.base_url}/wiki"
+        client.download("#{base_url}/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
       end
     end
   end

lib/connectors_sdk/share_point/authorization.rb

@@ -35,7 +35,7 @@ def token_credential_uri
         end
 
         def additional_parameters
-          { :prompt => 'consent' }
+          { :prompt => 'select_account' }
         end
       end
     end