Try using checks API

.github/workflows/ci.yml

@@ -11,6 +11,7 @@ jobs:
         with:
           github_token: ${{ secrets.github_token }}
           reporter: github-check
+
   rspec:
     runs-on: ubuntu-latest
     steps:
@@ -33,8 +34,9 @@ jobs:
           bundle config path vendor/bundle
           bundle install --jobs 4 --retry 3
       - name: Run RSpec
-        run: bundle exec rspec spec/rspec
-  e2e:
+        run: bundle exec rspec spec/rspec --format RSpec::Github::Formatter
+
+  e2e-formatter:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
@@ -56,8 +58,34 @@ jobs:
           bundle config path vendor/bundle
           bundle install --jobs 4 --retry 3
       - name: Run RSpec in GITHUB_WORKSPACE
-        run: '! bundle exec rspec spec/integration/failing_spec.rb'
+        run: '! bundle exec rspec spec/integration/failing_spec.rb --format RSpec::Github::Formatter'
       - name: Run RSpec in sub-directory of GITHUB_WORKSPACE
         run: |
           cd spec/integration
           bundle exec rspec relative_path/pending_spec.rb --require ../spec_helper --format RSpec::Github::Formatter
+
+  e2e-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v1
+      - name: Setup caching for ruby gems
+        uses: actions/cache@v1
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+      - name: Setup Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Install gems
+        run: |
+          gem install bundler:2.1.4 --no-doc
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+      - name: Run RSpec with Annotator
+        env:
+          OCTOKIT_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: '! bundle exec rspec spec/integration/failing_spec.rb --format RSpec::Github::Annotator'

.rspec

@@ -1,4 +1,3 @@
 --color
 --require spec_helper
 --format documentation
---format RSpec::Github::Formatter

Gemfile.lock

@@ -2,17 +2,28 @@ PATH
   remote: .
   specs:
     rspec-github (1.0.0.develop)
+      octokit (~> 4.0)
       rspec-core (~> 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
     diff-lcs (1.3)
+    faraday (1.1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
     jaro_winkler (1.5.4)
+    multipart-post (2.1.1)
+    octokit (4.19.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
     parallel (1.19.1)
     parser (2.7.1.0)
       ast (~> 2.4.0)
+    public_suffix (4.0.6)
     rainbow (3.0.0)
     rexml (3.2.4)
     rspec (3.9.0)
@@ -37,6 +48,10 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
     ruby-progressbar (1.10.1)
+    ruby2_keywords (0.0.2)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
     unicode-display_width (1.7.0)
 
 PLATFORMS

lib/rspec/github/annotator.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'octokit'
+require 'rspec/core'
+require 'rspec/core/formatters/base_formatter'
+
+module RSpec
+  module Github
+    class Annotator < RSpec::Core::Formatters::BaseFormatter
+      RSpec::Core::Formatters.register self, :start, :dump_failures, :dump_pending, :close
+
+      REQUIRED_ENV_VARIABLES = %w[OCTOKIT_ACCESS_TOKEN GITHUB_REPOSITORY GITHUB_SHA]
+
+      def start(notification)
+        missing_env_variables = REQUIRED_ENV_VARIABLES - ENV.keys
+        raise "Missing environment variables: #{missing_env_variables}" if missing_env_variables.any?
+
+        # Call check_run to create the pending check run
+        check_run
+      end
+
+      def dump_failures(examples_notification)
+        @failures = examples_notification.failure_notifications
+      end
+
+      def dump_pending(examples_notification)
+        @pending = examples_notification.pending_notifications
+      end
+
+      def close(null_notification)
+        annotations.each_slice(50) do |annotations_group|
+          octokit_client.update_check_run(
+            ENV.fetch('GITHUB_REPOSITORY'),
+            check_run.id,
+            status: 'completed',
+            conclusion: conclusion,
+            output: {
+              title: 'RSpec output',
+              summary: 'RSpec output',
+              annotations: annotations_group
+            },
+            accept: Octokit::Preview::PREVIEW_TYPES[:checks]
+          )
+        end
+      end
+
+      private
+
+      def conclusion
+        return 'failure' if @failures.any?
+        return 'neutral' if @pending.any?
+
+        'success'
+      end
+
+      def annotations
+        (@failures + @pending).map do |notification|
+          NotificationDecorator.new(notification)
+        end
+      end
+
+      def check_run
+        @check_run ||= octokit_client.create_check_run(
+          ENV.fetch('GITHUB_REPOSITORY'),
+          'RSpec',
+          ENV.fetch('GITHUB_SHA'),
+          status: 'in_progress',
+          accept: Octokit::Preview::PREVIEW_TYPES[:checks]
+        )
+      end
+
+      def octokit_client
+        @octokit_client ||= Octokit::Client.new
+      end
+    end
+  end
+end

lib/rspec/github/notification_decorator.rb

@@ -15,7 +15,7 @@ def initialize(notification)
       end
 
       def line
-        example.location.split(':')[1]
+        example.location.split(':')[1].to_i
       end
 
       def annotation
@@ -27,6 +27,21 @@ def path
         File.realpath(raw_path).delete_prefix("#{workspace}#{File::SEPARATOR}")
       end
 
+      def to_h
+        {
+          path: path,
+          start_line: line,
+          end_line: line,
+          annotation_level: 'warning',
+          title: example.full_description,
+          message: message
+        }
+      end
+
+      def to_json(*args)
+        to_h.to_json
+      end
+
       private
 
       def example

rspec-github.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   spec.files = Dir['{lib}/**/*']
 
+  spec.add_runtime_dependency 'octokit', '~> 4.0'
   spec.add_runtime_dependency 'rspec-core', '~> 3.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 0.81.0'