Report formatter base

database/scripts/format_report.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require_relative 'lib/report_formatter'
+require 'json'
+
+report_name = ARGV.shift
+
+report = JSON.load_file(report_name)
+
+report['urgent']['build_regressions'] = ReportFormatter::build_regressions(report['urgent']['build_regressions'])
+report['urgent']['test_regressions_consecutive'] = ReportFormatter::test_regressions_consecutive(report['urgent']['test_regressions_consecutive'])
+report['urgent']['test_regressions_flaky'] = ReportFormatter::test_regressions_flaky(report['urgent']['test_regressions_flaky'])
+
+# Sample output:
+# puts report['urgent']['build_regressions']
+puts ReportFormatter::format_report report

database/scripts/generate_report.rb

@@ -28,8 +28,8 @@ def generate_report(report_name, exclude_set)
     report = {
         'urgent' => {
             'build_regressions' => urgent_build_regressions = BuildfarmToolsLib::build_regressions_today(filter_known: true),
-            'test_regressions_consecutive' => urgent_consistent_test_regressions = BuildfarmToolsLib::test_regressions_today(filter_known: true, only_consistent: true),
-            'test_regressions_flaky' => urgent_flaky_test_regressions = BuildfarmToolsLib::flaky_test_regressions(filter_known: true),
+            'test_regressions_consecutive' => urgent_consistent_test_regressions = BuildfarmToolsLib::test_regressions_today(filter_known: true, only_consistent: true, group_issues: true),
+            'test_regressions_flaky' => urgent_flaky_test_regressions = BuildfarmToolsLib::flaky_test_regressions(filter_known: true, group_issues: true),
        },
        'maintenance' => {
             'jobs_failing' => [],
@@ -40,7 +40,6 @@ def generate_report(report_name, exclude_set)
            'build_regressions_known' => [],
            'test_regressions_all' => [],
            'test_regressions_known' => [],
-           'build_regressions_known' => [],
        }
     }
 
@@ -51,3 +50,4 @@ def generate_report(report_name, exclude_set)
 end
 
 generate_report(options[:report_name], options[:exclude])
+puts options[:report_name]

database/scripts/lib/buildfarm_tools.rb

@@ -34,7 +34,7 @@ def self.error_appearances_in_job(test_name, job_name)
     run_command('./sql_run.sh error_appearances_in_job.sql', args: [test_name, job_name])
   end
 
-  def self.test_regressions_today(filter_known: false, only_consistent: false)
+  def self.test_regressions_today(filter_known: false, only_consistent: false, group_issues: false)
     # Keys: job_name, build_number, error_name, build_datetime, node_name
     out = run_command('./sql_run.sh errors_check_last_build.sql')
     if filter_known
@@ -45,15 +45,22 @@ def self.test_regressions_today(filter_known: false, only_consistent: false)
       out.filter! { |tr| tr['age'].to_i >= CONSECUTIVE_THRESHOLD || tr['age'].to_i == WARNING_AGE_CONSTANT }
       out.sort_by! { |tr| -tr['age'].to_i }
     end
+    out.each do |e|
+      e['reports'] = test_regression_reported_issues e['error_name']
+    end
+    if group_issues
+      # Group by (job_name, age)
+      out = out.group_by { |o| [o['job_name'], o['age']] }.to_a.map { |e| e[1] }
+    end
     out
   end
 
-  def self.flaky_test_regressions(filter_known: false, time_range: FLAKY_BUILDS_DEFAULT_RANGE)
+  def self.flaky_test_regressions(filter_known: false, group_issues: false, time_range: FLAKY_BUILDS_DEFAULT_RANGE)
     # Keys: job_name, build_number, error_name, build_datetime, node_name, flakiness
     out = []
     today_regressions = test_regressions_today(filter_known: filter_known)
     today_regressions.each do |tr|
-      next if !tr['age'].to_i.nil? && tr['age'].to_i >= CONSECUTIVE_THRESHOLD
+      next if !tr['age'].to_i.nil? && (tr['age'].to_i >= CONSECUTIVE_THRESHOLD || tr['age'].to_i == WARNING_AGE_CONSTANT)
 
       tr_flakiness = test_regression_flakiness(tr['error_name'], time_range: time_range)
       if tr_flakiness.nil?
@@ -66,6 +73,9 @@ def self.flaky_test_regressions(filter_known: false, time_range: FLAKY_BUILDS_DE
       end
     end
     out.sort_by! { |e| -e['flakiness'][0]['failure_percentage'].to_f }
+    if group_issues
+      out = out.group_by { |o| o['flakiness'] }.values
+    end
     out
   end
 

database/scripts/lib/report_formatter.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module ReportFormatter
+  JOB_URL_PATTERN = {
+    /^gz|^sdformat|^ros_gz/ => 'https://build.osrfoundation.org/job/',
+    /^[A-Z]ci/ => 'https://build.ros2.org/job/',
+    /^nightly_|^packaging_/ => 'https://ci.ros2.org/job/',
+  }
+
+  def self.format_reference_build(issue_hash)
+    job_name = issue_hash['job_name']
+    build_number = issue_hash['build_number']
+    base_url = ""
+
+    JOB_URL_PATTERN.each_pair do |pattern, url|
+      if pattern.match? job_name
+        base_url = url
+        break
+      end
+    end
+
+    "[#{job_name}##{build_number}](#{base_url}#{job_name}/#{build_number})"
+  end
+
+  def self.format_datetime(datetime)
+    date, time = datetime.split
+    hour, minute, _ = time.split(':')
+    "#{date} #{hour}:#{minute}"
+  end
+
+  def self.format_flakiness(flakiness_arr)
+    table_head = "<thead><tr><th>Job Name</th><th>Last Fail</th><th>First Fail</th><th>Build Count</th><th>Failure Count</th><th>Failure Percentage</th></tr></thead>"
+    table_body = ""
+    flakiness_arr.each do |e|
+      table_body += "<tr><td>#{e['job_name']}</td><td>#{e['last_fail']}</td><td>#{e['first_fail']}</td><td>#{e['build_count']}</td><td>#{e['failure_count']}</td><td>#{e['failure_percentage']}%</td></tr>"
+    end
+    table_body = "<tbody>#{table_body}</tbody>"
+    table = "<table>#{table_head}#{table_body}</table>"
+    table
+  end
+
+  def self.build_regressions(br_array)
+    return "" if br_array.empty?
+    table = "| Reference Build | Failure DateTime | Failure Reason |\n| -- | -- | -- |\n"
+    br_array.each do |br_hash|
+      reference_build = format_reference_build(br_hash)
+      table += "| #{reference_build} | #{format_datetime(br_hash['build_datetime'])} | #{br_hash['failure_reason']} |\n"
+    end
+    table
+  end
+
+  def self.test_regressions_consecutive(tr_array)
+    return "" if tr_array.empty?
+    table = "| Reference build | Age | Failure DateTime | Errors | Reports |\n| -- | -- | -- | -- | -- |\n"
+    warnings_table = table
+    tr_array.each do |tr_issue|
+      reference_build = format_reference_build(tr_issue[0])
+      age = tr_issue.first['age'].to_i
+      failure_datetime = tr_issue.first['build_datetime']
+      errors = ""
+      reports = []
+      tr_issue.each do |e|
+        errors += "<li>#{e['error_name']}</li>"
+        reports += e['reports']
+      end
+      errors = "<ul>#{errors}</ul>"
+
+      if reports.size > 0
+        reports_str = reports.uniq.map { |e| "<li>`#{e['github_issue']}` (#{e['status'].capitalize})</li>"}.join
+        reports_str = "<ul>#{reports_str}</ul>"
+      else
+        reports_str = "No reports found!"
+      end
+
+      # If output is too long, wrap it in a <details>
+      errors = "<details><summary>#{tr_issue.size} errors</summary>#{errors}</details>" if tr_issue.size >= 10
+
+      if age == -1
+        warnings_table += "| #{reference_build} | #{age} | #{failure_datetime} | #{errors} | #{reports_str} |\n"
+      else
+        table += "| #{reference_build} | #{age} | #{failure_datetime} | #{errors} | #{reports_str} |\n"
+      end
+    end
+    out = "### Test regressions\n#{table}\n"
+    out += "### Warnings\n#{warnings_table}\n" if warnings_table.count("\n") > 2
+    out
+  end
+
+  def self.test_regressions_flaky(tr_array)
+    return "" if tr_array.empty?
+    table = "| Reference builds | Errors | Flaky report | Reports |\n| -- | -- | -- | -- |\n"
+    warnings_table = table
+    tr_array.each do |tr|
+      jobs = []
+      errors = []
+      reports = []
+      tr.each do |e|
+        jobs << format_reference_build(e)
+        errors << e['error_name']
+        reports += e['reports']
+      end
+      jobs = jobs.uniq.map { |e| "<li>#{e}</li>" }
+      errors.map! { |e| "<li>#{e}</li>" }
+
+      jobs_str = "<ul>#{jobs.join}</ul>"
+      jobs_str = "<details><summary>#{jobs.size} items</summary>\n#{jobs_str}</details>" if jobs.size >= 10
+
+      errors_str = "<ul>#{errors.join}</ul>"
+      errors_str = "<details><summary>#{errors.size} items</summary>\n#{errors_str}</details>" if errors.size >= 10
+
+      if reports.size > 0
+        reports_str = reports.uniq.map { |e| "<li>`#{e['github_issue']}` (#{e['status'].capitalize})</li>"}.join
+        reports_str = "<ul>#{reports_str}</ul>"
+      else
+        reports_str = "No reports found!"
+      end
+
+      if tr.first['age'].to_i == -1
+        warnings_table += "|#{jobs_str}|#{errors_str}|<details>#{format_flakiness(tr.first['flakiness'])}</details>|#{reports_str}|\n"
+      else
+        table += "|#{jobs_str}|#{errors_str}|<details>#{format_flakiness(tr.first['flakiness'])}</details>|#{reports_str}|\n"
+      end
+    end
+    out = "### Test regressions\n#{table}\n"
+    out += "### Warnings\n#{warnings_table}\n" if warnings_table.count("\n") > 2
+    out
+  end
+
+
+  def self.format_report(report_hash)
+    # Use <details> and <summary> tags to prevent long reports
+    output_report = ""
+
+    report_hash.each_pair do |category, subcategory_hash|
+      output_report += "<h1>#{category.gsub('_', ' ').capitalize}</h1>\n"
+      subcategory_hash.each_pair do |subcategory, subcategory_report| # Assume that we're traversing a hash of hashes
+        next if subcategory_report.empty?
+
+        subcategory_report_title = "<h2>#{subcategory.gsub('_', ' ').capitalize}</h2>\n"
+        subcategory_report_str = "#{subcategory_report_title}\n#{subcategory_report}\n"
+        subcategory_report_str = "<details><summary>#{subcategory_report_title}</summary>\n#{subcategory_report}<details>\n" unless category == 'urgent'
+        output_report += subcategory_report_str
+      end
+    end
+    output_report
+  end
+end