diff --git a/database/scripts/format_report.rb b/database/scripts/format_report.rb new file mode 100755 index 0000000..8f40648 --- /dev/null +++ b/database/scripts/format_report.rb @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby + +require_relative 'lib/report_formatter' +require 'json' + +report_name = ARGV.shift + +report = JSON.load_file(report_name) + +report['urgent']['build_regressions'] = ReportFormatter::build_regressions(report['urgent']['build_regressions']) +report['urgent']['test_regressions_consecutive'] = ReportFormatter::test_regressions_consecutive(report['urgent']['test_regressions_consecutive']) +report['urgent']['test_regressions_flaky'] = ReportFormatter::test_regressions_flaky(report['urgent']['test_regressions_flaky']) + +# Sample output: +# puts report['urgent']['build_regressions'] +puts ReportFormatter::format_report report diff --git a/database/scripts/generate_report.rb b/database/scripts/generate_report.rb index d9757f0..2329726 100755 --- a/database/scripts/generate_report.rb +++ b/database/scripts/generate_report.rb @@ -28,8 +28,8 @@ def generate_report(report_name, exclude_set) report = { 'urgent' => { 'build_regressions' => urgent_build_regressions = BuildfarmToolsLib::build_regressions_today(filter_known: true), - 'test_regressions_consecutive' => urgent_consistent_test_regressions = BuildfarmToolsLib::test_regressions_today(filter_known: true, only_consistent: true), - 'test_regressions_flaky' => urgent_flaky_test_regressions = BuildfarmToolsLib::flaky_test_regressions(filter_known: true), + 'test_regressions_consecutive' => urgent_consistent_test_regressions = BuildfarmToolsLib::test_regressions_today(filter_known: true, only_consistent: true, group_issues: true), + 'test_regressions_flaky' => urgent_flaky_test_regressions = BuildfarmToolsLib::flaky_test_regressions(filter_known: true, group_issues: true), }, 'maintenance' => { 'jobs_failing' => [], @@ -40,7 +40,6 @@ def generate_report(report_name, exclude_set) 'build_regressions_known' => [], 'test_regressions_all' => [], 'test_regressions_known' => [], - 'build_regressions_known' => [], } } @@ -51,3 +50,4 @@ def generate_report(report_name, exclude_set) end generate_report(options[:report_name], options[:exclude]) +puts options[:report_name] diff --git a/database/scripts/lib/buildfarm_tools.rb b/database/scripts/lib/buildfarm_tools.rb index 703f1aa..0083057 100644 --- a/database/scripts/lib/buildfarm_tools.rb +++ b/database/scripts/lib/buildfarm_tools.rb @@ -34,7 +34,7 @@ def self.error_appearances_in_job(test_name, job_name) run_command('./sql_run.sh error_appearances_in_job.sql', args: [test_name, job_name]) end - def self.test_regressions_today(filter_known: false, only_consistent: false) + def self.test_regressions_today(filter_known: false, only_consistent: false, group_issues: false) # Keys: job_name, build_number, error_name, build_datetime, node_name out = run_command('./sql_run.sh errors_check_last_build.sql') if filter_known @@ -45,15 +45,22 @@ def self.test_regressions_today(filter_known: false, only_consistent: false) out.filter! { |tr| tr['age'].to_i >= CONSECUTIVE_THRESHOLD || tr['age'].to_i == WARNING_AGE_CONSTANT } out.sort_by! { |tr| -tr['age'].to_i } end + out.each do |e| + e['reports'] = test_regression_reported_issues e['error_name'] + end + if group_issues + # Group by (job_name, age) + out = out.group_by { |o| [o['job_name'], o['age']] }.to_a.map { |e| e[1] } + end out end - def self.flaky_test_regressions(filter_known: false, time_range: FLAKY_BUILDS_DEFAULT_RANGE) + def self.flaky_test_regressions(filter_known: false, group_issues: false, time_range: FLAKY_BUILDS_DEFAULT_RANGE) # Keys: job_name, build_number, error_name, build_datetime, node_name, flakiness out = [] today_regressions = test_regressions_today(filter_known: filter_known) today_regressions.each do |tr| - next if !tr['age'].to_i.nil? && tr['age'].to_i >= CONSECUTIVE_THRESHOLD + next if !tr['age'].to_i.nil? && (tr['age'].to_i >= CONSECUTIVE_THRESHOLD || tr['age'].to_i == WARNING_AGE_CONSTANT) tr_flakiness = test_regression_flakiness(tr['error_name'], time_range: time_range) if tr_flakiness.nil? @@ -66,6 +73,9 @@ def self.flaky_test_regressions(filter_known: false, time_range: FLAKY_BUILDS_DE end end out.sort_by! { |e| -e['flakiness'][0]['failure_percentage'].to_f } + if group_issues + out = out.group_by { |o| o['flakiness'] }.values + end out end diff --git a/database/scripts/lib/report_formatter.rb b/database/scripts/lib/report_formatter.rb new file mode 100644 index 0000000..5a5ec1f --- /dev/null +++ b/database/scripts/lib/report_formatter.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'json' + +module ReportFormatter + JOB_URL_PATTERN = { + /^gz|^sdformat|^ros_gz/ => 'https://build.osrfoundation.org/job/', + /^[A-Z]ci/ => 'https://build.ros2.org/job/', + /^nightly_|^packaging_/ => 'https://ci.ros2.org/job/', + } + + def self.format_reference_build(issue_hash) + job_name = issue_hash['job_name'] + build_number = issue_hash['build_number'] + base_url = "" + + JOB_URL_PATTERN.each_pair do |pattern, url| + if pattern.match? job_name + base_url = url + break + end + end + + "[#{job_name}##{build_number}](#{base_url}#{job_name}/#{build_number})" + end + + def self.format_datetime(datetime) + date, time = datetime.split + hour, minute, _ = time.split(':') + "#{date} #{hour}:#{minute}" + end + + def self.format_flakiness(flakiness_arr) + table_head = "Job NameLast FailFirst FailBuild CountFailure CountFailure Percentage" + table_body = "" + flakiness_arr.each do |e| + table_body += "#{e['job_name']}#{e['last_fail']}#{e['first_fail']}#{e['build_count']}#{e['failure_count']}#{e['failure_percentage']}%" + end + table_body = "#{table_body}" + table = "#{table_head}#{table_body}
" + table + end + + def self.build_regressions(br_array) + return "" if br_array.empty? + table = "| Reference Build | Failure DateTime | Failure Reason |\n| -- | -- | -- |\n" + br_array.each do |br_hash| + reference_build = format_reference_build(br_hash) + table += "| #{reference_build} | #{format_datetime(br_hash['build_datetime'])} | #{br_hash['failure_reason']} |\n" + end + table + end + + def self.test_regressions_consecutive(tr_array) + return "" if tr_array.empty? + table = "| Reference build | Age | Failure DateTime | Errors | Reports |\n| -- | -- | -- | -- | -- |\n" + warnings_table = table + tr_array.each do |tr_issue| + reference_build = format_reference_build(tr_issue[0]) + age = tr_issue.first['age'].to_i + failure_datetime = tr_issue.first['build_datetime'] + errors = "" + reports = [] + tr_issue.each do |e| + errors += "
  • #{e['error_name']}
  • " + reports += e['reports'] + end + errors = "" + + if reports.size > 0 + reports_str = reports.uniq.map { |e| "
  • `#{e['github_issue']}` (#{e['status'].capitalize})
  • "}.join + reports_str = "" + else + reports_str = "No reports found!" + end + + # If output is too long, wrap it in a
    + errors = "
    #{tr_issue.size} errors#{errors}
    " if tr_issue.size >= 10 + + if age == -1 + warnings_table += "| #{reference_build} | #{age} | #{failure_datetime} | #{errors} | #{reports_str} |\n" + else + table += "| #{reference_build} | #{age} | #{failure_datetime} | #{errors} | #{reports_str} |\n" + end + end + out = "### Test regressions\n#{table}\n" + out += "### Warnings\n#{warnings_table}\n" if warnings_table.count("\n") > 2 + out + end + + def self.test_regressions_flaky(tr_array) + return "" if tr_array.empty? + table = "| Reference builds | Errors | Flaky report | Reports |\n| -- | -- | -- | -- |\n" + warnings_table = table + tr_array.each do |tr| + jobs = [] + errors = [] + reports = [] + tr.each do |e| + jobs << format_reference_build(e) + errors << e['error_name'] + reports += e['reports'] + end + jobs = jobs.uniq.map { |e| "
  • #{e}
  • " } + errors.map! { |e| "
  • #{e}
  • " } + + jobs_str = "" + jobs_str = "
    #{jobs.size} items\n#{jobs_str}
    " if jobs.size >= 10 + + errors_str = "" + errors_str = "
    #{errors.size} items\n#{errors_str}
    " if errors.size >= 10 + + if reports.size > 0 + reports_str = reports.uniq.map { |e| "
  • `#{e['github_issue']}` (#{e['status'].capitalize})
  • "}.join + reports_str = "" + else + reports_str = "No reports found!" + end + + if tr.first['age'].to_i == -1 + warnings_table += "|#{jobs_str}|#{errors_str}|
    #{format_flakiness(tr.first['flakiness'])}
    |#{reports_str}|\n" + else + table += "|#{jobs_str}|#{errors_str}|
    #{format_flakiness(tr.first['flakiness'])}
    |#{reports_str}|\n" + end + end + out = "### Test regressions\n#{table}\n" + out += "### Warnings\n#{warnings_table}\n" if warnings_table.count("\n") > 2 + out + end + + + def self.format_report(report_hash) + # Use
    and tags to prevent long reports + output_report = "" + + report_hash.each_pair do |category, subcategory_hash| + output_report += "

    #{category.gsub('_', ' ').capitalize}

    \n" + subcategory_hash.each_pair do |subcategory, subcategory_report| # Assume that we're traversing a hash of hashes + next if subcategory_report.empty? + + subcategory_report_title = "

    #{subcategory.gsub('_', ' ').capitalize}

    \n" + subcategory_report_str = "#{subcategory_report_title}\n#{subcategory_report}\n" + subcategory_report_str = "
    #{subcategory_report_title}\n#{subcategory_report}
    \n" unless category == 'urgent' + output_report += subcategory_report_str + end + end + output_report + end +end