Merge pull request os-autoinst#3858 from Martchus/ar

Add support for archived jobs

assets/stylesheets/test-details.scss

@@ -392,3 +392,11 @@ ul.modcategory {
 .pt-2-and-half {
     padding-top: 0.75rem !important;
 }
+
+#job-archived-badge {
+    font-size: 110%;
+    padding: 0.5rem 0.6rem;
+    margin-top: -1rem;
+    margin-bottom: -1rem;
+    margin-left: 0.25rem;
+}

docs/GettingStarted.asciidoc

@@ -322,6 +322,8 @@ be completely removed from the database.
 *NOTE* Jobs which do not belong to a job group are currently not affected by
 the mentioned cleanup properties.
 
+*NOTE* Archiving of important jobs can be enabled. Checkout the related settings
+within the `[archiving]` section of the config file for details.
 
 == Using the client script
 :openqa-personal-configuration: ~/.config/openqa/client.conf

etc/openqa/openqa.ini

@@ -171,6 +171,11 @@ blocklist = job_grab job_done
 # (still experimental, relies on df)
 #results_min_free_disk_space_percentage = 0
 
+[archiving]
+# Moves logs of jobs which are preserved during the cleanup because they are considered important
+# to "${OPENQA_ARCHIVEDIR:-${OPENQA_BASEDIR:-/var/lib}/openqa/archive}/testresults"
+#archive_preserved_important_jobs = 0
+
 [job_settings_ui]
 # Specify the keys of job settings which reference a file and should therefore be rendered
 # as links to those files within the job settings tab.

lib/OpenQA/Schema/Result/JobGroups.pm

@@ -189,9 +189,7 @@ sub matches_nested {
 }
 
 # check the group comments for important builds
-sub important_builds {
-    my ($self) = @_;
-
+sub _important_builds ($self) {
     # determine relevant comments including those on the parent-level
     # note: Assigning to scalar first because ->comments would return all results at once when
     #       called in an array-context.
@@ -216,39 +214,54 @@ sub important_builds {
             }
         }
     }
-    return [sort keys %importants];
+    return \%importants;
 }
 
-sub _find_expired_jobs ($self, $important_builds, $keep_in_days, $keep_important_in_days) {
+sub important_builds ($self) { [sort keys %{$self->_important_builds}] }
+
+sub _find_expired_jobs ($self, $important_builds, $keep_in_days, $keep_important_in_days,
+    $preserved_important_jobs_out = undef)
+{
     return undef unless $keep_in_days;    # 0 means forever
 
-    # all jobs not in important builds that are expired
-    my $timecond = {'<' => time2str('%Y-%m-%d %H:%M:%S', time - ONE_DAY * $keep_in_days, 'UTC')};
+    my $now      = time;
+    my $timecond = {'<' => time2str('%Y-%m-%d %H:%M:%S', $now - ONE_DAY * $keep_in_days, 'UTC')};
 
-    # filter out linked jobs. As we use this function also for the homeless
-    # group (with id=null), we can't use $self->jobs, but need to add it directly
-    my $schema       = $self->result_source->schema;
-    my $expired_jobs = $schema->resultset('Jobs')->search(
+    # filter out linked jobs
+    # note: As we use this function also for the homeless group (with id=null), we can't use $self->jobs, but
+    #       need to add it directly.
+    my $jobs         = $self->result_source->schema->resultset('Jobs');
+    my @group_cond   = ('me.group_id' => $self->id);
+    my $expired_jobs = $jobs->search(
         {
-            BUILD         => {-not_in => $important_builds},
-            'me.group_id' => $self->id,
-            t_finished    => $timecond,
-            text          => {like => 'label:linked%'}
+            BUILD      => {-not_in => $important_builds},
+            text       => {like    => 'label:linked%'},
+            t_finished => $timecond,
+            @group_cond,
         },
         {order_by => 'me.id', join => 'comments'});
     my @linked_jobs = map { $_->id } $expired_jobs->all;
-    my @ors;
-    push(@ors, {BUILD => {-not_in => $important_builds}, t_finished => $timecond, id => {-not_in => \@linked_jobs}});
-
-    if ($keep_important_in_days) {
-        # expired jobs in important builds
-        my $timecond = {'<' => time2str('%Y-%m-%d %H:%M:%S', time - ONE_DAY * $keep_important_in_days, 'UTC')};
-        push(@ors,
-            {-or => [{BUILD => {-in => $important_builds}}, {id => {-in => \@linked_jobs}}], t_finished => $timecond},
-        );
+
+    # define condition for expired jobs in unimportant builds
+    my @ors = ({BUILD => {-not_in => $important_builds}, t_finished => $timecond, id => {-not_in => \@linked_jobs}});
+
+    # define condition for expired jobs in important builds
+    my ($important_timestamp, @important_cond);
+    if ($keep_important_in_days && $keep_important_in_days > $keep_in_days) {
+        $important_timestamp = time2str('%Y-%m-%d %H:%M:%S', $now - ONE_DAY * $keep_important_in_days, 'UTC');
+        @important_cond      = (-or => [{BUILD => {-in => $important_builds}}, {id => {-in => \@linked_jobs}}]);
+        push @ors, {@important_cond, t_finished => {'<' => $important_timestamp}};
     }
-    return $schema->resultset('Jobs')
-      ->search({-and => {'me.group_id' => $self->id, -or => \@ors}}, {order_by => qw(id)});
+
+    # make additional query for jobs not being expired because they're important
+    if ($important_timestamp && $preserved_important_jobs_out) {
+        my @time_cond   = (-and => [{t_finished => $timecond}, {t_finished => {'>=' => $important_timestamp}}]);
+        my @search_args = ({@important_cond, @group_cond, @time_cond}, {order_by => qw(id)});
+        $$preserved_important_jobs_out = $jobs->search(@search_args);
+    }
+
+    # make query for expired jobs
+    return $jobs->search({-and => {@group_cond, -or => \@ors}}, {order_by => qw(id)});
 }
 
 sub find_jobs_with_expired_results ($self, $important_builds = undef) {
@@ -258,23 +271,25 @@ sub find_jobs_with_expired_results ($self, $important_builds = undef) {
     return $expired ? [$expired->all] : [];
 }
 
-sub find_jobs_with_expired_logs ($self, $important_builds = undef) {
+sub find_jobs_with_expired_logs ($self, $important_builds = undef, $preserved_important_jobs_out = undef) {
     $important_builds //= $self->important_builds;
     my $expired
-      = $self->_find_expired_jobs($important_builds, $self->keep_logs_in_days, $self->keep_important_logs_in_days);
+      = $self->_find_expired_jobs($important_builds, $self->keep_logs_in_days, $self->keep_important_logs_in_days,
+        $preserved_important_jobs_out);
     return $expired ? [$expired->search({logs_present => 1})->all] : [];
 }
 
 # helper function for cleanup task
-sub limit_results_and_logs {
-    my ($self) = @_;
-    my $important_builds = $self->important_builds;
-    for my $job (@{$self->find_jobs_with_expired_results($important_builds)}) {
-        $job->delete;
-    }
-    for my $job (@{$self->find_jobs_with_expired_logs($important_builds)}) {
-        $job->delete_logs;
-    }
+sub limit_results_and_logs ($self, $preserved_important_jobs_out = undef) {
+    my $important_builds_hash = $self->_important_builds;
+    my @important_builds      = keys %$important_builds_hash;
+    my $expired_jobs          = $self->find_jobs_with_expired_results(\@important_builds);
+    $_->delete for @$expired_jobs;
+
+    my $config    = OpenQA::App->singleton->config;
+    my $preserved = $config->{archiving}->{archive_preserved_important_jobs} ? $preserved_important_jobs_out : undef;
+    my $jobs_with_expired_logs = $self->find_jobs_with_expired_logs(\@important_builds, $preserved);
+    $_->delete_logs for @$jobs_with_expired_logs;
 }
 
 sub tags {

lib/OpenQA/Schema/Result/Jobs.pm

@@ -37,6 +37,7 @@ use OpenQA::Jobs::Constants;
 use OpenQA::JobDependencies::Constants;
 use OpenQA::ScreenshotDeletion;
 use File::Basename qw(basename dirname);
+use File::Copy::Recursive qw();
 use File::Spec::Functions 'catfile';
 use File::Path ();
 use DBIx::Class::Timestamps 'now';
@@ -264,6 +265,29 @@ sub delete {
     return $ret;
 }
 
+sub archivable_result_dir ($self) {
+    return undef
+      if $self->archived || OpenQA::Jobs::Constants::meta_state($self->state) ne OpenQA::Jobs::Constants::FINAL;
+    my $result_dir = $self->result_dir;
+    return $result_dir && -d $result_dir ? $result_dir : undef;
+}
+
+sub archive ($self) {
+    return undef unless my $normal_result_dir = $self->archivable_result_dir;
+
+    my $archived_result_dir = $self->add_result_dir_prefix($self->remove_result_dir_prefix($normal_result_dir), 1);
+    if (!File::Copy::Recursive::dircopy($normal_result_dir, $archived_result_dir)) {
+        my $error = $!;
+        File::Path::rmtree($archived_result_dir);    # avoid leftovers
+        die "Unable to copy '$normal_result_dir' to '$archived_result_dir': $error";
+    }
+
+    $self->update({archived => 1});
+    $self->discard_changes;
+    File::Path::remove_tree($normal_result_dir);
+    return $archived_result_dir;
+}
+
 sub name {
     my ($self) = @_;
     return $self->{_name} if $self->{_name};
@@ -434,14 +458,12 @@ sub settings_hash {
     return $settings;
 }
 
-sub add_result_dir_prefix {
-    my ($self, $rd) = @_;
-    return $rd ? catfile($self->num_prefix_dir, $rd) : undef;
+sub add_result_dir_prefix ($self, $result_dir, $archived = undef) {
+    return $result_dir ? catfile($self->num_prefix_dir($archived), $result_dir) : undef;
 }
 
-sub remove_result_dir_prefix {
-    my ($self, $rd) = @_;
-    return $rd ? basename($rd) : undef;
+sub remove_result_dir_prefix ($self, $result_dir) {
+    return $result_dir ? basename($result_dir) : undef;
 }
 
 sub set_prio {
@@ -1240,10 +1262,9 @@ END_SQL
     return [map { $_->[0] } @{$sth->fetchall_arrayref // []}];
 }
 
-sub num_prefix_dir {
-    my ($self)    = @_;
+sub num_prefix_dir ($self, $archived = undef) {
     my $numprefix = sprintf "%05d", $self->id / 1000;
-    return catfile(resultdir(), $numprefix);
+    return catfile(resultdir($archived // $self->archived), $numprefix);
 }
 
 sub create_result_dir {

lib/OpenQA/Setup.pm

@@ -141,6 +141,9 @@ sub read_config {
             screenshot_cleanup_batches_per_minion_job => OpenQA::Task::Job::Limit::DEFAULT_BATCHES_PER_MINION_JOB,
             results_min_free_disk_space_percentage    => undef,
         },
+        archiving => {
+            archive_preserved_important_jobs => 0,
+        },
         job_settings_ui => {
             keys_to_render_as_links => '',
             default_data_dir        => 'data',

lib/OpenQA/Shared/Plugin/Gru.pm

@@ -44,6 +44,7 @@ sub register_tasks {
         qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit),
         qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete),
         qw(OpenQA::Task::Job::Limit),
+        qw(OpenQA::Task::Job::ArchiveResults),
         qw(OpenQA::Task::Job::FinalizeResults),
         qw(OpenQA::Task::Iso::Schedule),
         qw(OpenQA::Task::Bug::Limit),

lib/OpenQA/Task/Job/ArchiveResults.pm

@@ -0,0 +1,45 @@
+# Copyright (C) 2021 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::Task::Job::ArchiveResults;
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
+
+use Time::Seconds;
+
+sub register ($self, $app, @args) {
+    $app->minion->add_task(archive_job_results => \&_archive_results);
+}
+
+sub _archive_results ($minion_job, @args) {
+    my ($openqa_job_id) = @args;
+    my $app = $minion_job->app;
+    return $minion_job->fail('No job ID specified.') unless defined $openqa_job_id;
+
+    # avoid archiving during result cleanup, avoid running too many cleanup/archiving jobs in parallel
+    return $minion_job->retry({delay => ONE_MINUTE})
+      unless my $process_job_results_guard = $app->minion->guard('process_job_results_task', ONE_DAY, {limit => 5});
+    return $minion_job->retry({delay => ONE_MINUTE})
+      if $app->minion->is_locked('limit_results_and_logs_task');
+
+    # avoid running any kind of result post processing task for a particular openQA job in parallel
+    return $minion_job->retry({delay => 30})
+      unless my $guard = $app->minion->guard("process_job_results_for_$openqa_job_id", ONE_DAY);
+
+    my $openqa_job = $app->schema->resultset('Jobs')->find($openqa_job_id);
+    return $minion_job->finish("Job $openqa_job_id does not exist.") unless $openqa_job;
+    $minion_job->note(archived_path => $openqa_job->archive);
+}
+
+1;

lib/OpenQA/Task/Job/FinalizeResults.pm

@@ -28,8 +28,8 @@ sub _finalize_results {
 
     my $app = $minion_job->app;
     return $minion_job->fail('No job ID specified.') unless defined $openqa_job_id;
-    return $minion_job->finish("A finalize_job_results job for $openqa_job_id is already active")
-      unless my $guard = $app->minion->guard("finalize_job_results_for_$openqa_job_id", ONE_DAY);
+    return $minion_job->delay({delay => 30})
+      unless my $guard = $app->minion->guard("process_job_results_for_$openqa_job_id", ONE_DAY);
 
     # try to finalize each
     my $openqa_job = $app->schema->resultset('Jobs')->find($openqa_job_id);

lib/OpenQA/Task/Job/Limit.pm

@@ -39,8 +39,10 @@ sub register {
 sub _limit {
     my ($job, $args) = @_;
 
-    # prevent multiple limit_results_and_logs tasks and limit_screenshots_task to run in parallel
+    # prevent multiple limit_results_and_logs tasks and limit_screenshots_task/archive_job_results to run in parallel
     my $app = $job->app;
+    return $job->retry({delay => ONE_MINUTE})
+      unless my $process_job_results_guard = $app->minion->guard('process_job_results_task', ONE_DAY);
     return $job->finish('Previous limit_results_and_logs job is still active')
       unless my $limit_results_and_logs_guard = $app->minion->guard('limit_results_and_logs_task', ONE_DAY);
     return $job->finish('Previous limit_screenshots_task job is still active')
@@ -62,9 +64,19 @@ sub _limit {
     my $schema = $app->schema;
     $schema->resultset('JobGroups')->new({})->limit_results_and_logs;
 
-    my $groups = $schema->resultset('JobGroups');
+    my $groups  = $schema->resultset('JobGroups');
+    my $gru     = $app->gru;
+    my %options = (priority => 0, ttl => 2 * ONE_DAY);
     while (my $group = $groups->next) {
-        $group->limit_results_and_logs;
+        my $preserved_important_jobs;
+        $group->limit_results_and_logs(\$preserved_important_jobs);
+
+        # archive openQA jobs where logs were preserved because they are important
+        if ($preserved_important_jobs) {
+            for my $job ($preserved_important_jobs->all) {
+                $gru->enqueue(archive_job_results => [$job->id], \%options) if $job->archivable_result_dir;
+            }
+        }
     }
 
     # prevent enqueuing new limit_screenshot if there are still inactive/delayed ones
@@ -83,8 +95,6 @@ sub _limit {
     my $batches_per_minion_job = $args->{batches_per_minion_job}
       // $config->{screenshot_cleanup_batches_per_minion_job};
     my $screenshots_per_minion_job = $batches_per_minion_job * $screenshots_per_batch;
-    my $gru                        = $app->gru;
-    my %options                    = (priority => 0, ttl => 172800);
     my @screenshot_cleanup_info;
     my @parent_minion_job_ids = ($job->id);
     for (my $i = $min_id; $i < $max_id; $i += $screenshots_per_minion_job) {

lib/OpenQA/Utils.pm

@@ -17,6 +17,7 @@ package OpenQA::Utils;
 use strict;
 use warnings;
 
+use Mojo::Base -signatures;
 use Carp;
 use Cwd 'abs_path';
 use IPC::Run();
@@ -92,6 +93,7 @@ our @EXPORT  = qw(
 our @EXPORT_OK = qw(
   prjdir
   sharedir
+  archivedir
   resultdir
   assetdir
   imagesdir
@@ -120,7 +122,9 @@ sub prjdir { ($ENV{OPENQA_BASEDIR} || '/var/lib') . '/openqa' }
 
 sub sharedir { $ENV{OPENQA_SHAREDIR} || (prjdir() . '/share') }
 
-sub resultdir { prjdir() . '/testresults' }
+sub archivedir { $ENV{OPENQA_ARCHIVEDIR} || (prjdir() . '/archive') }
+
+sub resultdir ($archived = 0) { ($archived ? archivedir() : prjdir()) . '/testresults' }
 
 sub assetdir { sharedir() . '/factory' }