Move, unify upload_y2logs and add retry

save_y2logs is failing occasionally due to 'file changed as we read it'

- move upload_y2logs from virt_autotest::utils to utils, it's generic function
- use script_retry instead assert_script_run or script_run
- use tar.xz compression by default, today it's available on all versions
- file and compression can be defined via file argument
- use script_retry in collect_guest_installation_logs_via_ssh

lib/guest_installation_and_configuration_base.pm

@@ -3135,7 +3135,7 @@ sub collect_guest_installation_logs_via_ssh {
     $self->get_guest_ipaddr;
     if ((script_run("nmap $self->{guest_ipaddr} -PN -p ssh | grep -i open") eq 0) and ($self->{guest_ipaddr} ne '') and ($self->{guest_ipaddr} ne 'NO_IP_ADDRESS_FOUND_AT_THE_MOMENT')) {
         record_info("Guest $self->{guest_name} has ssh port open on ip address $self->{guest_ipaddr}.", "Try to collect logs via ssh but may fail.Open ssh port does not mean good ssh connection.");
-        script_run($guest_installation_and_configuration_metadata::host_params{ssh_command} . "\@$self->{guest_ipaddr} \"save_y2logs /tmp/$self->{guest_name}_y2logs.tar.gz\"");
+        script_retry($guest_installation_and_configuration_metadata::host_params{ssh_command} . "\@$self->{guest_ipaddr} \"save_y2logs /tmp/$self->{guest_name}_y2logs.tar.gz\"", timeout => 180, retry => 3);
         script_run("scp -r -vvv -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root\@$self->{guest_ipaddr}:/tmp/$self->{guest_name}_y2logs.tar.gz $self->{guest_log_folder}");
     }
     else {

lib/hacluster.pm

@@ -631,8 +631,7 @@ sub ha_export_logs {
     upload_logs('/tmp/crm.txt');
 
     # Extract YaST logs and upload them
-    script_run 'save_y2logs /tmp/y2logs.tar.bz2', 120;
-    upload_logs('/tmp/y2logs.tar.bz2', failok => 1);
+    upload_y2logs(failok => 1);
 
     # Generate the packages list
     script_run "rpm -qa > $packages_list";

lib/qam.pm

@@ -12,7 +12,7 @@ use base "Exporter";
 use Exporter;
 
 use testapi;
-use utils qw(zypper_call handle_screen zypper_repos);
+use utils qw(zypper_call handle_screen zypper_repos upload_y2logs);
 use JSON;
 use List::Util qw(max);
 use version_utils qw(is_sle is_transactional);
@@ -27,10 +27,7 @@ use constant ZYPPER_STATUS_COL => 5;
 sub capture_state {
     my ($state, $y2logs) = @_;
     if ($y2logs) {    #save y2logs if needed
-        my $compression = is_sle('=12-sp1') ? 'bz2' : 'xz';
-        assert_script_run "save_y2logs /tmp/y2logs_$state.tar.$compression";
-        upload_logs "/tmp/y2logs_$state.tar.$compression";
-        save_screenshot();
+        upload_y2logs(file => "/tmp/y2logs_$state.tar.xz");
     }
     #upload ip status
     script_run("ip a | tee /tmp/ip_a_$state.log");

lib/sles4sap.pm

@@ -1590,8 +1590,7 @@ sub post_fail_hook {
     select_console('root-console');
 
     # YaST logs
-    script_run "save_y2logs /tmp/y2logs.tar.xz";
-    upload_logs "/tmp/y2logs.tar.xz";
+    upload_y2logs;
 
     # HANA installation logs, if needed
     $self->upload_hana_install_log if get_var('HANA');

lib/utils.pm

@@ -119,6 +119,7 @@ our @EXPORT = qw(
   is_usb_boot
   remove_efiboot_entry
   empty_usb_disks
+  upload_y2logs
 );
 
 our @EXPORT_OK = qw(
@@ -3034,4 +3035,23 @@ sub empty_usb_disks {
     }
 }
 
+=head2 upload_y2logs
+
+  upload_y2logs(file => '/tmp/y2logs123.tar.bz2', failok => 1);
+
+No arguments are required, y2logs can be created and uploaded with custom C<file>
+name, upload_logs can fail and continue with failok C<failok> set to 1
+
+=cut
+
+sub upload_y2logs {
+    my (%args) = @_;
+    $args{file} //= '/tmp/y2logs.tar.xz';
+    $args{failok} //= 0;
+    # Create and Upload y2log for analysis
+    script_retry("save_y2logs $args{file}", timeout => 180, retry => 3);
+    upload_logs($args{file}, failok => $args{failok});
+    save_screenshot;
+}
+
 1;

lib/virt_autotest/utils.pm

@@ -28,7 +28,7 @@ use Carp;
 
 our @EXPORT = qw(is_vmware_virtualization is_hyperv_virtualization is_fv_guest is_pv_guest guest_is_sle is_guest_ballooned is_xen_host is_kvm_host reset_log_cursor check_failures_in_journal check_host_health check_guest_health
   is_monolithic_libvirtd turn_on_libvirt_debugging_log
-  print_cmd_output_to_file ssh_setup ssh_copy_id create_guest import_guest install_default_packages upload_y2logs ensure_default_net_is_active ensure_guest_started
+  print_cmd_output_to_file ssh_setup ssh_copy_id create_guest import_guest install_default_packages ensure_default_net_is_active ensure_guest_started
   ensure_online add_guest_to_hosts restart_libvirtd check_libvirtd remove_additional_disks remove_additional_nic collect_virt_system_logs shutdown_guests wait_guest_online start_guests restore_downloaded_guests save_original_guest_xmls restore_original_guests save_guests_xml_for_change restore_xml_changed_guests
   is_guest_online wait_guests_shutdown remove_vm setup_common_ssh_config add_alias_in_ssh_config parse_subnet_address_ipv4 backup_file manage_system_service setup_rsyslog_host
   check_port_state is_registered_system do_system_registration check_system_registration subscribe_extensions_and_modules download_script download_script_and_execute is_sev_es_guest upload_virt_logs recreate_guests download_vm_import_disks enable_nm_debug check_activate_network_interface upload_nm_debug_log restart_modular_libvirt_daemons check_modular_libvirt_daemons get_guest_regcode);
@@ -606,13 +606,6 @@ sub ensure_online {
     }
 }
 
-sub upload_y2logs {
-    # Create and Upload y2log for analysis
-    assert_script_run "save_y2logs /tmp/y2logs.tar.bz2", 180;
-    upload_logs("/tmp/y2logs.tar.bz2");
-    save_screenshot;
-}
-
 sub ensure_default_net_is_active {
     if (script_run("virsh net-list --all | grep default | grep ' active'", 90) != 0) {
         # Note: TBD for modular libvirt. See poo#129086 for detail.

lib/y2_base.pm

@@ -13,6 +13,7 @@ use warnings;
 
 use ipmi_backend_utils;
 use network_utils;
+use utils qw(upload_y2logs);
 use y2_logs_helper 'get_available_compression';
 
 use testapi qw(is_serial_terminal :DEFAULT);
@@ -122,8 +123,7 @@ sub save_upload_y2logs {
     if (can_upload_logs() || (!$args{no_ntwrk_recovery} && recover_network())) {
         script_run 'sed -i \'s/^tar \(.*$\)/tar --warning=no-file-changed -\1 || true/\' /usr/sbin/save_y2logs';
         my $filename = "/tmp/y2logs$args{suffix}.tar" . get_available_compression();
-        script_run "save_y2logs $filename", 180;
-        upload_logs($filename, failok => 1);
+        utils::upload_y2logs(file => $filename, failok => 1);
     } else {    # Redirect logs content to serial
         script_run("journalctl -b --no-pager -o short-precise > /dev/$serialdev");
         script_run("dmesg > /dev/$serialdev");

lib/y2snapper_common.pm

@@ -253,8 +253,7 @@ sub y2snapper_failure_analysis {
     export_logs;
 
     # Upload y2log for analysis if yast2 snapper fails
-    assert_script_run "save_y2logs /tmp/y2logs.tar.bz2";
-    upload_logs "/tmp/y2logs.tar.bz2";
+    upload_y2logs;
     save_screenshot;
     diag('check if at least snapper low-level commands still work');
     script_run('snapper ls');

tests/virt_autotest/reboot_and_wait_up_upgrade.pm

@@ -14,6 +14,7 @@ use virt_utils 'clean_up_red_disks';
 use base 'reboot_and_wait_up';
 use virt_autotest::utils;
 use opensusebasetest;
+use utils qw(upload_y2logs);
 
 #Explanation for parameters introduced to facilitate offline host upgrade:
 #OFFLINE_UPGRADE indicates whether host upgrade is offline which needs reboot
@@ -60,7 +61,7 @@ sub post_fail_hook {
     if (get_var('VIRT_PRJ2_HOST_UPGRADE')) {
         if (get_var('OFFLINE_UPGRADE')) {
             #host offline upgrade
-            $self->upload_y2logs;
+            utils::upload_y2logs;
             upload_logs("/mnt/root/autoupg.xml", failok => 1);
         }
         else {