Callback support for data transfer from child to parent process during parallel execution (#441)

Signed-off-by: Henry Cox <[email protected]>

Author: henry2cox

README

@@ -1,6 +1,6 @@
 -------------------------------------------------
 - README file for the LTP GCOV extension (LCOV) -
-- Last changes: 2024-12-25
+- Last changes: 2025-10-01
 -------------------------------------------------
 
 Description
@@ -405,12 +405,15 @@ LCOV features and capabilities fall into 7 major categories:
 
      The user can supply callbacks which are used to:
 
-        i) interface with the revision control system
+        i) interface with the revision control system - to determine source
+           code differences between versions and to find the author
+           and date of the most recent edits.
            Sample scripts:
              - Perforce:  see 'p4diff', 'p4annotate.pm', 'p4annotate'
              - Git: see 'gitdiff', 'gitblame.pm', 'gitblame'
-        ii) verify that source code versions are compatible, and
-            Sample scripts: see 'get_signature', 'P4version.pm', 'getp4version',
+        ii) verify that source code versions are compatible - e.g.,
+            before aggregating coverage data or displaying source code.
+            Sample scripts: 'get_signature', 'P4version.pm', 'getp4version',
             'gitversion', 'gitversion.pm', and 'batchGitVersion.pm'
         iii) enforce a desired code coverage criteria
              Sample script: criteria.pm/criteria
@@ -423,11 +426,15 @@ LCOV features and capabilities fall into 7 major categories:
            Sample script:  select.pm
         vi) keep track of environment and other settings - to aid
             infrastructure debugging in more complicated use cases.
+            Sample script:  context.pm
         vii) compress the 'function detail' table to improve readability
              by shortening long C++ template and function names.
+             Sample script:  simplify.pm
 
      The callback may be any desired script or executable - but there
      may be performance advantages if it is written as a Perl module.
+     See the "Additional considerations" and "Callbacks and parallel
+     execution" discussion in the genhtml man page.
 
      See the genhtml/lcov/geninfo man pages for details.
 

lib/lcovutil.pm

@@ -62,6 +62,7 @@ our @EXPORT_OK = qw($tool_name $tool_dir $lcov_version $lcov_url $VERSION
      @exclude_file_patterns @include_file_patterns %excluded_files
      @omit_line_patterns @exclude_function_patterns $case_insensitive
      munge_file_patterns warn_file_patterns transform_pattern
+     warn_pattern_list
      parse_cov_filters summarize_cov_filters
      disable_cov_filters reenable_cov_filters is_filter_enabled
      filterStringsAndComments simplifyCode balancedParens
@@ -219,6 +220,15 @@ our $opt_no_external;
 our @build_directory;
 
 our @configured_callbacks;
+# list of callbacks which support save/restore
+our @callback_save_restore;
+# list of callbacks which support 'finalize'
+our @callback_finalize;
+# list of callbacks which implement 'start' - which gets called when
+#  child process starts
+our @callback_start_list;
+# the callback data which is saved from child process/restored from child process
+our @callback_state;
 
 # optional callback to keep track of whatever user decides is important
 our @contextCallback;
@@ -988,6 +998,26 @@ sub configure_callback
             #$package->import(qw(new));
             # the first value in @_ is the script name
             $$cb = $class->new(@args);
+            if (exists($ENV{LCOV_FORCE_PARALLEL}) ||
+                (defined($lcovutil::maxParallelism) &&
+                    1 != $lcovutil::maxParallelism)
+            ) {
+                # don't set up for parallel processing if we aren't going to fork
+                if ($$cb->can('save')) {
+                    if ($$cb->can('restore')) {
+                        push(@callback_save_restore, [$class, $$cb]);
+                        push(@callback_start_list, [$class, $$cb])
+                            if ($$cb->can('start'));
+                    } else {
+                        lcovutil::ignorable_error($lcovutil::ERROR_PACKAGE,
+                                 "$class implements 'save' but not 'restore'.");
+                        return;
+                    }
+                }
+            }
+            # implement 'finalize', regardless of parallel/not parallel
+            push(@callback_finalize, [$class, $$cb])
+                if ($$cb->can('finalize'));
         };
         if ($@ ||
             !defined($$cb)) {
@@ -1745,23 +1775,40 @@ sub munge_file_patterns
     @suppress_function_patterns = map({ $_->[0] } @exclude_function_patterns);
 }
 
+sub warn_pattern_list
+{
+    my ($type, $patterns) = @_;
+    foreach my $pat (@$patterns) {
+        my $count = $pat->[-1];
+        if (0 == $count) {
+            my $str = $pat->[-2];
+            lcovutil::ignorable_error($ERROR_UNUSED,
+                                      "'$type' pattern '$str' is unused.");
+        }
+    }
+}
+
 sub warn_file_patterns
 {
+    # a bit of a hack...we need a place to call the 'finalize' methods
+    #  (if any are registered) - and this method is called very late in
+    #  the game, by lcov/genhtml/geninfo - so is a workable location
+    for (my $i = 0; $i <= $#lcovutil::callback_finalize; ++$i) {
+        my ($class, $cb) = @{$lcovutil::callback_finalize[$i]};
+        eval { $cb->finalize(); };
+        if ($@) {
+            lcovutil::ignorable_error($lcovutil::ERROR_CALLBACK,
+                                      "\"$class->finalize()\" failed: $@");
+        }
+    }
+
     foreach my $p (['include', \@include_file_patterns],
                    ['exclude', \@exclude_file_patterns],
                    ['substitute', \@file_subst_patterns],
                    ['omit-lines', \@omit_line_patterns],
                    ['exclude-functions', \@exclude_function_patterns],
     ) {
-        my ($type, $patterns) = @$p;
-        foreach my $pat (@$patterns) {
-            my $count = $pat->[scalar(@$pat) - 1];
-            if (0 == $count) {
-                my $str = $pat->[scalar(@$pat) - 2];
-                lcovutil::ignorable_error($ERROR_UNUSED,
-                                          "'$type' pattern '$str' is unused.");
-            }
-        }
+        warn_pattern_list(@$p);
     }
 }
 
@@ -2081,14 +2128,24 @@ sub initial_state
         }
     }
 
+    for (my $i = 0; $i <= $#lcovutil::callback_start_list; ++$i) {
+        my ($class, $cb) = @{$lcovutil::callback_start_list[$i]};
+        eval { $cb->start(); };
+        if ($@) {
+            lcovutil::ignorable_error($lcovutil::ERROR_CALLBACK,
+                                      "\"$class->start()\" failed: $@");
+        }
+    }
+
     return Storable::dclone([\@message_count, \%versionCache, \%resolveCache]);
 }
 
 sub compute_update
 {
     my $state = shift;
-    my @new_count;
     my ($initialCount, $initialVersionCache, $initialResolveCache) = @$state;
+
+    my @new_count;
     my $id = 0;
     foreach my $count (@message_count) {
         my $v = $count - $initialCount->[$id++];
@@ -2104,7 +2161,20 @@ sub compute_update
         $resolveUpdate{$f} = $v
             unless exists($initialResolveCache->{$f});
     }
-    my @rtn = (\@new_count,
+    my @cbData;
+    for (my $i = 0; $i <= $#lcovutil::callback_save_restore; ++$i) {
+        my ($class, $cb) = @{$lcovutil::callback_save_restore[$i]};
+        eval {
+            my $data = $cb->save();
+            push(@cbData, $data);
+        };
+        if ($@) {
+            lcovutil::ignorable_error($lcovutil::ERROR_CALLBACK,
+                                      "\"$class->save(...)\" failed: $@");
+        }
+    }
+    my @rtn = (\@cbData,
+               \@new_count,
                \%versionUpdate,
                \%resolveUpdate,
                \%message_types,
@@ -2131,6 +2201,15 @@ sub compute_update
 
 sub update_state
 {
+    my $callbackData = shift;
+    for (my $i = 0; $i <= $#$callbackData; ++$i) {
+        my ($class, $cb) = @{$lcovutil::callback_save_restore[$i]};
+        eval { $cb->restore($callbackData->[$i]); };
+        if ($@) {
+            lcovutil::ignorable_error($lcovutil::ERROR_CALLBACK,
+                                      "\"$class->restore(...)\" failed: $@");
+        }
+    }
     my $updateCount = shift;
     my $id          = 0;
     foreach my $count (@$updateCount) {

man/genhtml.1

@@ -631,7 +631,7 @@ and does not modify information in the coveage DB.
 .IP 2. 3
 The option may be specified as a single
 .I split_char
-separated string which is divied into words (see
+separated string which is divided into words (see
 .B man lcovrc(5)
 ), or as a list of arguments.
 The resulting command line is passed
@@ -778,6 +778,75 @@ The callback will occur in the child process (possibly simultaneously with other
 As a result:  if your callback needs to pass data back to the parent, you will need to arrange a communication mechanism to do so.
 .br
 
+.SS Callbacks and parallel execution
+
+Because callbacks may need to record data -
+.I e.g.,
+for error reporting or action summaries in the presence of parallel execution -
+.B genhtml (and
+.B lcov and
+.B geninfo
+) can call certain optional callback methods:
+
+.IP \- 3
+.I $callback->start()
+.br
+is called when the child process begins execution.  This method can be
+used to capture initial state -
+.I e.g.,
+to set the count of events in this child to zero.
+This method is optional.
+.PP
+
+.IP \- 3
+.I my $data = $callback->save()
+.br
+is called when processing is complete, just before the child process exits.
+The scalar
+.I $data
+returned by your
+.I $callback->save()
+method
+.I $callback->restore()
+method when the child process is reaped.
+.PP
+
+.IP \- 3
+.I $callback->restore($data)
+.br
+is called in the parent process when the child is reaped.
+.I $data
+is the data that was returned when your
+.I $callback->save()
+method was called in the child.  (Serialization/deserialization has happened under the covers.)
+.PP
+
+.IP \- 3
+.I $callback->finalize()
+.br
+is called in the parent process when all calculalations are complete
+and the parent setting up to report final results.
+This method is optional.
+.br
+Note that, unlike the other callback methods described in this section,
+.I finalize()
+is called in both parallel and serial execution contexts.
+.PP
+
+Note that your callback must implement
+.I $callback->restore()
+if it implements
+.I $callback->save().
+.I $callback->start()
+and
+.I $callback->finalize()
+are optional:  if they are implemented, then they will be called.
+
+These methods are available only for callbacks implemented a perl modules.
+If you callback is implemented as an executable script (say) - then you
+are free to implement parent/child data passing however you prefer.
+
+
 .SS Additional considerations
 
 If the

scripts/simplify.pm

@@ -92,23 +92,63 @@ EOF
 
     # verify that the patterns are valid...
     lcovutil::verify_regexp_patterns($script, \@patterns);
+    my @munged = map({ [$_, 0]; } @patterns);
 
-    return bless \@patterns, $class;
+    return bless \@munged, $class;
 }
 
 sub simplify
 {
     my ($self, $name) = @_;
 
     foreach my $p (@$self) {
+        my $orig = $name;
         # sadly, no support for pre-compiled patterns
-        eval "\$name =~ $p ;";    # apply pattern that user provided...
+        eval "\$name =~ $p->[0] ;";    # apply pattern that user provided...
             # $@ should never match:  we already checked pattern validity during
             #   initialization - above.  Still: belt and braces.
         die("invalid 'simplify' regexp '$p->[0]': $@")
             if ($@);
+        ++$p->[1]
+            if ($name ne $orig);
     }
     return $name;
 }
 
+sub start
+{
+    my $self = shift;
+    foreach my $p (@$self) {
+        $p->[1] = 0;
+    }
+}
+
+sub save
+{
+    my $self = shift;
+    my @data;
+    foreach my $p (@$self) {
+        push(@data, $p->[1]);
+    }
+    return \@data;
+}
+
+sub restore
+{
+    my ($self, $data) = @_;
+    die("unexpected restore: (" .
+        join(' ', @$self) . ") <- [" .
+        join(' ', @$data) . "]\n")
+        unless $#$self == $#$data;
+    for (my $i = 0; $i <= $#$self; ++$i) {
+        $self->[$i]->[-1] += $data->[$i];
+    }
+}
+
+sub finalize
+{
+    my $self = shift;
+    lcovutil::warn_pattern_list("simplify", $self);
+}
+
 1;

tests/Makefile

@@ -27,6 +27,7 @@ report:
                --version-script $(VERSION_SCRIPT)                   \
 	       --exclude genError.pm --exclude filter.pl            \
                --exclude brokenCallback.pm --exclude MsgContext.pm  \
+	       --exclude missingRestore.pm --exclude parallelFail.pm \
                --omit-lines 'ERROR_INTERNAL' --omit-lines '\bdie\b' \
 	       --ignore unsupported,unused,inconsistent             \
                --filter region ;                                    \