Add and use MANIFEST.SKIP file for Porting files

This change is merging the two ideas from #19523 and #19513
by relying on a traditional MANIFEST.SKIP file.

Case #19523 simplifies the way we can easily exclude common noise
Case #19513 provides a mechanism to avoid shipping not necessary
files as part of the tarball

This is adding a new 'Porting/Manifest.pm' file which provides helpers
to list files from MANIFEST taking into account the 'MANIFEST.SKIP'
using ExtUtils::Manifest.

Author: atoomic

.gitignore

@@ -139,6 +139,8 @@ lib/unicore/mktables.lst
 # generated by WinCE build
 xlib/
 
+/stuff
+
 # test byproducts
 t/rantests
 t/tmp*
@@ -150,6 +152,8 @@ t/test_state
 t/*.ph
 t/lib/*.ph
 
+/rel2abs2rel*.pl
+
 # t/op/require.t byproducts
 t/bleah.pm
 t/bleah.do

MANIFEST

@@ -5103,7 +5103,6 @@ Makefile.SH			A script that generates Makefile
 malloc.c			A version of malloc you might not want
 malloc_ctl.h			A version of malloc you might not want
 MANIFEST			This list of files
-MANIFEST.SKIP			Patterns for files not included in this file
 mathoms.c			A home for binary-compatible code artifacts
 META.json			Distribution meta-data in JSON
 META.yml			Distribution meta-data in YAML

MANIFEST.SKIP

@@ -1,3 +1,5 @@
+^MANIFEST\.SKIP
+
 \.mailmap
 \.gitignore
 \.gitattributes
@@ -7,12 +9,13 @@
 \.travis\.yml
 
 # Porting tools
-Porting/bisect.pl
 Porting/bisect-example.sh
 Porting/bisect-runner.pl
+Porting/bisect.pl
 Porting/checkAUTHORS.pl
 Porting/git-deltatool
 Porting/git-find-p4-change
 Porting/git-make-p4-refs
 Porting/GitUtils.pm
+Porting/Manifest.pm
 Porting/updateAUTHORS.pl

Porting/Maintainers.pm

@@ -28,35 +28,20 @@ require Exporter;
 use File::Find;
 use Getopt::Long;
 
+use Manifest; # aka 'use Porting::Manifest' - note: Porting is in lib
+
 my %MANIFEST;
 
 # (re)read the MANIFEST file, blowing away any previous effort
 
 sub reload_manifest {
-    %MANIFEST = ();
-
-    foreach my $manifest_path ('MANIFEST','Porting/MANIFEST.dev') {
-        if (! -e  $manifest_path) {
-            $manifest_path = "../MANIFEST";
-        }
-
-        if (open(my $manfh,  '<', $manifest_path )) {
-            while (<$manfh>) {
-                if (/^(\S+)/) {
-                    $MANIFEST{$1}++;
-                }
-                else {
-                    warn "MANIFEST:$.: malformed line: $_\n";
-                }
-            }
-            close $manfh;
-        } else {
-            die "$0: Failed to open MANIFEST for reading: $!\n";
-        }
-    }
+
+    %MANIFEST = ( map { $_ => 1 } Porting::Manifest::get_files_from_all_manifests( 1 ) );
+
+    return;
 }
 
-reload_manifest;
+reload_manifest();
 
 
 sub get_module_pat {

Porting/Manifest.pm

@@ -0,0 +1,79 @@
+#!perl
+
+package Porting::Manifest;
+
+use strict;
+use warnings;
+
+use v5.34;
+
+##
+## list of all files from MANIFEST and ignored by MANIFEST.SKIP
+##
+
+sub get_files_from_all_manifests {
+
+    my ($reload) = @_;
+
+    state @list;
+
+    @list = () if $reload;
+
+    return @list if scalar @list;
+
+    require ExtUtils::Manifest;
+    require Cwd;
+
+    local $ExtUtils::Manifest::Quiet = $ExtUtils::Manifest::Quiet = 1;    # no warnings 'once'
+
+    my @from_manifest;
+    my @from_manifest_skip;
+
+    my $skip;
+
+    my $cwd = Cwd::getcwd();
+    my @ls_files;
+    my $ls_status;
+    {
+        my $root_pwd = Cwd::abs_path( $INC{"strict.pm"} );
+
+        #my $strict_path = $INC{"strict.pm"};
+        $root_pwd =~ s{/*\Qlib/strict.pm\E$}{};
+        chdir($root_pwd);
+
+        # read the manifest files
+        @from_manifest = keys %{ ExtUtils::Manifest::maniread("MANIFEST") };
+        $skip          = ExtUtils::Manifest::maniskip("MANIFEST.SKIP");
+
+        @ls_files  = `git ls-files --full-name`;
+        $ls_status = $?
+    }
+    chdir($cwd);
+    die q[Fail to run git ls-files] if $ls_status;
+
+    chomp(@ls_files);
+
+    foreach my $f (@ls_files) {
+        next unless $skip->($f);
+        push @from_manifest_skip, $f;
+    }
+
+    my %uniq = map { $_ => 1 } @from_manifest, @from_manifest_skip;
+
+    return ( @list = sort keys %uniq );
+}
+
+##
+## list of Porting files listed in MANIFEST or ignored by MANIFEST.SKIP
+##
+
+sub get_porting_files {
+
+    return grep { $_ =~ qr{^Porting} && $_ !~ qr{\.(?:gitignore$|github|gitattributes|mailmap|travis)} } get_files_from_all_manifests();
+}
+
+sub get_porting_perl_files {
+    return grep { $_ !~ qr{\.sh} } get_porting_files();
+}
+
+1;

Porting/README.pod

@@ -106,7 +106,7 @@ file is normally updated each time F<Configure> is updated.
 
 =head2 F<core-cpan-diff>
 
-Compare CPAN modules with their equivalent in core. 
+Compare CPAN modules with their equivalent in core.
 Originally based on App::DualLivedDiff by Steffen Mueller.
 
 =head2 F<corecpan.pl>
@@ -188,15 +188,15 @@ digits which acts the internal unique identifier for this commit
 =head2 F<Glossary>
 
 This file is built by F<metaconfig>. This file contains a description of all
-the shell variables whose value is determined by the Configure script. 
+the shell variables whose value is determined by the Configure script.
 It later gets incorporated into the pod for F<Config.pm>.
 
 =head2 F<harness-timer-report.pl>
 
 For analyzing the output of "env HARNESS_TIMER=1 make test", to find
 outliers of test execution times.
 
-=head2 F<how_to_write_a_perldelta.pod> 
+=head2 F<how_to_write_a_perldelta.pod>
 
 This file contains a specification as to how to write a perldelta pod.
 Related file: F<perldelta_template.pod>
@@ -244,25 +244,17 @@ This script creates a release checklist as a simple HTML document.
 =head2 F<make_snapshot.pl>
 
 This script is a quick and dirty snapshot generator for the perl5.git.perl.org
-web page to use to generate the snapshot files. 
+web page to use to generate the snapshot files.
 
 =head2 F<manicheck>
 
-This script outputs a list of files in F<MANIFEST> and F<MANIFEST.dev>
-which don't exist and a list of files that exist and aren't in
-F<MANIFEST> or in F<MANIFEST.dev>
-
-=head2 F<MANIFEST.dev>
-
-This file lists all files that are not included in the F<MANIFEST>
-file in the root of the repo. The union of the files it contains with
-the files in C<MANIFEST> should be the same the list of file produced
-by C<git ls-files> from a git checkout.
+This script outputs a list of files in F<MANIFEST> which don't exist
+and a list of files that exist and aren't skipped by F<MANIFEST.SKIP>.
 
 =head2 F<manifest_lib.pl>
 
 This library provides functions used in checking and sorting the
-F<MANIFEST> and F<MANIFEST.dev> files. The sort order is similar to
+F<MANIFEST> files. The sort order is similar to
 dictionary sort order (alphabetical case insensitive) but where path
 components and extensions are sorted independently such that the
 following files would be sorted into the following order:
@@ -276,9 +268,18 @@ following files would be sorted into the following order:
 
 Currently this code does not support EBCDIC. Patches welcome.
 
+=head2 F<Manifest.pm>
+
+A package which provides helper to list files from F<MANIFEST>
+taking into account F<MANIFEST.SKIP>.
+
+=head2 F<MANIFEST.SKIP>
+
+List files that we do not want to ship and are versionned in git.
+
 =head2 F<manisort>
 
-This script sorts the files in F<MANIFEST> or F<MANIFEST.dev>. It uses
+This script sorts the files in F<MANIFEST>. It uses
 F<manifest_list.pl> to do the sorting. Use C<Porting/manisort --man>
 to see instructions for use.
 
@@ -337,7 +338,7 @@ Applies F<podtidy> to a file.
 =head2 F<pumpkin.pod>
 
 Pumpkin - Notes on handling the Perl Patch Pumpkin And Porting Perl.
-Many of these are out of date or superseded by other documents in 
+Many of these are out of date or superseded by other documents in
 this directory.
 
 =head2 F<README.pod>

Porting/checkcfguse.pl

@@ -11,6 +11,9 @@
 use strict;
 use warnings;
 
+use lib '.';
+use Porting::Manifest;
+
 my %SYM;
 
 my @PAT =
@@ -70,32 +73,29 @@
 my $SYM = join("|", sort { length($b) <=> length($a) || $a cmp $b } keys %SYM);
 
 my %found;
-foreach my $manifest_file ('MANIFEST', 'Porting/MANIFEST.dev') {
-  open(my $mani, '<', $manifest_file)
-    or die "$0: Failed to open '$manifest_file'\n";
 
-  while (<$mani>) {
-    if (/^(\S+)\s+/) {
-      my $fn = $1;
-      # Skip matches from the config files themselves,
-      # from metaconfig generated files that refer to
-      # the config symbols, and from pods.
-      next if $fn =~ m{^(?:config_h\.SH
-                          |Configure
-                          |configure\.com
-                          |Porting/(?:config|Glossary)
-                          |(?:plan9|win32)/(?:config|(?:GNU)?[Mm]akefile)
-                          |uconfig)
-                          |\.pod$}x;
-      open my $fh, '<', $fn or die qq[$0: Failed to open $fn: $!];
-      while (<$fh>) {
-        while (/\b($SYM)\b/go) {
-          $found{$1}{$fn}++;
-        }
-      }
+my @manifest_files = Porting::Manifest::get_files_from_all_manifests();
+
+for (@manifest_files) {
+  my $fn = $1;
+  # Skip matches from the config files themselves,
+  # from metaconfig generated files that refer to
+  # the config symbols, and from pods.
+  next if $fn =~ m{^(?:config_h\.SH
+                      |Configure
+                      |configure\.com
+                      |Porting/(?:config|Glossary)
+                      |(?:plan9|win32)/(?:config|(?:GNU)?[Mm]akefile)
+                      |uconfig)
+                      |\.pod$}x;
+  open my $fh, '<', $fn or die qq[$0: Failed to open $fn: $!];
+  while (<$fh>) {
+    while (/\b($SYM)\b/go) {
+      $found{$1}{$fn}++;
     }
   }
 }
+
 for my $sym (sort keys %SYM) {
   if (exists $found{$sym}) {
     my @found = keys %{$found{$sym}};

Porting/manicheck

@@ -11,7 +11,7 @@ use v5.14;
 use warnings;
 use File::Find;
 use Getopt::Long;
-use ExtUtils::Manifest 'maniskip';
+use ExtUtils::Manifest qw{ maniread maniskip };
 use constant MAX_EXIT_CODE => 124;
 
 my $exitstatus;
@@ -22,51 +22,41 @@ my $bonus = 0;
 my $skip = maniskip;
 
 sub read_manifest {
-    my ($manifest_file) = @_;
-    open my $fh, '<', $manifest_file
-        or die "Can't read '$manifest_file': $!\n";
+    my @from_manifest = sort keys %{ maniread("MANIFEST") };
+
     my %files;
     my $missing = 0;
-    for my $line (<$fh>) {
-        my ($file) = $line =~ /^(\S+)/;
-        if (!defined($file)) {
-            warn "No file specified in '$manifest_file' line $.\n";
-            next;
-        }
-        $files{$file} = $.;
+    foreach my $file (@from_manifest) {
+        $files{$file} = 1;
         next if -f $file;
         $missing++;
-        print "'$file' from '$manifest_file' at line $. doesn't exist\n";
+        print "'$file' from 'MANIFEST' doesn't exist\n";
     }
-    close $fh or die "Failed to close '$manifest_file': $!\n";
+
     return (\%files, $missing);
 }
 
 my ($manifest_files, $manifest_missing) = read_manifest("MANIFEST");
-my ($porting_files, $porting_missing)   = read_manifest("Porting/MANIFEST.dev");
+
+my @ls_files = `git ls-files --full-name`;
+my %git_files = map { $_ => 1 } @ls_files;
 
 find {
     no_chdir => 1,
     wanted => sub {
         return if -d;
         my $path = $File::Find::name =~ s!^\./!!r;
         return if $skip->($path);
-        return if $files{$path};
-
-        # return if -d $_;
-        # my $file = $_;
-        # $file =~s!^\./!!;
-        # return if $file=~m!^\.git/!;
-        # return if $porting_files->{$file};
-        # return if $manifest_files->{$file};
+        return if $manifest_files->{$path};
+        return unless $git_files{$path};
 
         ++$bonus;
-        print "$path\t\tnot in MANIFEST\n";
+        print "'$path'\t\tnot in MANIFEST or not skipped by MANIFEST.SKIP\n";
 
     },
 }, ".";
 
-my $problems = $manifest_missing + $porting_missing + $bonus;
+my $problems = $manifest_missing + $bonus;
 
 # We can't (meaningfully) exit with codes above 255, so we're going to have to
 # clamp them to some range whatever we do. So as we need the code anyway, use