diff --git a/.gitignore b/.gitignore index 309de23325ea..7369ec63a027 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,8 @@ lib/unicore/mktables.lst # generated by WinCE build xlib/ +/stuff + # test byproducts t/rantests t/tmp* @@ -150,6 +152,8 @@ t/test_state t/*.ph t/lib/*.ph +/rel2abs2rel*.pl + # t/op/require.t byproducts t/bleah.pm t/bleah.do diff --git a/MANIFEST b/MANIFEST index 90c3b0f920fb..269042c236f0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,7 +2,6 @@ .editorconfig EditorConfig style file .lgtm.yml LGTM.com configuration file .metaconf-exclusions.txt Symbols that should ignored when generating Configure -.travis.yml continuous integration on github (where enabled) amigaos4/amigaio.c AmigaOS4 port amigaos4/amigaio.h AmigaOS4 port amigaos4/amigaos.c AmigaOS4 port @@ -5311,8 +5310,8 @@ pod/perl58delta.pod Perl changes in version 5.8.0 pod/perlapio.pod Perl internal IO abstraction interface pod/perlartistic.pod Perl Artistic License pod/perlbook.pod Perl book information -pod/perlboot.pod -pod/perlbot.pod +pod/perlboot.pod Links to info on OO programming in Perl +pod/perlbot.pod Links to info on OO programming in Perl pod/perlcall.pod Perl calling conventions from C pod/perlcheat.pod Perl cheat sheet pod/perlclib.pod Internal replacements for standard C library functions @@ -5376,7 +5375,7 @@ pod/perlrecharclass.pod Perl regular expression character classes pod/perlref.pod Perl references, the rest of the story pod/perlreftut.pod Perl references short introduction pod/perlreguts.pod Perl regular expression engine internals -pod/perlrepository.pod +pod/perlrepository.pod Links to current info on the Perl source repository pod/perlrequick.pod Perl regular expressions quick start pod/perlreref.pod Perl regular expressions quick reference pod/perlretut.pod Perl regular expressions tutorial @@ -5389,9 +5388,9 @@ pod/perlsub.pod Perl subroutines pod/perlsyn.pod Perl syntax pod/perlthrtut.pod Perl threads tutorial pod/perltie.pod Perl objects hidden behind simple variables -pod/perltodo.pod -pod/perltooc.pod -pod/perltoot.pod +pod/perltodo.pod Link to the Perl to-do list +pod/perltooc.pod Links to info on OO programming in Perl +pod/perltoot.pod Links to info on OO programming in Perl pod/perltrap.pod Perl traps for the unwary pod/perlunicode.pod Perl Unicode support pod/perlunicook.pod Perl Unicode cookbook @@ -5409,13 +5408,9 @@ Porting/acknowledgements.pl Generate perldelta acknowledgements text Porting/add-package.pl Add/Update CPAN modules that are part of Core Porting/add-pod-file Utility to add new pod/*.pod file to core distribution Porting/bench.pl Run benchmarks against t/perf/benchmarks -Porting/bisect.pl A tool to make bisecting easy -Porting/bisect-example.sh Example script to use with git bisect run -Porting/bisect-runner.pl Tool to be called by git bisect run Porting/bump-perl-version bump the perl version in relevant files Porting/check-cpan-pollution Check for commits that may wrongly touch CPAN distros Porting/checkansi.pl Check source code for ANSI-C violations -Porting/checkAUTHORS.pl Check that the AUTHORS file is complete Porting/checkcfguse.pl Check that config symbols are being used Porting/checkcfgvar.pl Check that config scripts define all symbols Porting/checkpodencoding.pl Check POD encoding @@ -5438,10 +5433,6 @@ Porting/exec-bit.txt List of files that get +x in release tarball Porting/exercise_makedef.pl Brute force testing for makedef.pl Porting/expand-macro.pl A tool to expand C macro definitions in the Perl source Porting/findrfuncs Find reentrant variants of functions used in an executable -Porting/git-deltatool Mark commits for perldelta in git notes -Porting/git-find-p4-change Find the change for a p4 change number -Porting/git-make-p4-refs Output git refs for each p4 change number, suitable for appending to .git/packed-refs -Porting/GitUtils.pm Generate the contents of a .patch file Porting/Glossary Glossary of config.sh variables Porting/harness-timer-report.pl Analyze the timings from the test harness Porting/how_to_write_a_perldelta.pod Bluffer's guide to writing a perldelta. @@ -5479,7 +5470,6 @@ Porting/sync-with-cpan Sync with CPAN Porting/timecheck.c Test program for the 2038 fix Porting/timecheck2.c Test program for the 2038 fix Porting/todo.pod Perl things to do -Porting/updateAUTHORS.pl Tool to automatically update AUTHORS and .mailmap from git log data Porting/valgrindpp.pl Summarize valgrind reports Porting/vote_admin_guide.pod Perlgov Vote Administrator guide pp.c Push/Pop code diff --git a/MANIFEST.SKIP b/MANIFEST.SKIP new file mode 100644 index 000000000000..e8f73b6fb948 --- /dev/null +++ b/MANIFEST.SKIP @@ -0,0 +1,21 @@ +^MANIFEST\.SKIP + +\.mailmap +\.gitignore +\.gitattributes +\.git_patch +^\.git\b +^\.github/ +\.travis\.yml + +# Porting tools +Porting/bisect-example.sh +Porting/bisect-runner.pl +Porting/bisect.pl +Porting/checkAUTHORS.pl +Porting/git-deltatool +Porting/git-find-p4-change +Porting/git-make-p4-refs +Porting/GitUtils.pm +Porting/Manifest.pm +Porting/updateAUTHORS.pl \ No newline at end of file diff --git a/Porting/Maintainers.pm b/Porting/Maintainers.pm index 5ffcfb28c8fc..7f2b4d053583 100644 --- a/Porting/Maintainers.pm +++ b/Porting/Maintainers.pm @@ -21,41 +21,27 @@ our @EXPORT_OK = qw(%Modules %Maintainers show_results process_options files_to_modules finish_tap_output reload_manifest); -our $VERSION = 0.14; +our $VERSION = 0.15; require Exporter; use File::Find; use Getopt::Long; +use Manifest; # aka 'use Porting::Manifest' - note: Porting is in lib + my %MANIFEST; # (re)read the MANIFEST file, blowing away any previous effort sub reload_manifest { - %MANIFEST = (); - my $manifest_path = 'MANIFEST'; - if (! -e $manifest_path) { - $manifest_path = "../MANIFEST"; - } + %MANIFEST = ( map { $_ => 1 } Porting::Manifest::get_files_from_all_manifests( 1 ) ); - if (open(my $manfh, '<', $manifest_path )) { - while (<$manfh>) { - if (/^(\S+)/) { - $MANIFEST{$1}++; - } - else { - warn "MANIFEST:$.: malformed line: $_\n"; - } - } - close $manfh; - } else { - die "$0: Failed to open MANIFEST for reading: $!\n"; - } + return; } -reload_manifest; +reload_manifest(); sub get_module_pat { @@ -355,7 +341,8 @@ sub duplicated_maintainers { sub warn_maintainer { my $name = shift; - ok($files{$name}, "$name has a maintainer (see Porting/Maintainers.pl)"); + ok($name=~/\.gitignore\z/ or $files{$name}, + "$name has a maintainer (see Porting/Maintainers.pl)"); } sub missing_maintainers { @@ -380,4 +367,3 @@ sub finish_tap_output { } 1; - diff --git a/Porting/Manifest.pm b/Porting/Manifest.pm new file mode 100644 index 000000000000..a6136b5b3875 --- /dev/null +++ b/Porting/Manifest.pm @@ -0,0 +1,79 @@ +#!perl + +package Porting::Manifest; + +use strict; +use warnings; + +use v5.34; + +## +## list of all files from MANIFEST and ignored by MANIFEST.SKIP +## + +sub get_files_from_all_manifests { + + my ($reload) = @_; + + state @list; + + @list = () if $reload; + + return @list if scalar @list; + + require ExtUtils::Manifest; + require Cwd; + + local $ExtUtils::Manifest::Quiet = $ExtUtils::Manifest::Quiet = 1; # no warnings 'once' + + my @from_manifest; + my @from_manifest_skip; + + my $skip; + + my $cwd = Cwd::getcwd(); + my @ls_files; + my $ls_status; + { + my $root_pwd = Cwd::abs_path( $INC{"strict.pm"} ); + + #my $strict_path = $INC{"strict.pm"}; + $root_pwd =~ s{/*\Qlib/strict.pm\E$}{}; + chdir($root_pwd); + + # read the manifest files + @from_manifest = keys %{ ExtUtils::Manifest::maniread("MANIFEST") }; + $skip = ExtUtils::Manifest::maniskip("MANIFEST.SKIP"); + + @ls_files = `git ls-files --full-name`; + $ls_status = $? + } + chdir($cwd); + die q[Fail to run git ls-files] if $ls_status; + + chomp(@ls_files); + + foreach my $f (@ls_files) { + next unless $skip->($f); + push @from_manifest_skip, $f; + } + + my %uniq = map { $_ => 1 } @from_manifest, @from_manifest_skip; + + return ( @list = sort keys %uniq ); +} + +## +## list of Porting files listed in MANIFEST or ignored by MANIFEST.SKIP +## + +sub get_porting_files { + + return grep { $_ =~ qr{^Porting} && $_ !~ qr{\.(?:gitignore$|github|gitattributes|mailmap|travis)} } get_files_from_all_manifests(); +} + +sub get_porting_perl_files { + return grep { $_ !~ qr{\.sh} } get_porting_files(); +} + +1; diff --git a/Porting/README.pod b/Porting/README.pod index 0f22b5e99062..fcd0255769d2 100644 --- a/Porting/README.pod +++ b/Porting/README.pod @@ -106,7 +106,7 @@ file is normally updated each time F is updated. =head2 F -Compare CPAN modules with their equivalent in core. +Compare CPAN modules with their equivalent in core. Originally based on App::DualLivedDiff by Steffen Mueller. =head2 F @@ -188,7 +188,7 @@ digits which acts the internal unique identifier for this commit =head2 F This file is built by F. This file contains a description of all -the shell variables whose value is determined by the Configure script. +the shell variables whose value is determined by the Configure script. It later gets incorporated into the pod for F. =head2 F @@ -196,7 +196,7 @@ It later gets incorporated into the pod for F. For analyzing the output of "env HARNESS_TIMER=1 make test", to find outliers of test execution times. -=head2 F +=head2 F This file contains a specification as to how to write a perldelta pod. Related file: F @@ -244,20 +244,44 @@ This script creates a release checklist as a simple HTML document. =head2 F This script is a quick and dirty snapshot generator for the perl5.git.perl.org -web page to use to generate the snapshot files. +web page to use to generate the snapshot files. =head2 F -This script outputs a list of files in F which don't exist and a -list of files that exist and aren't in F. +This script outputs a list of files in F which don't exist +and a list of files that exist and aren't skipped by F. =head2 F -This library provides functions used in checking and sorting the F. +This library provides functions used in checking and sorting the +F files. The sort order is similar to +dictionary sort order (alphabetical case insensitive) but where path +components and extensions are sorted independently such that the +following files would be sorted into the following order: + + Configure + configure + lib/Foo/Bar + lib/Foo/Bar.pm + lib/Foo/Bar/Alpha.pm + lib/Foo-Alpha/Baz + +Currently this code does not support EBCDIC. Patches welcome. + +=head2 F + +A package which provides helper to list files from F +taking into account F. + +=head2 F + +List files that we do not want to ship and are versionned in git. =head2 F -This script sorts the files in F. +This script sorts the files in F. It uses +F to do the sorting. Use C +to see instructions for use. =head2 F @@ -294,6 +318,12 @@ distribution, provides functions useful during testing. Generate the sections of files listed in C<%Targets> from F. Mostly these are rules in Makefiles. +An example of what this tool does is to ensure that every podfile listed +in F is also listed in F with the same +description in F. If they differ in test mode it will +complain. To update the manifest entries run it from the command line +without the --test argument. + --verbose gives slightly more output --build-all tries to build everything --build-foo updates foo as follows @@ -308,7 +338,7 @@ Applies F to a file. =head2 F Pumpkin - Notes on handling the Perl Patch Pumpkin And Porting Perl. -Many of these are out of date or superseded by other documents in +Many of these are out of date or superseded by other documents in this directory. =head2 F @@ -398,4 +428,3 @@ leaks. Guide for Vote Administrators for running Steering Council elections. =cut - diff --git a/Porting/checkcfguse.pl b/Porting/checkcfguse.pl index 6dea4a84c6fe..693ca8a621d3 100755 --- a/Porting/checkcfguse.pl +++ b/Porting/checkcfguse.pl @@ -11,6 +11,9 @@ use strict; use warnings; +use lib '.'; +use Porting::Manifest; + my %SYM; my @PAT = @@ -69,21 +72,26 @@ my $SYM = join("|", sort { length($b) <=> length($a) || $a cmp $b } keys %SYM); -open(my $mani, '<', "MANIFEST") or die "$0: Failed to open MANIFEST\n"; - my %found; -while (<$mani>) { - if (/^(\S+)\s+/) { - my $fn = $1; - # Skip matches from the config files themselves, - # from metaconfig generated files that refer to - # the config symbols, and from pods. - next if $fn =~ m{^(?:config_h.SH|Configure|configure\.com|Porting/(?:config|Glossary)|(?:plan9|win32)/(?:config|(?:GNU)?[Mm]akefile)|uconfig)|\.pod$}; - open my $fh, '<', $fn or die qq[$0: Failed to open $fn: $!]; - while (<$fh>) { - while (/\b($SYM)\b/go) { - $found{$1}{$fn}++; - } + +my @manifest_files = Porting::Manifest::get_files_from_all_manifests(); + +for (@manifest_files) { + my $fn = $1; + # Skip matches from the config files themselves, + # from metaconfig generated files that refer to + # the config symbols, and from pods. + next if $fn =~ m{^(?:config_h\.SH + |Configure + |configure\.com + |Porting/(?:config|Glossary) + |(?:plan9|win32)/(?:config|(?:GNU)?[Mm]akefile) + |uconfig) + |\.pod$}x; + open my $fh, '<', $fn or die qq[$0: Failed to open $fn: $!]; + while (<$fh>) { + while (/\b($SYM)\b/go) { + $found{$1}{$fn}++; } } } diff --git a/Porting/makerel b/Porting/makerel index 3687a7e0f27e..e03e6fcdc892 100755 --- a/Porting/makerel +++ b/Porting/makerel @@ -24,7 +24,7 @@ use warnings; # # Tim Bunce, June 1997 -use ExtUtils::Manifest qw(fullcheck); +use ExtUtils::Manifest qw(fullcheck maniskip); $ExtUtils::Manifest::Quiet = 1; use Getopt::Std; use Digest::SHA; @@ -88,8 +88,8 @@ cleanup($relroot, $reldir) if $opts{c}; print "Cross-checking the MANIFEST...\n"; my ($missfile, $missentry) = fullcheck(); -@$missentry - = grep {$_ !~ m!^\.(?:git|github|mailmap)! and $_ !~ m!(?:/|^)\.gitignore!} @$missentry; +my $skip = maniskip; +@$missentry = grep !$skip->($_), @$missentry; if (@$missfile ) { warn "Can't make a release with MANIFEST files missing:\n"; warn "\t".$_."\n" for (@$missfile); diff --git a/Porting/manicheck b/Porting/manicheck index 6db1fd21204a..588bbd1e0f54 100644 --- a/Porting/manicheck +++ b/Porting/manicheck @@ -4,55 +4,68 @@ # a) files listed in MANIFEST which don't exist # b) files which exist but which aren't in MANIFEST +# this file should be refactored with t/porting/manifest.t and +# Porting/manifest_list.pm + use v5.14; use warnings; use File::Find; use Getopt::Long; -use constant SKIP => 125; +use ExtUtils::Manifest qw{ maniread maniskip }; +use constant MAX_EXIT_CODE => 124; my $exitstatus; GetOptions('exitstatus!', \$exitstatus) or die "$0 [--exitstatus]"; -my %files; -my $missing = 0; my $bonus = 0; +my $skip = maniskip; + +sub read_manifest { + my @from_manifest = sort keys %{ maniread("MANIFEST") }; -open my $fh, '<', 'MANIFEST' or die "Can't read MANIFEST: $!\n"; -for my $line (<$fh>) { - my ($file) = $line =~ /^(\S+)/; - ++$files{$file}; - next if -f $file; - ++$missing; - print "$file from MANIFEST doesn't exist\n"; + my %files; + my $missing = 0; + foreach my $file (@from_manifest) { + $files{$file} = 1; + next if -f $file; + $missing++; + print "'$file' from 'MANIFEST' doesn't exist\n"; + } + + return (\%files, $missing); } -close $fh; + +my ($manifest_files, $manifest_missing) = read_manifest("MANIFEST"); + +my @ls_files = `git ls-files --full-name`; +my %git_files = map { $_ => 1 } @ls_files; find { + no_chdir => 1, wanted => sub { return if -d; - return if $_ eq '.mailmap'; - return if $_ eq '.gitignore'; - return if $_ eq '.gitattributes'; - return if $_ eq '.git_patch'; - - my $x = $File::Find::name =~ s!^\./!!r; - return if $x =~ /^\.git\b/; - return if $x =~ m{^\.github/}; - return if $files{$x}; + my $path = $File::Find::name =~ s!^\./!!r; + return if $skip->($path); + return if $manifest_files->{$path}; + return unless $git_files{$path}; + ++$bonus; - print "$x\t\tnot in MANIFEST\n"; + print "'$path'\t\tnot in MANIFEST or not skipped by MANIFEST.SKIP\n"; + }, }, "."; -my $exitcode = $exitstatus ? $missing + $bonus : 0; +my $problems = $manifest_missing + $bonus; # We can't (meaningfully) exit with codes above 255, so we're going to have to # clamp them to some range whatever we do. So as we need the code anyway, use # 124 as our maximum instead, and then we can run as a useful git bisect run # script if needed... - -$exitcode = SKIP - 1 - if $exitcode > SKIP; +my $exitcode = (!$exitstatus) + ? 0 + : ($problems > MAX_EXIT_CODE) + ? MAX_EXIT_CODE + : $problems; exit $exitcode; diff --git a/Porting/manifest_lib.pl b/Porting/manifest_lib.pl index 95d49be9cd31..c0337ed7b8cd 100644 --- a/Porting/manifest_lib.pl +++ b/Porting/manifest_lib.pl @@ -1,5 +1,4 @@ #!/usr/bin/perl - use strict; =head1 NAME @@ -21,6 +20,9 @@ =head2 C =cut +# this file should be refactored with t/porting/manifest.t and +# Porting/manicheck + # Try to get a sane sort. case insensitive, more or less # sorted such that path components are compared independently, # and so that lib/Foo/Bar sorts before lib/Foo-Alpha/Baz diff --git a/Porting/manisort b/Porting/manisort index 167aada7345d..b89eeff7c43c 100644 --- a/Porting/manisort +++ b/Porting/manisort @@ -12,52 +12,227 @@ use strict; use warnings; $| = 1; +sub abort { + my ($exit_code, $message) = @_; + print STDERR $message, "\n"; + exit $exit_code; +} + # Get command line options -use Getopt::Long; +use Getopt::Long qw(GetOptions); +use Pod::Usage qw(pod2usage); require "./Porting/manifest_lib.pl"; +my $man = 0; +my $help = 0; my $outfile; -my $check_only = 0; -my $quiet = 0; -GetOptions ('output=s' => \$outfile, - 'check' => \$check_only, - 'quiet' => \$quiet); +my $fixfile; +my $quiet; +my $check; # legacy support - does nothing. +GetOptions( + 'check!' => \$check, + 'output=s' => \$outfile, + 'fix=s' => \$fixfile, + 'quiet!' => \$quiet, + 'help|?' => \$help, + 'man' => \$man +) or pod2usage(2); +pod2usage(42) if $help; +pod2usage(-verbose => 2, -exitval => 42) if $man; -my $file = (@ARGV) ? shift : 'MANIFEST'; +my $file = (@ARGV) ? shift : $fixfile ? $fixfile : 'MANIFEST'; + +if ($fixfile) { + $outfile //= $fixfile; + $check //= 0; +} + +if (defined $check){ + $quiet //= 1 unless $check; +} else { + $check = 1; +} + +if (!$check and !defined $outfile) { + abort(2, "Using --nocheck without providing a file makes no sense!"); +} # Read in the MANIFEST file open(my $IN, '<', $file) - or die("Can't read '$file': $!"); + or abort(3, "Can't read '$file': $!"); my @manifest = <$IN>; -close($IN) or die($!); +close($IN) + or abort(3, "Failed to close '$file': $!"); chomp(@manifest); -my %seen= ( '' => 1 ); # filter out blank lines +# note the seen filter only removes /exact/ dupes if the same file is +# listed twice with different descriptions or separators it will not +# filter them out. +my %seen = ( '' => 1 ); # filter out blank lines my @sorted = grep { !$seen{$_}++ } - sort_manifest(@manifest) -; - -# Check if the file is sorted or not -my $exit_code = 0; -for (my $ii = 0; $ii < $#manifest; $ii++) { - next if ($manifest[$ii] eq $sorted[$ii]); - $exit_code = 1; # Not sorted - last; + sort_manifest(@manifest); + +# Check if the file has any exact dupes, and if it is sorted or not. +my $was_well_formed = 1; +if (@sorted != @manifest) { + $was_well_formed = 0; +} else { + for my $idx (0 .. $#sorted) { + next if ($manifest[$idx] eq $sorted[$idx]); + $was_well_formed = 0; # Not sorted + last; + } } # Output sorted file if (defined($outfile)) { - open(my $OUT, '>', $outfile) - or die("Can't open output file '$outfile': $!"); - binmode($OUT); - print($OUT join("\n", @sorted), "\n"); - close($OUT) or die($!); + my $tmpfile= "$outfile.$$"; # avoid partially written output on error + open(my $OUT, '>', $tmpfile) + or abort(3, "Can't open output file '$tmpfile': $!"); + binmode($OUT) + or abort(3, "Can't binmode '$tmpfile': $!"); + print($OUT join("\n", @sorted), "\n") + or abort(3, "Can't print to '$tmpfile': $!"); + close($OUT) + or abort(3, "Can't print to '$tmpfile': $!"); + rename $tmpfile, $outfile + or abort(3, "Can't rename '$tmpfile' into place as '$outfile': $!"); } # Report on sort results printf(STDERR "'$file' is%s sorted properly\n", - (($exit_code) ? ' NOT' : '')) if (! $quiet); + ($was_well_formed ? '' : ' NOT')) + unless $quiet; # Exit with the sort results status -exit($exit_code); +if (!$check or $was_well_formed) { + exit 0; +} else { + exit 1; +} +__END__ + +=head1 NAME + +manisort - Sort MANIFEST files according to the Perl sort order conventions + +=head1 SYNOPSIS + +manisort [options] [MANIFEST_FILE] + + Options: + --help brief help message + --check check FILE, signal status via exit code (default) + --nocheck do not check file, status will be 0 after update + --man full documentation + --quiet do not output a message if the file is not sorted + --noquiet output a message if file is not sorted + --output=FILE specify file to write to + --fix=FILE quietly fix the specified file + + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Print a brief help message and exits with an exit code of 42. + +=item B<--man> + +Prints the manual page and exits with an exit code of 42. + +=item B<--check> + +This is the default mode of the tool, check the file for whether it is +sorted and whether it contains any exact duplicate lines, and then +return an exit code of 0 if it is fine, and an exit code of 1 if it is +not, optionally outputting a message to STDERR to reflect the status +which can be suppressed with the B<--quiet> option. + +=item B<--nocheck> + +Do not check the original state of the file, and return an exit code of +0 unless there were errors during processing. Intended to be combined +with the B<--output> option. Implies B<--quiet>. + +If this option is provided and the B<--output> option is not specified +then the tool will output an error message and return with an exit code +of 2. + +=item B<--quiet> + +Do not output any status messages during processing. + +=item B<--noquiet> + +Output a message if the specified MANIFEST_FILE is not already sorted. + +=item B<--output=FILE> + +Write the sorted results to the specified FILE. If this option is +not provided the tool checks the sort order of a file only. + +=item B<--fix=FILE> + +Read this file, sort it, and write it back. By default does so quietly +and return exit 0 unless there were errors during processing. Using this +option alone is equivalent to: + + manisort --nocheck --output=MANIFEST_FILE MANIFEST_FILE + +Can be combined with the B<--check>, B<--noquiet> or B<--output> options, +in which case it is an alternative way to specify the MANIFEST_FILE. + +=back + +=head1 DESCRIPTION + +B is used to verify the sort order and/or fix the sort order +of the F file ncluded in +the Perl project. The sort order is similar to dictionary sort order +(alphabetical case insensitive) but where path components and extensions +are sorted independently such that the following files would be sorted +into the following order: + + Configure + configure + lib/Foo/Bar + lib/Foo/Bar.pm + lib/Foo/Bar/Alpha.pm + lib/Foo-Alpha/Baz + +When provided with no options or arguments the B operates on +the F in the root directory of the project. This may be +overriden by providing a MANIFEST_FILE argument. + +By default if the file is already sorted correctly the tool will print a +message to STDERR saying so and return an exit code of 0. If the file is +B already sorted then a message will be printed to STDERR and it +will return an exit code of 1. The output message may be suppressed with +the B<--quiet> option. The B<--nocheck> option may be used to disable +the check feature, in which case the tool will return an exit code of +0 provided it did something, and 2 if it did not. + +If the B<--output=FILE> option is provided the sorted version of the +file will be written to the specified FILE. The FILE and MANIFEST_FILE +arguments may be the same, the file will be read first and then output. + +If the B<--fix=FILE> option is provided then it is equivalent to saying + + manisort --nocheck --output=FILE FILE + +This is likely what you want if you just want to ensure the file is sorted. + +In all cases if there is an error reading or writing the file a message +will be printed to STDERR and the tool will return an exit code of 3. + +=head1 SEE ALSO + +F - the code that does the sorting. + +F - test manifest files are well formed. + +F - find missing or superfluous files in manifests. -# EOF +=cut diff --git a/Porting/pod_lib.pl b/Porting/pod_lib.pl index 8f8ebf2ffe95..fb0d64d32896 100644 --- a/Porting/pod_lib.pl +++ b/Porting/pod_lib.pl @@ -439,7 +439,7 @@ sub __prime_state { $state{delta_target} = "$delta_leaf.pod"; $state{delta_version} = \@want; - # This way round so that keys can act as a MANIFEST skip list + # This way round so that keys can act as a MANIFEST.SKIP list # Targets will always be in the pod directory. Currently we can only cope # with sources being in the same directory. $state{copies}{$state{delta_target}} = $source; diff --git a/Porting/release_managers_guide.pod b/Porting/release_managers_guide.pod index f782aaedede1..007974316ee7 100644 --- a/Porting/release_managers_guide.pod +++ b/Porting/release_managers_guide.pod @@ -295,13 +295,16 @@ to the repository. =item * -For any files that are gone, remove them from C, and use +For any files that are gone, remove them from F, and use C to tell git the files will be gone. =item * -If the C file was changed in any of the previous steps, run -C. +If the F file was changed in any of the +previous steps, run C which will sort both files, or you +can manually run: + + perl Porting/manisort --fix MANIFEST =item * @@ -1613,4 +1616,3 @@ L, plus a whole bunch of other sources, including private correspondence. =cut - diff --git a/pod/perl.pod b/pod/perl.pod index 742d5560dd3b..30bd9f1b9aa8 100644 --- a/pod/perl.pod +++ b/pod/perl.pod @@ -309,12 +309,12 @@ aux h2ph h2xs perlbug pl2pm pod2html pod2man splain xsubpp =head2 Stubs for Deleted Documents - perlboot - perlbot - perlrepository - perltodo - perltooc - perltoot + perlboot Links to info on OO programming in Perl + perlbot Links to info on OO programming in Perl + perlrepository Links to current info on the Perl source repository + perltodo Link to the Perl to-do list + perltooc Links to info on OO programming in Perl + perltoot Links to info on OO programming in Perl =for buildtoc __END__ @@ -456,4 +456,3 @@ how many more is left as an exercise to the reader. The three principal virtues of a programmer are Laziness, Impatience, and Hubris. See the Camel Book for why. - diff --git a/t/TEST b/t/TEST index 72c865d4ce60..ab55602d7a22 100755 --- a/t/TEST +++ b/t/TEST @@ -402,10 +402,15 @@ sub _tests_from_manifest { } my @results; - my $mani = '../MANIFEST'; - if (open(MANI, $mani)) { - while () { - if (m!^((?:cpan|dist|ext)/(\S+)/+(?:[^/\s]+\.t|test\.pl)|lib/\S+?(?:\.t|test\.pl))\s!) { + + local @INC = ( @INC, '..' ); + require Porting::Manifest; + + for ( Porting::Manifest::get_files_from_all_manifests() ) { + next unless $_ =~ m!^( + (?:cpan|dist|ext) / (\S+) /+ (?:[^/\s]+\.t | test\.pl) + | lib / \S+? (?: \.t | test\.pl ) + )$!xms; my $t = $1; my $extension = $2; @@ -414,7 +419,7 @@ sub _tests_from_manifest { && $extension =~ m! \b (?: Archive-Tar/ | Config-Perl-V/ - | CPAN-Meta/ + | CPAN-Meta/ | CPAN-Meta-YAML/ | Digest-SHA/ | ExtUtils-MakeMaker/ @@ -455,12 +460,8 @@ sub _tests_from_manifest { push @results, $path; $::path_to_name{$path} = $t; } - } } - close MANI; - } else { - warn "$0: cannot open $mani: $!\n"; - } + return @results; } diff --git a/t/harness b/t/harness index 6da6071f1b67..987253a9b084 100644 --- a/t/harness +++ b/t/harness @@ -206,8 +206,8 @@ if (@ARGV) { my @nonexistent_serials = grep { not exists $all_dirs{$_} } keys %serials; if (@nonexistent_serials) { - die "These directories to be run serially don't exist." - . " Check your spelling:\n" . join "\n", @nonexistent_serials; + die "These directories to be run serially don't exist.\n" + . "Check your spelling:\n" . join( "\n", map { "'$_'" } @nonexistent_serials ); } # Remove the serial testing directories from the list of all diff --git a/t/porting/exec-bit.t b/t/porting/exec-bit.t index 3e80f8447977..8d241b6c1839 100644 --- a/t/porting/exec-bit.t +++ b/t/porting/exec-bit.t @@ -36,31 +36,32 @@ if ( $Config{usecrosscompile} ) { plan('no_plan'); -use ExtUtils::Manifest qw(maniread); +require '../Porting/Manifest.pm'; + +my @manifest = Porting::Manifest::get_files_from_all_manifests(); # Copied from Porting/makerel - these will get +x in the tarball # XXX refactor? -- dagolden, 2010-07-23 my %exe_list = - map { $_ => 1 } + map { $_ =~ s{^\Q../\E}{}; $_ => 1 } map { my ($f) = split; glob("../$f") } grep { $_ !~ /\A#/ && $_ !~ /\A\s*\z/ } map { split "\n" } do { local (@ARGV, $/) = '../Porting/exec-bit.txt'; <> }; -# Get MANIFEST -$ExtUtils::Manifest::Quiet = 1; -my @manifest = sort keys %{ maniread("../MANIFEST") }; # Check that +x files in repo get +x from makerel -for my $f ( map { "../$_" } @manifest ) { +for my $k ( @manifest ) { + my $f = "../$k"; + next unless -x $f; ok( has_shebang($f), "File $f has shebang" ); - ok( $exe_list{$f}, "tarball will chmod +x $f" ) + ok( $exe_list{$k}, "tarball will chmod +x $f" ) or diag( "Remove the exec bit or add '$f' to Porting/exec-bit.txt" ); - delete $exe_list{$f}; # seen it + delete $exe_list{$k}; # seen it } ok( ! %exe_list, "Everything in Porting/exec-bit.txt has +x in repo" ) diff --git a/t/porting/filenames.t b/t/porting/filenames.t index b65ab8e2c9e1..9428dd4424f4 100644 --- a/t/porting/filenames.t +++ b/t/porting/filenames.t @@ -29,19 +29,9 @@ BEGIN { use strict; use File::Basename; require './test.pl'; +require '../Porting/Manifest.pm'; - -my $manifest = '../MANIFEST'; - -open my $m, '<', $manifest or die "Can't open '$manifest': $!"; -my @files; -while (<$m>) { - chomp; - my($path) = split /\t+/; - push @files, $path; - -} -close $m or die $!; +my @files = Porting::Manifest::get_porting_files(); plan(scalar @files); @@ -49,6 +39,7 @@ PATHNAME: for my $pathname (@files) { my @path_components = split('/',$pathname); my $filename = pop @path_components; for my $component (@path_components) { + next if $component eq ".github"; if ($component =~ /\./) { fail("$pathname has directory components containing '.'"); next PATHNAME; diff --git a/t/porting/known_pod_issues.dat b/t/porting/known_pod_issues.dat index ac554f21d13c..4d1923e46f18 100644 --- a/t/porting/known_pod_issues.dat +++ b/t/porting/known_pod_issues.dat @@ -384,7 +384,7 @@ ext/pod-html/corpus/perlvar-copy.pod Verbatim line length including indents exce ext/vms-filespec/lib/vms/filespec.pm Verbatim line length including indents exceeds 78 by 1 install ? Should you be using F<...> or maybe L<...> instead of 1 install Verbatim line length including indents exceeds 78 by 2 -pod/perl.pod Verbatim line length including indents exceeds 78 by 5 +pod/perl.pod Verbatim line length including indents exceeds 78 by 6 pod/perlandroid.pod Verbatim line length including indents exceeds 78 by 3 pod/perlbook.pod Verbatim line length including indents exceeds 78 by 1 pod/perldebguts.pod Verbatim line length including indents exceeds 78 by 24 diff --git a/t/porting/manifest.t b/t/porting/manifest.t index 666c2d9204fa..5ab1207c003c 100644 --- a/t/porting/manifest.t +++ b/t/porting/manifest.t @@ -1,122 +1,175 @@ #!./perl -w -# What does this test? -# This tests the well-formed-ness of the MANIFEST file. +# * What does this test? * # -# Why do we test this? -# TK +# This tests the well-formed-ness of the MANIFEST file +# and of the MANIFEST.SKIP file, and if git is present +# it will confirm that every file known to git ls-files is +# listed in one of the two MANIFEST files. # -# It's broken - how do I fix it? -# If MANIFEST is not sorted properly, you will get this error output: -# got ''MANIFEST' is NOT sorted properly -# # ' +# * Why do we test this? * +# +# We want to keep the files sorted according to our preferred +# sort order. See t/manifest_lib.pm for details. We also want +# to ensure that the file doesn't have any issues that would +# break any parsers that process it. +# +# * It's broken - how do I fix it? * +# +# If MANIFEST is not sorted properly, +# you will get this error output something like this: +# +# got ''MANIFEST' is NOT sorted properly' # # expected /(?^:is sorted properly)/ # -# To correct this, run either: +# To correct this, run this: +# +# make manisort +# +# which will output something like "WARNING: re-sorting MANIFEST" but which +# will also correct the problem. This will sort BOTH files with one simple +# command so it is the preferred option. +# +# Alternatively you can manually run what it does under the hood: # # ./perl -Ilib Porting/manisort -o MANIFEST MANIFEST # -# which will output "'MANIFEST' is NOT sorted properly" but which will -# correct the problem; or: +# which will output something like "'MANIFEST' is NOT sorted properly" but +# which will also correct the problem. # -# make manisort # -# which will output "WARNING: re-sorting MANIFEST" but which will also -# correct the problem. +# Note the file format for MANIFEST files uses *tabs* as a separator, NOT +# spaces, and mixed tabs and spaces as a separator will trigger a test +# failure. Make sure the separator does not contain spaces. You may add a +# description for each file after the tabs, but it is not required. If you +# omit the description then you should trim trailing spaces from the line. +# The description itself may contain spaces however. +# use Config; +use ExtUtils::Manifest 'maniskip'; BEGIN { - @INC = '..' if -f '../TestInit.pm'; + if (-f '../TestInit.pm') { + @INC = '..'; + } elsif (-f './TestInit.pm') { + @INC = '.'; + } } -use TestInit qw(T); # T is chdir to the top level +use TestInit qw(T); # T is chdir to the top level require './t/test.pl'; +use strict; skip_all("Cross-compiling, the entire source might not be available") if $Config{usecrosscompile}; +find_git_or_skip('all'); plan('no_plan'); -my $manifest = 'MANIFEST'; - -open my $m, '<', $manifest or die "Can't open '$manifest': $!"; -my @files; -# Test that MANIFEST uses tabs - not spaces - after the name of the file. -while (<$m>) { - chomp; - unless( /\s/ ) { - push @files, $_; - # no need for further tests on lines without whitespace (i.e., filename only) - next; - } - my ($file, $separator) = /^(\S+)(\s+)/; - push @files, $file; - - isnt($file, undef, "Line $. doesn't start with a blank") or next; - ok(-f $file, "File $file exists"); - if ($separator !~ tr/\t//c) { - # It's all tabs - next; - } elsif ($separator !~ tr/ //c) { - # It's all spaces - fail("Spaces in entry for $file in MANIFEST at line $."); - } elsif ($separator =~ tr/\t//) { - fail("Mixed tabs and spaces in entry for $file in MANIFEST at line $."); - } else { - fail("Odd whitespace in entry for $file in MANIFEST at line $."); +# this script should be refactored with Porting/manifest_lib.pm and +# Porting/manicheck + +sub read_manifest { + my ($manifest) = @_; + + open my $m, '<', $manifest or die "Can't open '$manifest': $!"; + my @files_array; + my %files_hash; + + # Test that MANIFEST uses tabs - not spaces - after the name of the file. + while (defined(my $input_line = <$m>)) { + chomp($input_line); + my $file_and_line = "'$manifest' at line $."; + + my ($file, $sep, $descr) = $input_line =~ m/^(\S+)(\s+)?(\S+.*)?\z/; + + isnt($file, undef, "Line does not start with whitespace in $file_and_line") + or next; + + ok(-f $file, "File '$file' exists in $file_and_line"); + + if (my $prev_line = $files_hash{$file}) { + fail("File '$file' is not duplicated in $file_and_line # previous at line $prev_line"); + } + + push @files_array, $file unless $files_hash{$file}; + $files_hash{$file} = $.; + + next unless defined $sep; + + $sep =~ s/\t/\\t/g; + + ok($sep !~ /\s/, "Separator is all tabs in $file_and_line") + or diag("Separator is '$sep'"); + + ok(defined($descr), "Line with separator also has description in $file_and_line"); + } + close $m or die "Failed to close '$manifest': $!"; + return (\@files_array, \%files_hash); } -close $m or die $!; +my $manifest_file = "MANIFEST"; +my $manifest_skip_file = "MANIFEST.SKIP"; + +my ($manifest_files, $manifest_hash) = read_manifest("MANIFEST"); # Test that MANIFEST is properly sorted SKIP: { - skip("Sorting order is different under EBCDIC", 1) if $::IS_EBCDIC || $::IS_EBCDIC; - skip("'Porting/manisort' not found", 1) if (! -f 'Porting/manisort'); - - my $result = runperl('progfile' => 'Porting/manisort', - 'args' => [ '-c', $manifest ], + my @files = ($manifest_file); + skip("Sorting order is different under EBCDIC", 0+@files) + if $::IS_EBCDIC || $::IS_EBCDIC; + skip("'Porting/manisort' not found", 0+@files) + if (! -f 'Porting/manisort'); + + for my $file ( @files ) { + my $result = runperl( + 'progfile' => 'Porting/manisort', + 'args' => [ $file ], 'stderr' => 1, 'nolib' => 1 ); - - like($result, qr/is sorted properly/, 'MANIFEST sorted properly'); + chomp $result; + like($result, qr/is sorted properly/, + "'$file' sorted properly"); + } } SKIP: { - find_git_or_skip(6); - my %seen; # De-dup ls-files output (can appear more than once) - my @repo= grep { - chomp(); - !m{\.git_patch$} && - !m{\.gitattributes$} && - !m{\.gitignore$} && - !m{\.mailmap$} && - !m{^\.github/} && - -e $_ && - !$seen{$_}++ - } `git ls-files`; - skip("git ls-files didnt work",3) - if !@repo; - is( 0+@repo, 0+@files, "git ls-files gives the same number of files as MANIFEST lists"); - my %repo; - ++$repo{$_} for @repo; - my %mani; - ++$mani{$_} for @files; - is( 0+keys %mani, 0+@files, "no duplicate files in MANIFEST") - or diag(join("\n ", "Duplicates:",grep $mani{$_} > 1, keys %mani)); - delete $mani{$_} for @repo; - delete $repo{$_} for @files; - my @not_in_mani= keys %repo; - my @still_in_mani= keys %mani; - - is( 0+@not_in_mani, 0, "Nothing added to the repo that isn't in MANIFEST"); - is( "not in MANIFEST: @not_in_mani", "not in MANIFEST: ", - "Nothing added to the repo that isn't in MANIFEST"); - is( 0+@still_in_mani, 0, "Nothing in the MANIFEST that isn't tracked by git"); - is( "should not be in MANIFEST: @still_in_mani", "should not be in MANIFEST: ", - "Nothing in the MANIFEST that isn't tracked by git"); + my $skip = maniskip; + my %repo_seen; # De-dup ls-files output (can appear more than once) + my @unlisted; + foreach my $file (`git ls-files`) { + chomp($file); + next if $skip->($file); + next if !-e($file) + or $repo_seen{$file}++; + if (!$manifest_hash->{$file}) { + push @unlisted, $file; + } + } + skip("git ls-files didnt work (this shouldn't happen)",5) + unless keys %repo_seen; + is("Unlisted: @unlisted", "Unlisted: ", + "All files are listed in either '$manifest_file' or skipped by '$manifest_skip_file'"); + + foreach my $file (keys %repo_seen) { + delete $manifest_hash->{$file}; + } + my @still_in_manifest = ( sort keys %$manifest_hash ); + + is("Git does not know file: @still_in_manifest", + "Git does not know file: ", + "Git knows about all files in '$manifest_file'"); + is(0+keys(%repo_seen),scalar @$manifest_files, + "git ls-files has the same count of files as our manifests"); + + delete $repo_seen{$_} for @$manifest_files; + my @still_in_repo_seen = sort keys %repo_seen; + is("Not listed in either manifest: @still_in_repo_seen", + "Not listed in either manifest: ", + "All files from git ls-files are in one of our manifests"); } # EOF diff --git a/t/porting/podcheck.t b/t/porting/podcheck.t index 22abfba8c0d7..a1d9738cc2b3 100644 --- a/t/porting/podcheck.t +++ b/t/porting/podcheck.t @@ -37,6 +37,8 @@ BEGIN { require '../regen/regen_lib.pl'; } +require '../Porting/Manifest.pm'; + sub DEBUG { 0 }; =pod @@ -357,7 +359,9 @@ my $digest_type = "SHA-1"; my $original_dir = File::Spec->rel2abs(File::Spec->curdir); my $data_dir = File::Spec->catdir($original_dir, 'porting'); my $known_issues = File::Spec->catfile($data_dir, 'known_pod_issues.dat'); -my $MANIFEST = File::Spec->catfile(File::Spec->updir($original_dir), 'MANIFEST'); + +my @files_from_manifests = Porting::Manifest::get_files_from_all_manifests(); + my $copy_fh; my $MAX_LINE_LENGTH = 78; # 78 columns @@ -475,21 +479,18 @@ my $C_path_re = qr{ ^ # for .PL files and get their full path names, so we can exclude each such # file explicitly. This works because other porting tests prohibit having two # files with the same names except for case. -open my $manifest_fh, '<:bytes', $MANIFEST or die "Can't open $MANIFEST"; -while (<$manifest_fh>) { +foreach my $f (@files_from_manifests) { # While we have MANIFEST open, on VMS platforms, look for files that match # the magic VMS file names that have to be handled specially. Add these # to the list of them. - if ($^O eq 'VMS' && / ^ ( [^\t]* $vms_re ) \t /x) { + if ($^O eq 'VMS' && $f =~ m/ ^ ( [^\t]* $vms_re ) /x) { $special_vms_files{$1} = 1; } - if (/ ^ ( [^\t]* \. PL ) \t /x) { + if ( $f =~ m/ ^ ( [^\t]* \. PL ) /x) { $excluded_files{canonicalize($1)} = 1; } } -close $manifest_fh, or die "Can't close $MANIFEST"; - # Pod::Checker messages to suppress my @suppressed_messages = ( diff --git a/t/porting/readme.t b/t/porting/readme.t index 1d79d2caed7a..337c60e87e36 100644 --- a/t/porting/readme.t +++ b/t/porting/readme.t @@ -10,14 +10,14 @@ BEGIN { use TestInit qw(T); # T is chdir to the top level use strict; use warnings; + require './t/test.pl'; -my @porting_files; -open my $man, "MANIFEST" or die "Can't open MANIFEST: $!"; -while(<$man>) { - /^Porting\// and s/[\t\n].*//s, push @porting_files, $_; -} -close $man or die "Can't close MANIFEST: $!"; +use lib '.'; +require Porting::Manifest; + +my @porting_files = Porting::Manifest::get_porting_files(); + # It seems that dying here is nicer than having several dozen failing tests # later. But that assumes one will see the message from die. die "Can't get contents of Porting/ directory.\n" unless @porting_files > 1; @@ -27,8 +27,11 @@ open(my $fh, '<', 'Porting/README.pod') or die("Can't open Porting/README.pod: $ my (@current_order, @sorted_order, %files_in_pod); while(<$fh>) { next unless $_ =~ /^=head/; + my @matches = $_ =~ m/F<([^>]+)>/g; for my $file (@matches) { + next if $file eq 'MANIFEST.SKIP'; # file not shipped but in README.pod + $files_in_pod{$file} = 1; push @current_order, $file; } diff --git a/t/porting/utils.t b/t/porting/utils.t index 92eafbd3b9a2..034417c85a2f 100644 --- a/t/porting/utils.t +++ b/t/porting/utils.t @@ -19,10 +19,12 @@ BEGIN { @INC = '..' if -f '../TestInit.pm'; } + use TestInit qw(T); # T is chdir to the top level use strict; require './t/test.pl'; +require './Porting/Manifest.pm'; # It turns out that, since the default @INC will include your old 5.x libs, if # you have them, the Porting utils might load a library that no longer compiles @@ -31,15 +33,9 @@ require './t/test.pl'; # needed for porters, anyway. -- rjbs, 2012-05-10 find_git_or_skip('all'); -my @maybe; - -open my $fh, '<', 'MANIFEST' or die "Can't open MANIFEST: $!"; -while (<$fh>) { - push @maybe, $1 if m!^(Porting/\S+)!; -} -close $fh or die $!; +my @maybe = Porting::Manifest::get_porting_perl_files(); -open $fh, '<', 'utils.lst' or die "Can't open utils.lst: $!"; +open my $fh, '<', 'utils.lst' or die "Can't open utils.lst: $!"; while (<$fh>) { die unless m!^(\S+)!; push @maybe, $1;