Rework the perl fallback functionality

The module with_fallback.pm was kind of clunky and required a transfer
module.  This change replaces if with a much more generic pragma type
module, which simply appends given directories to @INC (as opposed to
the 'lib' pragma, which prepends the directories to @INC).

This also supports having a file MODULES.txt with sub-directories to
modules.  This ensures that we don't have to spray individual module
paths throughout our perl code, but can have them collected in one
place.

(do note that there is a 'fallback' module on CPAN.  However, it isn't
part of the core perl, and it has no support the any MODULES.txt kind
of construct)

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9826)

Configure

@@ -17,6 +17,7 @@ use lib "$FindBin::Bin/util/perl";
 use File::Basename;
 use File::Spec::Functions qw/:DEFAULT abs2rel rel2abs splitdir/;
 use File::Path qw/mkpath/;
+use OpenSSL::fallback "$FindBin::Bin/external/perl/MODULES.txt";
 use OpenSSL::Glob;
 
 # see INSTALL for instructions.
@@ -1617,7 +1618,7 @@ my %unified_info = ();
 
 my $buildinfo_debug = defined($ENV{CONFIGURE_DEBUG_BUILDINFO});
 if ($builder eq "unified") {
-    use with_fallback qw(Text::Template);
+    use Text::Template 1.46;
 
     sub cleandir {
         my $base = shift;

external/perl/MODULES.txt

@@ -0,0 +1 @@
+Text-Template-1.46/lib

external/perl/transfer/Text/Template.pm

@@ -1,23 +0,0 @@
-# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
-#
-# Licensed under the Apache License 2.0 (the "License").  You may not use
-# this file except in compliance with the License.  You can obtain a copy
-# in the file LICENSE in the source distribution or at
-# https://www.openssl.org/source/license.html
-
-# Quick transfer to the downloaded Text::Template
-
-package transfer::Text::Template;
-$VERSION = 1.46;
-
-BEGIN {
-    use File::Spec::Functions;
-    use File::Basename;
-    use lib catdir(dirname(__FILE__), "..", "..", "Text-Template-1.46", "lib");
-    # Some unpackers on VMS convert periods in directory names to underscores
-    use lib catdir(dirname(__FILE__), "..", "..", "Text-Template-1_46", "lib");
-    use Text::Template;
-    shift @INC;                 # Takes away the effect of use lib
-    shift @INC;                 # Takes away the effect of use lib
-}
-1;

test/generate_ssl_tests.pl

@@ -22,10 +22,12 @@ BEGIN {
     OpenSSL::Test::setup("no_test_here");
 }
 
-use lib srctop_dir("util", "perl");  # for with_fallback
-use lib srctop_dir("test", "ssl-tests");  # for ssltests_base
+use FindBin;
+use lib "$FindBin::Bin/../util/perl";
+use OpenSSL::fallback "$FindBin::Bin/../external/perl/MODULES.txt";
+use Text::Template 1.46;
 
-use with_fallback qw(Text::Template);
+use lib "$FindBin::Bin/ssl-tests";
 
 use vars qw/@ISA/;
 push (@ISA, qw/Text::Template/);

util/dofile.pl

@@ -14,8 +14,9 @@
 use strict;
 use warnings;
 
-use FindBin;
 use Getopt::Std;
+use FindBin;
+use lib "$FindBin::Bin/perl";
 
 # We actually expect to get the following hash tables from configdata:
 #
@@ -37,10 +38,8 @@ package OpenSSL::Template;
 
 # Because we know that Text::Template isn't a core Perl module, we use
 # a fallback in case it's not installed on the system
-use File::Basename;
-use File::Spec::Functions;
-use lib "$FindBin::Bin/perl";
-use with_fallback "Text::Template 1.46";
+use OpenSSL::fallback "$FindBin::Bin/../external/perl/MODULES.txt";
+use Text::Template 1.46;
 
 #use parent qw/Text::Template/;
 use vars qw/@ISA/;

util/perl/OpenSSL/fallback.pm

@@ -0,0 +1,112 @@
+# Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+=head1 NAME
+
+OpenSSL::fallback - push directories to the end of @INC at compile time
+
+=cut
+
+package OpenSSL::fallback;
+
+use strict;
+use warnings;
+use Carp;
+
+our $VERSION = '0.01';
+
+=head1 SYNOPSIS
+
+    use OpenSSL::fallback LIST;
+
+=head1 DESCRIPTION
+
+This small simple module simplifies the addition of fallback directories
+in @INC at compile time.
+
+It is used to add extra directories at the end of perl's search path so
+that later "use" or "require" statements will find modules which are not
+located on perl's default search path.
+
+This is similar to L<lib>, except the paths are I<appended> to @INC rather
+than prepended, thus allowing the use of a newer module on perl's default
+search path if there is one.
+
+=head1 CAVEAT
+
+Just like with B<lib>, this only works with Unix filepaths.
+Just like with L<lib>, this doesn't mean that it only works on Unix, but that
+non-Unix users must first translate their file paths to Unix conventions.
+
+    # VMS users wanting to put [.my.stuff] into their @INC should write:
+    use fallback 'my/stuff';
+
+=head1 NOTES
+
+If you try to add a file to @INC as follows, you will be warned, and the file
+will be ignored:
+
+    use fallback 'file.txt';
+
+The sole exception is the file F<MODULES.txt>, which must contain a list of
+sub-directories relative to the location of that F<MODULES.txt> file.
+All these sub-directories will be appended to @INC.
+
+=cut
+
+# Forward declare
+sub glob;
+
+use constant DEBUG => 0;
+
+sub import {
+    shift;                      # Skip module name
+
+    foreach (@_) {
+        my $path = $_;
+
+        if ($path eq '') {
+            carp "Empty compile time value given to use fallback";
+            next;
+        }
+
+        print STDERR "DEBUG: $path\n" if DEBUG;
+
+        unless (-e $path
+                && ($path =~ m/(?:^|\/)MODULES.txt/ || -d $path)) {
+            croak "Parameter to use fallback must be a directory, not a file";
+            next;
+        }
+
+        my @dirs = ();
+        if (-f $path) {         # It's a MODULES.txt file
+            (my $dir = $path) =~ s|/[^/]*$||; # quick dirname
+            open my $fh, $path or die "Could not open $path: $!\n";
+            while (my $l = <$fh>) {
+                $l =~ s|\R$||;        # Better chomp
+                my $d = "$dir/$l";
+                croak "All lines in $path must be a directory, not a file: $l"
+                    unless -e $d && -d $d;
+                push @INC, $d;
+            }
+        } else {                # It's a directory
+            push @INC, $path;
+        }
+    }
+}
+
+=head1 SEE ALSO
+
+L<FindBin> - optional module which deals with paths relative to the source
+file.
+
+=head1 AUTHOR
+
+Richard Levitte, 2019
+
+=cut
+

util/perl/with_fallback.pm

@@ -1,27 +0,0 @@
-# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
-#
-# Licensed under the Apache License 2.0 (the "License").  You may not use
-# this file except in compliance with the License.  You can obtain a copy
-# in the file LICENSE in the source distribution or at
-# https://www.openssl.org/source/license.html
-
-package with_fallback;
-
-sub import {
-    shift;
-
-    use File::Basename;
-    use File::Spec::Functions;
-    foreach (@_) {
-	eval "use $_";
-	if ($@) {
-	    unshift @INC, catdir(dirname(__FILE__),
-                                 "..", "..", "external", "perl");
-	    my $transfer = "transfer::$_";
-	    eval "use $transfer";
-	    shift @INC;
-	    warn $@ if $@;
-	}
-    }
-}
-1;