build.info: Make it possible to use compiled programs as generators

Our goal is to be able to produce fipsmodule.cnf with the help of
'openssl fipsinstall', using the openssl program that we build.

This refactors the generatesrc code in all the build file templates to
replace $generator and $generator_incs with $gen0, $gen_args and $gen_incs,
which makes it easier and more consistent to manipulate different bits
of the generator command, and also keeps the variable names consistent
while not overly long.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14320)

Configurations/descrip.mms.tmpl

@@ -805,19 +805,27 @@ EOF
 
   sub generatesrc {
       my %args = @_;
-      my $generator = join(" ", @{$args{generator}});
+      my $gen0 = $args{generator}->[0];
-      my $generator_incs = join("", map { ' "-I'.$_.'"' } @{$args{generator_incs}});
+      my $gen_args = join('', map { " $_" }
+                              @{$args{generator}}[1..$#{$args{generator}}]);
+      my $gen_incs = join("", map { ' "-I'.$_.'"' } @{$args{generator_incs}});
       my $deps = join(", -\n\t\t", @{$args{generator_deps}}, @{$args{deps}});
 
       if ($args{src} =~ /\.html$/) {
-	  my $title = basename($args{src}, ".html");
+          #
-	  my $pod = $args{generator}->[0];
+          # HTML generator
-	  my $mkpod2html = sourcefile('util', 'mkpod2html.pl');
+          #
-	  return <<"EOF";
+          my $title = basename($args{src}, ".html");
+          my $pod = $gen0;
+          my $mkpod2html = sourcefile('util', 'mkpod2html.pl');
+          return <<"EOF";
 $args{src}: $pod
 	\$(PERL) $mkpod2html -i $pod -o \$\@ -t "$title" -r "\$(SRCDIR)/doc"
 EOF
       } elsif (platform->isdef($args{src})) {
+          #
+          # Linker script-ish generator
+          #
           my $target = platform->def($args{src});
           my $mkdef = sourcefile('util', 'mkdef.pl');
           my $ord_ver = $args{intent} eq 'lib' ? ' --version $(VERSION)' : '';
@@ -827,31 +835,13 @@ EOF
               $target{$args{intent}.'_cflags'} =~ m|/NAMES=[^/]*AS_IS|i
               ? '' : ' --case-insensitive';
           return <<"EOF";
-$target : $args{generator}->[0] $deps $mkdef
+$target : $gen0 $deps $mkdef
-	\$(PERL) $mkdef$ord_ver --ordinals $args{generator}->[0] --name $ord_name "--OS" "VMS"$case_insensitive > $target
+	\$(PERL) $mkdef$ord_ver --ordinals $gen0 --name $ord_name "--OS" "VMS"$case_insensitive > $target
 EOF
-      } elsif (!platform->isasm($args{src})) {
+      } elsif (platform->isasm($args{src})) {
-          my $target = $args{src};
+          #
-          if ($args{generator}->[0] =~ m|^.*\.in$|) {
+          # Assembler generator
-	      my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+          #
-                                                   "util", "dofile.pl")),
-                                   rel2abs($config{builddir}));
-              my @modules = ( 'configdata.pm',
-                              grep { $_ =~ m|\.pm$| } @{$args{deps}} );
-              my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
-              @modules = map { '"-M'.basename($_, '.pm').'"' } @modules;
-              my $modules = join(' ', '', sort keys %moduleincs, @modules);
-              return <<"EOF";
-$target : $args{generator}->[0] $deps
-	\$(PERL)$modules $dofile "-o$target{build_file}" $generator > \$\@
-EOF
-	  } else {
-              return <<"EOF";
-$target : $args{generator}->[0] $deps
-	\$(PERL)$generator_incs $generator > \$\@
-EOF
-	  }
-      } else {
           my $cppflags = {
               shlib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
               lib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
@@ -870,13 +860,14 @@ EOF
           my $defs = join("", map { ",".$_ } @{$args{defs}});
           my $target = platform->asm($args{src});
 
-          if ($args{generator}->[0] =~ /\.pl$/) {
+          my $generator;
-              $generator = '$(PERL)'.$generator_incs.' '.$generator
+          if ($gen0 =~ /\.pl$/) {
+              $generator = '$(PERL)'.$gen_incs.' '.$gen0.$gen_args
                   .' '.$cppflags;
-          } elsif ($args{generator}->[0] =~ /\.S$/) {
+          } elsif ($gen0 =~ /\.S$/) {
               $generator = undef;
           } else {
-              die "Generator type for $src unknown: $generator\n";
+              die "Generator type for $src unknown: $gen0.$gen_args\n";
           }
 
           if (defined($generator)) {
@@ -884,7 +875,7 @@ EOF
               # end up generating foo.s in two steps.
               if ($args{src} =~ /\.S$/) {
                    return <<"EOF";
-$target : $args{generator}->[0] $deps
+$target : $gen0 $deps
 	$generator \$\@-S
         \@ $incs_on
         \@ extradefines = "$defs"
@@ -898,7 +889,7 @@ EOF
               }
               # Otherwise....
               return <<"EOF";
-$target : $args{generator}->[0] $deps
+$target : $gen0 $deps
         \@ $incs_on
         \@ extradefines = "$defs"
 	$generator \$\@
@@ -907,14 +898,66 @@ $target : $args{generator}->[0] $deps
 EOF
           }
           return <<"EOF";
-$target : $args{generator}->[0] $deps
+$target : $gen0 $deps
         \@ $incs_on
         \@ extradefines = "$defs"
         SHOW SYMBOL qual_includes
-        PIPE \$(CPP) $cppflags $args{generator}->[0] | -
+        PIPE \$(CPP) $cppflags $gen0 | -
         \$(PERL) "-ne" "/^#(\\s*line)?\\s*[0-9]+\\s+""/ or print" > \$\@
         \@ DELETE/SYMBOL/LOCAL extradefines
         \@ $incs_off
+EOF
+      } elsif ($gen0 =~ m|^.*\.in$|) {
+          #
+          # "dofile" generator (file.in -> file)
+          #
+          my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+                                               "util", "dofile.pl")),
+                               rel2abs($config{builddir}));
+          my @modules = ( 'configdata.pm',
+                          grep { $_ =~ m|\.pm$| } @{$args{deps}} );
+          my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
+          $deps = join(' ', $deps, @modules);
+          @modules = map { '"-M'.basename($_, '.pm').'"' } @modules;
+          my $modules = join(' ', '', sort keys %moduleincs, @modules);
+          return <<"EOF";
+$target : $gen0 $deps
+	\$(PERL)$modules $dofile "-o$target{build_file}" $gen0$gen_args > \$\@
+EOF
+      } elsif (grep { $_ eq $gen0 } @{$unified_info{programs}}) {
+          #
+          # Generic generator using OpenSSL programs
+          #
+
+          # Redo $deps, because programs aren't expected to have deps of their
+          # own.  This is a little more tricky, though, because running programs
+          # may have dependencies on all sorts of files, so we search through
+          # our database of programs and modules to see if our dependencies
+          # are one of those.
+          $deps = join(' ', map { my $x = $_;
+                                  if (grep { $x eq $_ }
+                                          @{$unified_info{programs}}) {
+                                      platform->bin($x);
+                                  } elsif (grep { $x eq $_ }
+                                          @{$unified_info{modules}}) {
+                                      platform->dso($x);
+                                  } else {
+                                      $x;
+                                  }
+                                } @{$args{deps}});
+          # Also redo $gen0, to ensure that we have the proper extension
+          $gen0 = platform->bin($gen0);
+          return <<"EOF";
+$args{src}: $gen0 $deps
+	PIPE $gen0$gen_args > \$@
+EOF
+      } else {
+          #
+          # Generic generator using Perl
+          #
+          return <<"EOF";
+$target : $gen0 $deps
+	\$(PERL)$gen_incs $gen0$gen_args > \$\@
 EOF
       }
   }

Configurations/unix-Makefile.tmpl

@@ -1315,58 +1315,52 @@ EOF
 
   sub generatesrc {
       my %args = @_;
-      my $generator = join(" ", @{$args{generator}});
+      my $gen0 = $args{generator}->[0];
-      my $generator_incs = join("", map { " -I".$_ } @{$args{generator_incs}});
+      my $gen_args = join('', map { " $_" }
+                              @{$args{generator}}[1..$#{$args{generator}}]);
+      my $gen_incs = join("", map { " -I".$_ } @{$args{generator_incs}});
       my $incs = join("", map { " -I".$_ } @{$args{incs}});
       my $defs = join("", map { " -D".$_ } @{$args{defs}});
       my $deps = join(" ", @{$args{generator_deps}}, @{$args{deps}});
 
       if ($args{src} =~ /\.html$/) {
-	  my $title = basename($args{src}, ".html");
+          #
-	  my $pod = $args{generator}->[0];
+          # HTML generator
-	  return <<"EOF";
+          #
+          my $title = basename($args{src}, ".html");
+          my $pod = $gen0;
+          return <<"EOF";
 $args{src}: $pod
 	\$(PERL) \$(SRCDIR)/util/mkpod2html.pl -i "$pod" -o \$\@ -t "$title" -r "\$(SRCDIR)/doc"
 EOF
       } elsif ($args{src} =~ /\.(\d)$/) {
-	  my $section = $1;
+          #
-	  my $name = uc basename($args{src}, ".$section");
+          # Man-page generator
-	  my $pod = $args{generator}->[0];
+          #
-	  return <<"EOF";
+          my $section = $1;
+          my $name = uc basename($args{src}, ".$section");
+          my $pod = $gen0;
+          return <<"EOF";
 $args{src}: $pod
 	pod2man --name=$name --section=$section --center=OpenSSL \\
 		--release=\$(VERSION) $pod >\$\@
 EOF
       } elsif (platform->isdef($args{src})) {
+          #
+          # Linker script-ish generator
+          #
           my $target = platform->def($args{src});
           (my $mkdef_os = $target{shared_target}) =~ s|-shared$||;
           my $ord_ver = $args{intent} eq 'lib' ? ' --version $(VERSION)' : '';
           my $ord_name = $args{generator}->[1] || $args{product};
           return <<"EOF";
-$target: $args{generator}->[0] $deps \$(SRCDIR)/util/mkdef.pl
+$target: $gen0 $deps \$(SRCDIR)/util/mkdef.pl
-	\$(PERL) \$(SRCDIR)/util/mkdef.pl$ord_ver --ordinals $args{generator}->[0]  --name $ord_name --OS $mkdef_os > $target
+	\$(PERL) \$(SRCDIR)/util/mkdef.pl$ord_ver --ordinals $gen0  --name $ord_name --OS $mkdef_os > $target
 EOF
-      } elsif (!platform->isasm($args{src})) {
+      } elsif (platform->isasm($args{src})) {
-          if ($args{generator}->[0] =~ m|^.*\.in$|) {
+          #
-              my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+          # Assembler generator
-                                                   "util", "dofile.pl")),
+          #
-                                   rel2abs($config{builddir}));
-              my @modules = ( 'configdata.pm',
-                              grep { $_ =~ m|\.pm$| } @{$args{deps}} );
-              my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
-              @modules = map { "-M".basename($_, '.pm') } @modules;
-              my $modules = join(' ', '', sort keys %moduleincs, @modules);
-              return <<"EOF";
-$args{src}: $args{generator}->[0] $deps \$(BLDDIR)/configdata.pm
-	\$(PERL)$modules "$dofile" "-o$target{build_file}" $generator > \$@
-EOF
-	  } else {
-              return <<"EOF";
-$args{src}: $args{generator}->[0] $deps
-	\$(PERL)$generator_incs $generator > \$@
-EOF
-	  }
-      } else {
           my $cppflags = {
               shlib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
               lib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
@@ -1374,27 +1368,81 @@ EOF
               bin => '$(BIN_CFLAGS) $(BIN_CPPFLAGS)'
           } -> {$args{intent}};
 
-          if ($args{generator}->[0] =~ /\.pl$/) {
+          my $generator;
-              $generator = 'CC="$(CC)" $(PERL)'.$generator_incs.' '.$generator
+          if ($gen0 =~ /\.pl$/) {
+              $generator = 'CC="$(CC)" $(PERL)'.$gen_incs.' '.$gen0.$gen_args
                   .' "$(PERLASM_SCHEME)"'.$incs.' '.$cppflags.$defs.' $(PROCESSOR)';
-          } elsif ($args{generator}->[0] =~ /\.m4$/) {
+          } elsif ($gen0 =~ /\.m4$/) {
-              $generator = 'm4 -B 8192'.$generator_incs.' '.$generator.' >'
+              $generator = 'm4 -B 8192'.$gen_incs.' '.$gen0.$gen_args.' >'
-          } elsif ($args{generator}->[0] =~ /\.S$/) {
+          } elsif ($gen0 =~ /\.S$/) {
               $generator = undef;
           } else {
-              die "Generator type for $args{src} unknown: $generator\n";
+              die "Generator type for $args{src} unknown: $gen0\n";
           }
 
           if (defined($generator)) {
               return <<"EOF";
-$args{src}: $args{generator}->[0] $deps
+$args{src}: $gen0 $deps
 	$generator \$@
 EOF
           }
           return <<"EOF";
-$args{src}: $args{generator}->[0] $deps
+$args{src}: $gen0 $deps
-	\$(CC) $incs $cppflags $defs -E $args{generator}->[0] | \\
+	\$(CC) $incs $cppflags $defs -E $gen0 | \\
 	\$(PERL) -ne '/^#(line)?\\s*[0-9]+/ or print' > \$@
+EOF
+      } elsif ($gen0 =~ m|^.*\.in$|) {
+          #
+          # "dofile" generator (file.in -> file)
+          #
+          my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+                                               "util", "dofile.pl")),
+                               rel2abs($config{builddir}));
+          my @modules = ( 'configdata.pm',
+                          grep { $_ =~ m|\.pm$| } @{$args{deps}} );
+          my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
+          $deps = join(' ', $deps, @modules);
+          @modules = map { "-M".basename($_, '.pm') } @modules;
+          my $modules = join(' ', '', sort keys %moduleincs, @modules);
+          return <<"EOF";
+$args{src}: $gen0 $deps
+	\$(PERL)$modules "$dofile" "-o$target{build_file}" $gen0$gen_args > \$@
+EOF
+      } elsif (grep { $_ eq $gen0 } @{$unified_info{programs}}) {
+          #
+          # Generic generator using OpenSSL programs
+          #
+
+          # Redo $deps, because programs aren't expected to have deps of their
+          # own.  This is a little more tricky, though, because running programs
+          # may have dependencies on all sorts of files, so we search through
+          # our database of programs and modules to see if our dependencies
+          # are one of those.
+          $deps = join(' ', map { my $x = $_;
+                                  if (grep { $x eq $_ }
+                                          @{$unified_info{programs}}) {
+                                      platform->bin($x);
+                                  } elsif (grep { $x eq $_ }
+                                          @{$unified_info{modules}}) {
+                                      platform->dso($x);
+                                  } else {
+                                      $x;
+                                  }
+                                } @{$args{deps}});
+          # Also redo $gen0, to ensure that we have the proper extension where
+          # necessary.
+          $gen0 = platform->bin($gen0);
+          return <<"EOF";
+$args{src}: $gen0 $deps \$(BLDDIR)/util/wrap.pl
+	\$(BLDDIR)/util/wrap.pl $gen0$gen_args > \$@
+EOF
+      } else {
+          #
+          # Generic generator using Perl
+          #
+          return <<"EOF";
+$args{src}: $gen0 $deps
+	\$(PERL)$gen_incs $gen0$gen_args > \$@
 EOF
       }
   }

Configurations/windows-makefile.tmpl

@@ -637,22 +637,29 @@ EOF
 
   sub generatesrc {
       my %args = @_;
-      my ($gen0, @gens) = @{$args{generator}};
+      my $gen0 = $args{generator}->[0];
-      my $generator = '"'.$gen0.'"'.join('', map { " $_" } @gens);
+      my $gen_args = join('', map { " $_" }
-      my $generator_incs = join("", map { " -I \"$_\"" } @{$args{generator_incs}});
+                              @{$args{generator}}[1..$#{$args{generator}}]);
+      my $gen_incs = join("", map { " -I \"$_\"" } @{$args{generator_incs}});
       my $incs = join("", map { " -I \"$_\"" } @{$args{incs}});
       my $defs = join("", map { " -D".$_ } @{$args{defs}});
       my $deps = @{$args{deps}} ?
           '"'.join('" "', @{$args{generator_deps}}, @{$args{deps}}).'"' : '';
 
       if ($args{src} =~ /\.html$/) {
-	  my $title = basename($args{src}, ".html");
+          #
-	  my $pod = $args{generator}->[0];
+          # HTML generator
-	  return <<"EOF";
+          #
+          my $title = basename($args{src}, ".html");
+          my $pod = $gen0;
+          return <<"EOF";
 $args{src}: "$pod"
 	\$(PERL) \$(SRCDIR)/util/mkpod2html.pl -i "$pod" -o \$\@ -t "$title" -r "\$(SRCDIR)/doc"
 EOF
       } elsif (platform->isdef($args{src})) {
+          #
+          # Linker script-ish generator
+          #
           my $target = platform->def($args{src});
           my $mkdef = abs2rel(rel2abs(catfile($config{sourcedir},
                                               "util", "mkdef.pl")),
@@ -661,31 +668,13 @@ EOF
           my $ord_name =
               $args{generator}->[1] || platform->dsoname($args{product});
           return <<"EOF";
-$target: $args{generator}->[0] $deps $mkdef
+$target: $gen0 $deps $mkdef
-	"\$(PERL)" $mkdef$ord_ver --ordinals $args{generator}->[0] --name $ord_name --OS windows > $target
+	"\$(PERL)" $mkdef$ord_ver --ordinals $gen0 --name $ord_name --OS windows > $target
 EOF
-      } elsif (!platform->isasm($args{src})) {
+      } elsif (platform->isasm($args{src})) {
-          my $target = $args{src};
+          #
-          if ($args{generator}->[0] =~ m|^.*\.in$|) {
+          # Assembler generator
-              my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+          #
-                                                   "util", "dofile.pl")),
-                                   rel2abs($config{builddir}));
-              my @modules = ( 'configdata.pm',
-                              grep { $_ =~ m|\.pm$| } @{$args{deps}} );
-              my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
-              @modules = map { "-M".basename($_, '.pm') } @modules;
-              my $modules = join(' ', '', sort keys %moduleincs, @modules);
-              return <<"EOF";
-$target: "$args{generator}->[0]" $deps
-	"\$(PERL)"$modules "$dofile" "-o$target{build_file}" $generator > \$@
-EOF
-	  } else {
-              return <<"EOF";
-$target: "$args{generator}->[0]" $deps
-	"\$(PERL)"$generator_incs $generator > \$@
-EOF
-	  }
-      } else {
           my $cppflags = {
               shlib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
               lib => '$(LIB_CFLAGS) $(LIB_CPPFLAGS)',
@@ -694,13 +683,14 @@ EOF
           } -> {$args{intent}};
           my $target = platform->asm($args{src});
 
-          if ($args{generator}->[0] =~ /\.pl$/) {
+          my $generator;
-              $generator = '"$(PERL)"'.$generator_incs.' '.$generator
+          if ($gen0 =~ /\.pl$/) {
+              $generator = '"$(PERL)"'.$gen_incs.' '.$gen0.$gen_args
                   .' "$(PERLASM_SCHEME)"'.$incs.' '.$cppflags.$defs.' $(PROCESSSOR)';
-          } elsif ($args{generator}->[0] =~ /\.S$/) {
+          } elsif ($gen0 =~ /\.S$/) {
               $generator = undef;
           } else {
-              die "Generator type for $src unknown: $generator\n";
+              die "Generator type for $src unknown: $gen0\n";
           }
 
           if (defined($generator)) {
@@ -708,7 +698,7 @@ EOF
               # end up generating foo.s in two steps.
               if ($args{src} =~ /\.S$/) {
                    return <<"EOF";
-$target: "$args{generator}->[0]" $deps
+$target: "$gen0" $deps
 	set ASM=\$(AS)
 	$generator \$@.S
 	\$(CPP) $incs $cppflags $defs \$@.S > \$@.i && move /Y \$@.i \$@
@@ -717,14 +707,66 @@ EOF
               }
               # Otherwise....
               return <<"EOF";
-$target: "$args{generator}->[0]" $deps
+$target: "$gen0" $deps
 	set ASM=\$(AS)
 	$generator \$@
 EOF
           }
           return <<"EOF";
-$target: "$args{generator}->[0]" $deps
+$target: "$gen0" $deps
-	\$(CPP) $incs $cppflags $defs "$args{generator}->[0]" > \$@.i && move /Y \$@.i \$@
+	\$(CPP) $incs $cppflags $defs "$gen0" > \$@.i && move /Y \$@.i \$@
+EOF
+      } elsif ($gen0 =~ m|^.*\.in$|) {
+          #
+          # "dofile" generator (file.in -> file)
+          #
+          my $dofile = abs2rel(rel2abs(catfile($config{sourcedir},
+                                               "util", "dofile.pl")),
+                               rel2abs($config{builddir}));
+          my @modules = ( 'configdata.pm',
+                          grep { $_ =~ m|\.pm$| } @{$args{deps}} );
+          my %moduleincs = map { '"-I'.dirname($_).'"' => 1 } @modules;
+          $deps = join(' ', $deps, @modules);
+          @modules = map { "-M".basename($_, '.pm') } @modules;
+          my $modules = join(' ', '', sort keys %moduleincs, @modules);
+          return <<"EOF";
+$args{src}: "$gen0" $deps
+	"\$(PERL)"$modules "$dofile" "-o$target{build_file}" "$gen0"$gen_args > \$@
+EOF
+      } elsif (grep { $_ eq $gen0 } @{$unified_info{programs}}) {
+          #
+          # Generic generator using OpenSSL programs
+          #
+
+          # Redo $deps, because programs aren't expected to have deps of their
+          # own.  This is a little more tricky, though, because running programs
+          # may have dependencies on all sorts of files, so we search through
+          # our database of programs and modules to see if our dependencies
+          # are one of those.
+          $deps = join(' ', map { my $x = $_;
+                                  if (grep { $x eq $_ }
+                                          @{$unified_info{programs}}) {
+                                      platform->bin($x);
+                                  } elsif (grep { $x eq $_ }
+                                          @{$unified_info{modules}}) {
+                                      platform->dso($x);
+                                  } else {
+                                      $x;
+                                  }
+                                } @{$args{deps}});
+          # Also redo $gen0, to ensure that we have the proper extension.
+          $gen0 = platform->bin($gen0);
+          return <<"EOF";
+$args{src}: $gen0 $deps
+	$gen0$gen_args > \$@
+EOF
+      } else {
+          #
+          # Generic generator using Perl
+          #
+          return <<"EOF";
+$args{src}: "$gen0" $deps
+	"\$(PERL)"$gen_incs $gen0$gen_args > \$@
 EOF
       }
   }

Configure

@@ -2320,7 +2320,7 @@ EOF
             $generator[0] = cleanfile($sourced, $gen, $blddir);
 
             # If the generator is itself generated, it's in the build tree
-            if ($generate{$gen}) {
+            if ($generate{$gen} || ! -f $generator[0]) {
                 $generator[0] = cleanfile($buildd, $gen, $blddir);
             }
             $check_generate{$ddest}->{$generator[0]}++;