Merge pull request #223261 from xworld21/copy-tarballs-use-all-urls

copy-tarballs: use all the urls of each file

Author: raboof

maintainers/scripts/copy-tarballs.pl

@@ -162,13 +162,18 @@ sub uploadFile {
     # Check every fetchurl call discovered by find-tarballs.nix.
     my $mirrored = 0;
     my $have = 0;
-    foreach my $fetch (sort { $a->{url} cmp $b->{url} } @{$fetches}) {
-        my $url = $fetch->{url};
+    foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
+        my $urls = $fetch->{urls};
         my $algo = $fetch->{type};
         my $hash = $fetch->{hash};
         my $name = $fetch->{name};
         my $isPatch = $fetch->{isPatch};
 
+        if ($isPatch) {
+            print STDERR "skipping $urls->[0] (support for patches is missing)\n";
+            next;
+        }
+
         if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
             $algo = $1;
             $hash = `nix hash to-base16 $hash` or die;
@@ -183,62 +188,60 @@ sub uploadFile {
             chomp $hash;
         }
 
-        if (defined $ENV{DEBUG}) {
-            print "$url $algo $hash\n";
-            next;
-        }
-
-        if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
-            print STDERR "skipping $url (unsupported scheme)\n";
-            next;
-        }
-
-        if ($isPatch) {
-            print STDERR "skipping $url (support for patches is missing)\n";
-            next;
-        }
+        my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
 
-        next if defined $exclude && $url =~ /$exclude/;
+        for my $url (@$urls) {
+            if (defined $ENV{DEBUG}) {
+                print "$url $algo $hash\n";
+                next;
+            }
 
-        if (alreadyMirrored($algo, $hash)) {
-            $have++;
-            next;
-        }
+            if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
+                print STDERR "skipping $url (unsupported scheme)\n";
+                next;
+            }
 
-        my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
+            next if defined $exclude && $url =~ /$exclude/;
 
-        print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
+            if (alreadyMirrored($algo, $hash)) {
+                $have++;
+                last;
+            }
 
-        if ($dryRun) {
-            $mirrored++;
-            next;
-        }
+            print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
 
-        # Substitute the output.
-        if (!isValidPath($storePath)) {
-            system("nix-store", "-r", $storePath);
-        }
+            if ($dryRun) {
+                $mirrored++;
+                last;
+            }
 
-        # Otherwise download the file using nix-prefetch-url.
-        if (!isValidPath($storePath)) {
-            $ENV{QUIET} = 1;
-            $ENV{PRINT_PATH} = 1;
-            my $fh;
-            my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
-            waitpid($pid, 0) or die;
-            if ($? != 0) {
-                print STDERR "failed to fetch $url: $?\n";
-                next;
+            # Substitute the output.
+            if (!isValidPath($storePath)) {
+                system("nix-store", "-r", $storePath);
             }
-            <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
-            if ($storePath ne $storePath2) {
-                warn "strange: $storePath != $storePath2\n";
-                next;
+
+            # Otherwise download the file using nix-prefetch-url.
+            if (!isValidPath($storePath)) {
+                $ENV{QUIET} = 1;
+                $ENV{PRINT_PATH} = 1;
+                my $fh;
+                my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
+                waitpid($pid, 0) or die;
+                if ($? != 0) {
+                    print STDERR "failed to fetch $url: $?\n";
+                    next;
+                }
+                <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
+                if ($storePath ne $storePath2) {
+                    warn "strange: $storePath != $storePath2\n";
+                    next;
+                }
             }
-        }
 
-        uploadFile($storePath, $url);
-        $mirrored++;
+            uploadFile($storePath, $url);
+            $mirrored++;
+            last;
+        }
     }
 
     print STDERR "mirrored $mirrored files, already have $have files\n";

maintainers/scripts/find-tarballs.nix

@@ -9,12 +9,12 @@ let
 
   root = expr;
 
-  uniqueUrls = map (x: x.file) (genericClosure {
-    startSet = map (file: { key = file.url; inherit file; }) urls;
+  uniqueFiles = map (x: x.file) (genericClosure {
+    startSet = map (file: { key = with file; (if type == null then "" else type + "+") + hash; inherit file; }) files;
     operator = const [ ];
   });
 
-  urls = map (drv: { url = head (drv.urls or [ drv.url ]); hash = drv.outputHash; isPatch = (drv?postFetch && drv.postFetch != ""); type = drv.outputHashAlgo; name = drv.name; }) fetchurlDependencies;
+  files = map (drv: { urls = drv.urls or [ drv.url ]; hash = drv.outputHash; isPatch = (drv?postFetch && drv.postFetch != ""); type = drv.outputHashAlgo; name = drv.name; }) fetchurlDependencies;
 
   fetchurlDependencies =
     filter
@@ -47,4 +47,4 @@ let
 
   canEval = val: (builtins.tryEval val).success;
 
-in uniqueUrls
+in uniqueFiles