Implement XBPS_DISTFILES_MIRROR (second try)

The idea is to remove some burden from the original download
locations for distfiles by specifying one or more mirror locations.

+ Works with http:// or ftp:// mirrors for all $XBPS_CHROOT_CMD variants
+ Works with file:// mirrors together with uchroot and proot

README.md

@@ -74,6 +74,27 @@ To enable it:
     $ cd void-packages
     $ echo XBPS_CHROOT_CMD=proot >> etc/conf
 
+### Distfiles mirror(s)
+
+In etc/conf you may optionally define a mirror or a list of mirrors to search for distfiles.
+    $ echo 'XBPS_DISTFILES_MIRROR="ftp://192.168.100.5/gentoo/distfiles"' >> etc/conf
+
+If more than one mirror is to be searched, you can either specify multiple URLs separated
+with blanks, or add to the variable like this
+    $ echo 'XBPS_DISTFILES_MIRROR+=" http://repo.voidlinux.de/distfiles"' >> etc/conf
+Make sure to put the blank after the first double quote in this case.
+
+The mirrors are searched in order for the distfiles to build a package until the
+checksum of the downloaded file matches the one specified in the template.
+
+Ultimately, if no mirror carries the distfile, or in case all downloads failed the
+checksum verification, the original download location is used.
+
+If you use `proot` or `uchroot` for your XBPS_CHROOT_CMD, you may also specify a local path
+using the `file://` prefix or simply an absolute path on your build host (e.g. /mnt/distfiles).
+Mirror locations specified this way are bind mounted inside the chroot environment
+under $XBPS_MASTERDIR and searched for distfiles just the same as remote locations.
+
 ### Quick setup in Void
 
 Clone the `void-packages` git repository, install the bootstrap packages:

common/hooks/do-fetch/00-distfiles.sh

@@ -51,6 +51,47 @@ link_cksum() {
 	fi
 }
 
+try_mirrors() {
+	local curfile="$1" distfile="$2" dfcount="$3" subdir="$4" f="$5"
+	local filesum cksum basefile mirror path scheme
+	[ -z "$XBPS_DISTFILES_MIRROR" ] && return
+	basefile="$(basename $f)"
+	cksum=$(get_cksum $curfile $dfcount)
+	for mirror in $XBPS_DISTFILES_MIRROR; do
+		scheme="file"
+		if [[ $mirror == *://* ]]; then
+			scheme="${mirror%%:/*}"
+			path="${mirror#${scheme}://}"
+		else
+			path="$mirror"
+		fi
+		if [ "$scheme" == "file" ]; then
+			# Skip file:// mirror locations (/some/where or file:///some/where)
+			# where the specified directory does not exist
+			if [ ! -d "$path" ]; then
+				msg_warn "$pkgver: mount point $path does not exist...\n"
+				continue
+			fi
+		fi
+		if [[ "$mirror" == *voidlinux* ]]; then
+			# For distfiles.voidlinux.* append the subdirectory
+			mirror="$mirror/$subdir"
+		fi
+		msg_normal "$pkgver: fetching distfile '$curfile' from '$mirror'...\n"
+		$XBPS_FETCH_CMD "$mirror/$basefile"
+		# If basefile was not found, but a curfile file may exist, try to fetch it
+		if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then
+			$XBPS_FETCH_CMD "$mirror/$curfile"
+		fi
+		[ ! -f "$distfile" ] && continue
+		flock -n ${distfile}.part rm -f ${distfile}.part
+		filesum=$(${XBPS_DIGEST_CMD} "$distfile")
+		[ "$cksum" == "$filesum" ] && break
+		msg_normal "$pkgver: checksum failed - removing '$curfile'...\n"
+		rm -f ${distfile}
+	done
+}
+
 hook() {
 	local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version"
 	local dfcount=0 errors=0
@@ -79,13 +120,17 @@ hook() {
 		if [ ! -f "$distfile" ]; then
 			link_cksum $curfile $distfile $dfcount
 		fi
-		# If distfile does not exist, download it.
+		# If distfile does not exist, download it from a mirror location.
+		if [ ! -f "$distfile" ]; then
+			try_mirrors $curfile $distfile $dfcount $pkgname-$version $f
+		fi
+		# If distfile does not exist, download it from the original location.
 		if [ ! -f "$distfile" ]; then
 			msg_normal "$pkgver: fetching distfile '$curfile'...\n"
 			flock "${distfile}.part" $XBPS_FETCH_CMD "$f"
-			if [ ! -f "$distfile" ]; then
-				msg_error "$pkgver: failed to fetch $curfile.\n"
-			fi
+		fi
+		if [ ! -f "$distfile" ]; then
+			msg_error "$pkgver: failed to fetch $curfile.\n"
 		fi
 		# distfile downloaded, verify sha256 hash.
 		flock -n ${distfile}.part rm -f ${distfile}.part

xbps-src

@@ -351,6 +351,32 @@ read_pkg() {
     setup_pkg $XBPS_TARGET_PKG $XBPS_CROSS_BUILD
 }
 
+setup_distfiles_mirror() {
+    local mirror scheme path
+
+    # Scheme file:// mirror locations only work with uchroot or proot
+    for mirror in $XBPS_DISTFILES_MIRROR; do
+        scheme="file"
+        if [[ "$mirror" == *://* ]]; then
+            scheme="${mirror%%://*}"
+            path="${mirror#${scheme}://}"
+        else
+            path="$mirror"
+        fi
+        [ "$scheme" != "file" ] && continue
+        if [ "$XBPS_CHROOT_CMD" == "uchroot" -o "$XBPS_CHROOT_CMD" == "proot" ]; then
+            if [ ! -d "$path" ]; then
+                msg_warn "Invalid path in XBPS_DISTFILES_MIRROR ($mirror)\n"
+                continue
+            fi
+            mkdir -p "$XBPS_MASTERDIR/$path"
+            XBPS_CHROOT_CMD_ARGS+=" -b $path:$path"
+        else
+            msg_warn "File URLs ($mirror) don't work with '$XBPS_CHROOT_CMD'\n"
+        fi
+    done
+}
+
 readonly XBPS_VERSION_REQ="0.46"
 readonly XBPS_VERSION=$(xbps-uhelper -V|awk '{print $2}')
 readonly XBPS_SRC_VERSION="113"
@@ -516,7 +542,7 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \
     XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \
     XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \
     XBPS_SRC_VERSION XBPS_DESTDIR XBPS_MACHINE XBPS_TEMP_MASTERDIR \
-    XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR
+    XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR XBPS_DISTFILES_MIRROR
 
 for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do
     eval val="\$XBPS_$i"
@@ -571,6 +597,9 @@ check_build_requirements
 
 if [ -z "$IN_CHROOT" ]; then
     trap 'exit_func' INT TERM
+    if [ -n "$XBPS_DISTFILES_MIRROR" ]; then
+        setup_distfiles_mirror
+    fi
 fi
 
 #