diff --git a/README.md b/README.md index 849d88b5221..2287f5ca13b 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,27 @@ To enable it: $ cd void-packages $ echo XBPS_CHROOT_CMD=proot >> etc/conf +### Distfiles mirror(s) + +In etc/conf you may optionally define a mirror or a list of mirrors to search for distfiles. + $ echo 'XBPS_DISTFILES_MIRROR="ftp://192.168.100.5/gentoo/distfiles"' >> etc/conf + +If more than one mirror is to be searched, you can either specify multiple URLs separated +with blanks, or add to the variable like this + $ echo 'XBPS_DISTFILES_MIRROR+=" http://repo.voidlinux.de/distfiles"' >> etc/conf +Make sure to put the blank after the first double quote in this case. + +The mirrors are searched in order for the distfiles to build a package until the +checksum of the downloaded file matches the one specified in the template. + +Ultimately, if no mirror carries the distfile, or in case all downloads failed the +checksum verification, the original download location is used. + +If you use `proot` or `uchroot` for your XBPS_CHROOT_CMD, you may also specify a local path +using the `file://` prefix or simply an absolute path on your build host (e.g. /mnt/distfiles). +Mirror locations specified this way are bind mounted inside the chroot environment +under $XBPS_MASTERDIR and searched for distfiles just the same as remote locations. + ### Quick setup in Void Clone the `void-packages` git repository, install the bootstrap packages: diff --git a/common/hooks/do-fetch/00-distfiles.sh b/common/hooks/do-fetch/00-distfiles.sh index 43cc298b6c4..fa43f1a4abe 100644 --- a/common/hooks/do-fetch/00-distfiles.sh +++ b/common/hooks/do-fetch/00-distfiles.sh @@ -51,6 +51,47 @@ link_cksum() { fi } +try_mirrors() { + local curfile="$1" distfile="$2" dfcount="$3" subdir="$4" f="$5" + local filesum cksum basefile mirror path scheme + [ -z "$XBPS_DISTFILES_MIRROR" ] && return + basefile="$(basename $f)" + cksum=$(get_cksum $curfile $dfcount) + for mirror in $XBPS_DISTFILES_MIRROR; do + scheme="file" + if [[ $mirror == *://* ]]; then + scheme="${mirror%%:/*}" + path="${mirror#${scheme}://}" + else + path="$mirror" + fi + if [ "$scheme" == "file" ]; then + # Skip file:// mirror locations (/some/where or file:///some/where) + # where the specified directory does not exist + if [ ! -d "$path" ]; then + msg_warn "$pkgver: mount point $path does not exist...\n" + continue + fi + fi + if [[ "$mirror" == *voidlinux* ]]; then + # For distfiles.voidlinux.* append the subdirectory + mirror="$mirror/$subdir" + fi + msg_normal "$pkgver: fetching distfile '$curfile' from '$mirror'...\n" + $XBPS_FETCH_CMD "$mirror/$basefile" + # If basefile was not found, but a curfile file may exist, try to fetch it + if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then + $XBPS_FETCH_CMD "$mirror/$curfile" + fi + [ ! -f "$distfile" ] && continue + flock -n ${distfile}.part rm -f ${distfile}.part + filesum=$(${XBPS_DIGEST_CMD} "$distfile") + [ "$cksum" == "$filesum" ] && break + msg_normal "$pkgver: checksum failed - removing '$curfile'...\n" + rm -f ${distfile} + done +} + hook() { local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version" local dfcount=0 errors=0 @@ -79,13 +120,17 @@ hook() { if [ ! -f "$distfile" ]; then link_cksum $curfile $distfile $dfcount fi - # If distfile does not exist, download it. + # If distfile does not exist, download it from a mirror location. + if [ ! -f "$distfile" ]; then + try_mirrors $curfile $distfile $dfcount $pkgname-$version $f + fi + # If distfile does not exist, download it from the original location. if [ ! -f "$distfile" ]; then msg_normal "$pkgver: fetching distfile '$curfile'...\n" flock "${distfile}.part" $XBPS_FETCH_CMD "$f" - if [ ! -f "$distfile" ]; then - msg_error "$pkgver: failed to fetch $curfile.\n" - fi + fi + if [ ! -f "$distfile" ]; then + msg_error "$pkgver: failed to fetch $curfile.\n" fi # distfile downloaded, verify sha256 hash. flock -n ${distfile}.part rm -f ${distfile}.part diff --git a/xbps-src b/xbps-src index 04a8ae35933..1f6685c0d48 100755 --- a/xbps-src +++ b/xbps-src @@ -351,6 +351,32 @@ read_pkg() { setup_pkg $XBPS_TARGET_PKG $XBPS_CROSS_BUILD } +setup_distfiles_mirror() { + local mirror scheme path + + # Scheme file:// mirror locations only work with uchroot or proot + for mirror in $XBPS_DISTFILES_MIRROR; do + scheme="file" + if [[ "$mirror" == *://* ]]; then + scheme="${mirror%%://*}" + path="${mirror#${scheme}://}" + else + path="$mirror" + fi + [ "$scheme" != "file" ] && continue + if [ "$XBPS_CHROOT_CMD" == "uchroot" -o "$XBPS_CHROOT_CMD" == "proot" ]; then + if [ ! -d "$path" ]; then + msg_warn "Invalid path in XBPS_DISTFILES_MIRROR ($mirror)\n" + continue + fi + mkdir -p "$XBPS_MASTERDIR/$path" + XBPS_CHROOT_CMD_ARGS+=" -b $path:$path" + else + msg_warn "File URLs ($mirror) don't work with '$XBPS_CHROOT_CMD'\n" + fi + done +} + readonly XBPS_VERSION_REQ="0.46" readonly XBPS_VERSION=$(xbps-uhelper -V|awk '{print $2}') readonly XBPS_SRC_VERSION="113" @@ -516,7 +542,7 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \ XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \ XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \ XBPS_SRC_VERSION XBPS_DESTDIR XBPS_MACHINE XBPS_TEMP_MASTERDIR \ - XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR + XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR XBPS_DISTFILES_MIRROR for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do eval val="\$XBPS_$i" @@ -571,6 +597,9 @@ check_build_requirements if [ -z "$IN_CHROOT" ]; then trap 'exit_func' INT TERM + if [ -n "$XBPS_DISTFILES_MIRROR" ]; then + setup_distfiles_mirror + fi fi #