From 500a3c8f491e77cabfb9e5db0a00706eb0600ad4 Mon Sep 17 00:00:00 2001 From: Duncaen Date: Sat, 6 Aug 2022 04:54:05 +0200 Subject: [PATCH] xbps-src: clean up fetch hook and allow mirroring --- common/environment/build-style/R-cran.sh | 3 +- common/hooks/do-fetch/00-distfiles.sh | 201 ++++++++++++----------- srcpkgs/R-cran-pillar/template | 3 +- 3 files changed, 110 insertions(+), 97 deletions(-) diff --git a/common/environment/build-style/R-cran.sh b/common/environment/build-style/R-cran.sh index ded66793396..905c3b9dc0c 100644 --- a/common/environment/build-style/R-cran.sh +++ b/common/environment/build-style/R-cran.sh @@ -4,5 +4,6 @@ wrksrc="${XBPS_BUILDDIR}/${pkgname#R-cran-}" # default to cran if [ -z "$distfiles" ]; then - distfiles="https://cran.r-project.org/src/contrib/${pkgname#R-cran-}_${version//r/-}.tar.gz" + distfiles="https://cran.r-project.org/src/contrib/${pkgname#R-cran-}_${version//r/-}.tar.gz + https://cran.r-project.org/src/contrib/Archive/${pkgname#R-cran-}/${pkgname#R-cran-}_${version//r/-}.tar.gz" fi diff --git a/common/hooks/do-fetch/00-distfiles.sh b/common/hooks/do-fetch/00-distfiles.sh index e2bf54378fe..5fa394c9c79 100644 --- a/common/hooks/do-fetch/00-distfiles.sh +++ b/common/hooks/do-fetch/00-distfiles.sh @@ -2,24 +2,6 @@ # the $distfiles variable and then verifies its sha256 checksum comparing # its value with the one stored in the $checksum variable. -# Get the checksum for $curfile at index $dfcount -get_cksum() { - local curfile="$1" dfcount="$2" ckcount cksum i - - ckcount=0 - cksum=0 - for i in ${checksum}; do - if [ $dfcount -eq $ckcount -a -n "$i" ]; then - cksum=$i - fi - ckcount=$((ckcount + 1)) - done - if [ -z "$cksum" ]; then - msg_error "$pkgver: cannot find checksum for $curfile.\n" - fi - echo "$cksum" -} - # Return the checksum of the contents of a tarball contents_cksum() { local curfile="$1" cursufx cksum @@ -110,9 +92,7 @@ contents_cksum() { # Verify the checksum for $curfile stored at $distfile and index $dfcount verify_cksum() { - local curfile="$1" distfile="$2" dfcount="$3" filesum cksum - - cksum=$(get_cksum $curfile $dfcount) + local curfile="$1" distfile="$2" cksum="$3" filesum # If the checksum starts with an commercial at (@) it is the contents checksum if [ "${cksum:0:1}" = "@" ]; then @@ -121,7 +101,7 @@ verify_cksum() { filesum=$(contents_cksum "$curfile") if [ "${cksum}" != "$filesum" ]; then echo - msg_red "SHA256 mismatch for '$curfile:'\n@$filesum\n" + msg_red "SHA256 mismatch for '${curfile}:'\n@${filesum}\n" errors=$((errors + 1)) else msg_normal_append "OK.\n" @@ -131,7 +111,7 @@ verify_cksum() { filesum=$(${XBPS_DIGEST_CMD} "$distfile") if [ "$cksum" != "$filesum" ]; then echo - msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n" + msg_red "SHA256 mismatch for '${curfile}:'\n${filesum}\n" errors=$((errors + 1)) else if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then @@ -145,22 +125,20 @@ verify_cksum() { # Link an existing cksum $distfile for $curfile at index $dfcount link_cksum() { - local curfile="$1" distfile="$2" dfcount="$3" filesum cksum - - cksum=$(get_cksum $curfile $dfcount) - + local curfile="$1" distfile="$2" cksum="$3" if [ -n "$cksum" -a -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then ln -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" "$distfile" msg_normal "$pkgver: using known distfile $curfile.\n" + return 0 fi + return 1 } try_mirrors() { - local curfile="$1" distfile="$2" dfcount="$3" subdir="$4" f="$5" - local filesum cksum basefile mirror path scheme - [ -z "$XBPS_DISTFILES_MIRROR" ] && return + local curfile="$1" distfile="$2" cksum="$3" f="$4" + local filesum basefile mirror path scheme good + [ -z "$XBPS_DISTFILES_MIRROR" ] && return 1 basefile="${f##*/}" - cksum=$(get_cksum $curfile $dfcount) for mirror in $XBPS_DISTFILES_MIRROR; do scheme="file" if [[ $mirror == *://* ]]; then @@ -179,28 +157,80 @@ try_mirrors() { fi if [[ "$mirror" == *voidlinux* ]]; then # For distfiles.voidlinux.* append the subdirectory - mirror="$mirror/$subdir" + mirror="$mirror/$pkgname-$version" fi - msg_normal "$pkgver: fetching distfile '$curfile' from '$mirror'...\n" + msg_normal "$pkgver: fetching distfile '$curfile' from mirror '$mirror'...\n" $fetch_cmd "$mirror/$curfile" # If basefile was not found, but a curfile file may exist, try to fetch it - if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then - $fetch_cmd "$mirror/$basefile" - fi + # if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then + # msg_normal "$pkgver: fetching distfile '$basefile' from mirror '$mirror'...\n" + # $fetch_cmd "$mirror/$basefile" + # fi [ ! -f "$distfile" ] && continue flock -n ${distfile}.part rm -f ${distfile}.part filesum=$(${XBPS_DIGEST_CMD} "$distfile") - [ "$cksum" == "$filesum" ] && break + if [ "$cksum" == "$filesum" ]; then + return 0 + fi msg_normal "$pkgver: checksum failed - removing '$curfile'...\n" rm -f ${distfile} done + return 1 +} + +try_urls() { + local curfile="$1" + local good= + for i in ${_file_idxs["$curfile"]}; do + local cksum=${_checksums["$i"]} + local url=${_distfiles["$i"]} + + # If distfile does not exist, download it from the original location. + if [[ "$FTP_RETRIES" && "${url}" =~ ^ftp:// ]]; then + max_retries="$FTP_RETRIES" + else + max_retries=1 + fi + for retry in $(seq 1 1 $max_retries); do + if [ ! -f "$distfile" ]; then + if [ "$retry" == 1 ]; then + msg_normal "$pkgver: fetching distfile '$curfile' from '$url'...\n" + else + msg_normal "$pkgver: fetch attempt $retry of $max_retries...\n" + fi + flock "${distfile}.part" $fetch_cmd "$url" + fi + done + + if [ ! -f "$distfile" ]; then + continue + fi + + # distfile downloaded, verify sha256 hash. + flock -n "${distfile}.part" rm -f "${distfile}.part" + verify_cksum "$f" "$distfile" "$cksum" + return 0 + done + return 1 } hook() { local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version" local dfcount=0 dfgood=0 errors=0 max_retries - if [ ! -d "$srcdir" ]; then + local -a _distfiles=($distfiles) + local -a _checksums=($checksum) + local -A _file_idxs + + # Create a map from target file to index in _distfiles/_checksums + for i in ${!_distfiles[@]}; do + f="${_distfiles[$i]}" + curfile="${f#*>}" + curfile="${curfile##*/}" + _file_idxs["$curfile"]+=" $i" + done + + if [[ ! -d "$srcdir" ]]; then mkdir -p -m775 "$srcdir" chgrp $(id -g) "$srcdir" fi @@ -212,87 +242,68 @@ hook() { # Detect bsdtar and GNU tar (in that order of preference) TAR_CMD="$(command -v bsdtar)" - if [ -z "$TAR_CMD" ]; then + if [[ -z "$TAR_CMD" ]]; then TAR_CMD="$(command -v tar)" fi # Detect distfiles with obsolete checksum and purge them from the cache - for f in ${distfiles}; do - curfile="${f#*>}" - curfile="${curfile##*/}" - distfile="$srcdir/$curfile" - - if [ -f "$distfile" ]; then - cksum=$(get_cksum $curfile $dfcount) - if [ "${cksum:0:1}" = "@" ]; then - cksum=${cksum:1} - filesum=$(contents_cksum "$distfile") - else - filesum=$(${XBPS_DIGEST_CMD} "$distfile") + for f in ${!_file_idxs[@]}; do + distfile="$srcdir/$f" + for i in ${_file_idxs["$f"]}; do + if [[ -f $distfile ]]; then + cksum=${_checksums["$i"]} + if [[ ${cksum:0:1} = @ ]]; then + cksum=${cksum:1} + filesum=$(contents_cksum "$distfile") + else + filesum=$(${XBPS_DIGEST_CMD} "$distfile") + fi + if [[ $cksum = $filesum ]]; then + dfgood=$((dfgood + 1)) + else + inode=$(stat "$distfile" --printf "%i") + msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n" + find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print + fi fi - if [ "$cksum" = "$filesum" ]; then - dfgood=$((dfgood + 1)) - else - inode=$(stat "$distfile" --printf "%i") - msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n" - find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print - fi - fi - dfcount=$((dfcount + 1)) + dfcount=$((dfcount + 1)) + done done # We're done, if all distfiles were found and had good checksums - [ $dfcount -eq $dfgood ] && return + [[ $dfcount -eq $dfgood ]] && return # Download missing distfiles and verify their checksums - dfcount=0 - for f in ${distfiles}; do - curfile="${f#*>}" - curfile="${curfile##*/}" + for curfile in ${!_file_idxs[@]}; do distfile="$srcdir/$curfile" # If file lock cannot be acquired wait until it's available. - while true; do - flock -w 1 ${distfile}.part true - [ $? -eq 0 ] && break + while ! flock -w 1 "${distfile}.part" true; do msg_warn "$pkgver: ${curfile} is already being downloaded, waiting for 1s ...\n" done + + if [[ -f "$distfile" ]]; then + continue + fi + # If distfile does not exist, try to link to it. - if [ ! -f "$distfile" ]; then - link_cksum $curfile $distfile $dfcount + if link_cksum "$curfile" "$distfile" "${_checksums[0]}"; then + continue fi + # If distfile does not exist, download it from a mirror location. - if [ ! -f "$distfile" ]; then - try_mirrors $curfile $distfile $dfcount $pkgname-$version $f + if try_mirrors "$curfile" "$distfile" "${_checksums[0]}" "${_distfiles[0]}"; then + continue fi - # If distfile does not exist, download it from the original location. - if [[ "$FTP_RETRIES" && "${f}" =~ ^ftp:// ]]; then - max_retries="$FTP_RETRIES" - else - max_retries=1 + + if ! try_urls "$curfile"; then + msg_error "$pkgver: failed to fetch '$curfile'.\n" fi - for retry in $(seq 1 1 $max_retries); do - if [ ! -f "$distfile" ]; then - if [ "$retry" == 1 ]; then - msg_normal "$pkgver: fetching distfile '$curfile'...\n" - else - msg_normal "$pkgver: fetch attempt $retry of $max_retries...\n" - fi - flock "${distfile}.part" $fetch_cmd "$f" - fi - done - if [ ! -f "$distfile" ]; then - msg_error "$pkgver: failed to fetch $curfile.\n" - fi - # distfile downloaded, verify sha256 hash. - flock -n ${distfile}.part rm -f ${distfile}.part - verify_cksum $curfile $distfile $dfcount - dfcount=$((dfcount + 1)) done unset TAR_CMD - if [ $errors -gt 0 ]; then + if [[ $errors -gt 0 ]]; then msg_error "$pkgver: couldn't verify distfiles, exiting...\n" fi } diff --git a/srcpkgs/R-cran-pillar/template b/srcpkgs/R-cran-pillar/template index e041a7a7c4b..d8d381b8e4a 100644 --- a/srcpkgs/R-cran-pillar/template +++ b/srcpkgs/R-cran-pillar/template @@ -11,4 +11,5 @@ short_desc="Coloured Formatting for Columns" maintainer="Florian Wagner " license="GPL-3.0-or-later" homepage="https://github.com/r-lib/pillar/" -checksum=0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672 +checksum="0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672 + 0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672"