[Asterisk-code-review] pjproject bundled: Improve reliability of pjproject download (asterisk[14])
Anonymous Coward
asteriskteam at digium.com
Mon Nov 21 06:22:09 CST 2016
Anonymous Coward #1000019 has submitted this change and it was merged. ( https://gerrit.asterisk.org/4458 )
Change subject: pjproject_bundled: Improve reliability of pjproject download
......................................................................
pjproject_bundled: Improve reliability of pjproject download
The download process now has a timeout which will cause wget to retry
if it stops retrieving data for 5 seconds and fetch and curl to timeout
if the whole retrieval take smore than 30 seconds.
If the tarball retrieval works, the MD5SUM file is retrieved from
the downloads site and the md5 checksum is verified.
If either the tarball retrieval or MD5SUM retrieval fails, or the
checksums don't match, the entire process is retried once. If it
fails again, any incomplete tarball is deleted.
.DELETE_ON_ERROR: was also added to the Makefile. Not only does
this delete the tarball on failure, it till also delete corrupted
library files from the pjproject source directory should they
fail to build correctly.
Tested all the way back to FreeBSD 9, CentOS 6, Debian 6 and
Ubuntu 14.
Change-Id: Iea7d33b96a31622ab1b6e54baebaf271959514e1
---
M configure
M configure.ac
M makeopts.in
M third-party/Makefile.rules
M third-party/pjproject/Makefile
M third-party/pjproject/configure.m4
6 files changed, 137 insertions(+), 6 deletions(-)
Approvals:
Kevin Harwell: Looks good to me, but someone else must approve
Anonymous Coward #1000019: Verified
Joshua Colp: Looks good to me, approved
diff --git a/configure b/configure
index 4ab32c9..8a6de48 100755
--- a/configure
+++ b/configure
@@ -1223,6 +1223,7 @@
OPENSSL
SHA1SUM
LDCONFIG
+DOWNLOAD_TIMEOUT
DOWNLOAD_TO_STDOUT
DOWNLOAD
FETCH
@@ -1249,6 +1250,8 @@
FIND
PYTHON
FLEX
+CUT
+CAT
CMP
BISON
GNU_LD
@@ -6792,6 +6795,88 @@
fi
+# Extract the first word of "cat", so it can be a program name with args.
+set dummy cat; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_CAT+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $CAT in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_CAT="$CAT" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_CAT="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_path_CAT" && ac_cv_path_CAT=":"
+ ;;
+esac
+fi
+CAT=$ac_cv_path_CAT
+if test -n "$CAT"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CAT" >&5
+$as_echo "$CAT" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+# Extract the first word of "cut", so it can be a program name with args.
+set dummy cut; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_CUT+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $CUT in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_CUT="$CUT" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_path_CUT="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_path_CUT" && ac_cv_path_CUT=":"
+ ;;
+esac
+fi
+CUT=$ac_cv_path_CUT
+if test -n "$CUT"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CUT" >&5
+$as_echo "$CUT" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
# Extract the first word of "flex", so it can be a program name with args.
set dummy flex; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -7862,9 +7947,11 @@
if test "${WGET}" != ":" ; then
DOWNLOAD=${WGET}
DOWNLOAD_TO_STDOUT="${WGET} -O-"
+ DOWNLOAD_TIMEOUT='--timeout=$1'
else if test "${CURL}" != ":" ; then
DOWNLOAD="${CURL} -O --progress-bar -w \"%{url_effective}\n\""
DOWNLOAD_TO_STDOUT="${CURL} -L --progress-bar -w \"%{url_effective}\n\""
+ DOWNLOAD_TIMEOUT='--max-time $(or $2,$1)'
else
# Extract the first word of "fetch", so it can be a program name with args.
set dummy fetch; ac_word=$2
@@ -7909,8 +7996,11 @@
DOWNLOAD=${FETCH}
DOWNLOAD_TO_STDOUT="${FETCH} -o-"
+ DOWNLOAD_TIMEOUT='--timeout=$(or $2,$1)'
fi
fi
+
+
@@ -9235,8 +9325,14 @@
if test "${NM}" = ":" ; then
as_fn_error $? "nm is required to build bundled pjproject" "$LINENO" 5
fi
+ if test "${MD5}" = ":" ; then
+ as_fn_error $? "md5dum is required to build bundled pjproject" "$LINENO" 5
+ fi
+ if test "${CAT}" = ":" ; then
+ as_fn_error $? "cat is required to build bundled pjproject" "$LINENO" 5
+ fi
- export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT
+ export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT DOWNLOAD_TIMEOUT DOWNLOAD MD5 CAT
${GNU_MAKE} --quiet --no-print-directory -C ${PJPROJECT_DIR} EXTERNALS_CACHE_DIR=${EXTERNALS_CACHE_DIR} configure
if test $? -ne 0 ; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: failed" >&5
diff --git a/configure.ac b/configure.ac
index 66c8971..0016f68 100644
--- a/configure.ac
+++ b/configure.ac
@@ -263,6 +263,7 @@
AC_PATH_PROG([BISON], [bison], :)
AC_PATH_PROG([CMP], [cmp], :)
+AC_PATH_PROG([CAT], [cat], :)
AC_PATH_PROG([FLEX], [flex], :)
AC_PATH_PROG([GREP], [grep], :)
AC_PATH_PROG([PYTHON], [python], :)
@@ -293,18 +294,23 @@
if test "${WGET}" != ":" ; then
DOWNLOAD=${WGET}
DOWNLOAD_TO_STDOUT="${WGET} -O-"
+ DOWNLOAD_TIMEOUT='--timeout=$1'
else if test "${CURL}" != ":" ; then
DOWNLOAD="${CURL} -O --progress-bar -w \"%{url_effective}\n\""
DOWNLOAD_TO_STDOUT="${CURL} -L --progress-bar -w \"%{url_effective}\n\""
+ DOWNLOAD_TIMEOUT='--max-time $(or $2,$1)'
else
AC_PATH_PROG([FETCH], [fetch], [:])
DOWNLOAD=${FETCH}
DOWNLOAD_TO_STDOUT="${FETCH} -o-"
+ DOWNLOAD_TIMEOUT='--timeout=$(or $2,$1)'
fi
fi
AC_SUBST(DOWNLOAD)
AC_SUBST(DOWNLOAD_TO_STDOUT)
+AC_SUBST(DOWNLOAD_TIMEOUT)
+
AC_PATH_PROG([LDCONFIG], [ldconfig], :)
AC_PATH_PROG([SHA1SUM], [sha1sum], $ac_aux_dir/build_tools/sha1sum-sh)
AC_PATH_PROG([OPENSSL], [openssl], :)
diff --git a/makeopts.in b/makeopts.in
index 9ff40d7..cf2f790 100644
--- a/makeopts.in
+++ b/makeopts.in
@@ -28,6 +28,7 @@
FETCH=@FETCH@
DOWNLOAD=@DOWNLOAD@
DOWNLOAD_TO_STDOUT=@DOWNLOAD_TO_STDOUT@
+DOWNLOAD_MAX_TIMEOUT=@DOWNLOAD_MAX_TIMEOUT@
SOUNDS_CACHE_DIR=@SOUNDS_CACHE_DIR@
EXTERNALS_CACHE_DIR=@EXTERNALS_CACHE_DIR@
RUBBER=@RUBBER@
@@ -46,6 +47,7 @@
PATCH=@PATCH@
SED=@SED@
NM=@NM@
+CAT=@CAT@
BUILD_PLATFORM=@BUILD_PLATFORM@
BUILD_CPU=@BUILD_CPU@
diff --git a/third-party/Makefile.rules b/third-party/Makefile.rules
index 4f804dd..f8b72ba 100644
--- a/third-party/Makefile.rules
+++ b/third-party/Makefile.rules
@@ -26,4 +26,8 @@
export PATCH
export SED
export NM
+export MD5
+export CAT
export DOWNLOAD
+export DOWNLOAD_TO_STDOUT
+export DOWNLOAD_TIMEOUT
diff --git a/third-party/pjproject/Makefile b/third-party/pjproject/Makefile
index 106938b..ff3266c 100644
--- a/third-party/pjproject/Makefile
+++ b/third-party/pjproject/Makefile
@@ -75,14 +75,31 @@
include Makefile.rules
ECHO_PREFIX := $(ECHO_PREFIX) echo '[pjproject] '
+SHELL_ECHO_PREFIX := echo '[pjproject] '
_all: $(TARGETS)
-DOWNLOAD_DIR := $(or $(EXTERNALS_CACHE_DIR),$(TMPDIR),$(wildcard /tmp),.)
+define download_from_pjproject
+ ($(SHELL_ECHO_PREFIX) Downloading $(PJPROJECT_URL)/$(@F) to $@ ;\
+ $(DOWNLOAD_TO_STDOUT) $(call DOWNLOAD_TIMEOUT,5,10) $(PJPROJECT_URL)/$(@F) > $@ &&\
+ $(SHELL_ECHO_PREFIX) Downloading $(PJPROJECT_URL)/MD5SUM to $(PJMD5SUM) &&\
+ $(DOWNLOAD_TO_STDOUT) $(call DOWNLOAD_TIMEOUT,5,10) $(PJPROJECT_URL)/MD5SUM.TXT > $(PJMD5SUM) &&\
+ ($(SHELL_ECHO_PREFIX) Verifying $@ &&\
+ tarball_sum=$$($(CAT) $@ | $(MD5) | $(SED) -n -r -e "s/^([^ ]+)\s+.*/\1/gp") ;\
+ required_sum=$$($(SED) -n -r -e "s/^([^ ]+)\s+$(@F)/\1/gp" $(PJMD5SUM)) ;\
+ if [ "$$tarball_sum" != "$$required_sum" ] ; then $(SHELL_ECHO_PREFIX) Verify failed ; exit 1 ; fi) &&\
+ $(SHELL_ECHO_PREFIX) Verify successful ; exit 0)
+endef
-$(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2: ../versions.mak
- $(ECHO_PREFIX) Downloading $(PJPROJECT_URL)/$(@F) to $@
- $(CMD_PREFIX) $(DOWNLOAD_TO_STDOUT) $(PJPROJECT_URL)/$(@F) > $@
+.DELETE_ON_ERROR:
+
+DOWNLOAD_DIR := $(or $(EXTERNALS_CACHE_DIR),$(TMPDIR),$(wildcard /tmp),.)
+TARBALL = $(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2
+PJMD5SUM = $(patsubst %.tar.bz2,%.md5,$(TARBALL))
+
+$(TARBALL): ../versions.mak
+ $(CMD_PREFIX) $(download_from_pjproject) || (rm -rf $@ ;\
+ $(SHELL_ECHO_PREFIX) Retrying download ; sleep 3 ; $(download_from_pjproject))
source/.unpacked: $(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2
$(ECHO_PREFIX) Unpacking $<
diff --git a/third-party/pjproject/configure.m4 b/third-party/pjproject/configure.m4
index 7c60c2a..8294d8e 100644
--- a/third-party/pjproject/configure.m4
+++ b/third-party/pjproject/configure.m4
@@ -28,8 +28,14 @@
if test "${NM}" = ":" ; then
AC_MSG_ERROR(nm is required to build bundled pjproject)
fi
+ if test "${MD5}" = ":" ; then
+ AC_MSG_ERROR(md5sum is required to build bundled pjproject)
+ fi
+ if test "${CAT}" = ":" ; then
+ AC_MSG_ERROR(cat is required to build bundled pjproject)
+ fi
- export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT
+ export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT DOWNLOAD_TIMEOUT DOWNLOAD MD5 CAT
${GNU_MAKE} --quiet --no-print-directory -C ${PJPROJECT_DIR} EXTERNALS_CACHE_DIR=${EXTERNALS_CACHE_DIR} configure
if test $? -ne 0 ; then
AC_MSG_RESULT(failed)
--
To view, visit https://gerrit.asterisk.org/4458
To unsubscribe, visit https://gerrit.asterisk.org/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iea7d33b96a31622ab1b6e54baebaf271959514e1
Gerrit-PatchSet: 2
Gerrit-Project: asterisk
Gerrit-Branch: 14
Gerrit-Owner: George Joseph <gjoseph at digium.com>
Gerrit-Reviewer: Anonymous Coward #1000019
Gerrit-Reviewer: Joshua Colp <jcolp at digium.com>
Gerrit-Reviewer: Kevin Harwell <kharwell at digium.com>
Gerrit-Reviewer: Mark Michelson <mmichelson at digium.com>
More information about the asterisk-code-review
mailing list