[Asterisk-code-review] build: Add download capability for external packages (asterisk[14])
George Joseph
asteriskteam at digium.com
Wed Aug 31 14:12:13 CDT 2016
George Joseph has uploaded a new change for review.
https://gerrit.asterisk.org/3768
Change subject: build: Add download capability for external packages
......................................................................
build: Add download capability for external packages
The DPMA and g729a, silk, siren7 and siren14 codecs hosted at
http://downloads.digium.com/pub/telephony/ are now listed in the
"External" sections of the "Resource Modules" and "Codec Translators"
pages in menuselect. Any that are selected will automatically be
downloaded and installed when "make install" is run. Their LICENSE and
README (if avaialble) files will be installed to
ASTVARLIBDIR/documentation/thirdparty/<product_name>.
Example use with codecs:
The codecs/codecs.xml file is a menuselect style xml file that lists
the codecs to be included. Their support levels are 'external', which
triggers the download and install, and defaultenabled is no. Also
because codec_g729a is actually in a directory named codec_g729 on the
download server, the newly added 'member_data' element is used to
override the default of the directory name being the codec name.
A new configure option '--with-externals-cache' was added and like
'--with-sounds-cache' it allows the installer to cache tarballs so
they're not downloaded every time.
To assist with the download and install process, each external package
now has a manifest.xml file that, among other things, contains a package
version and checksums for each file in the tarball. The manifest is
saved to both the cache directory and ASTMODDIR and together with the
manifest.xml on the downloads site, tells the install scripts whether
a download and/or update is needed.
xmlstarlet is required for downloader operation. If it's not installed
the external items in menuselect will be unavailable.
Change-Id: Id3dcf1289ffd3cb0bbd7dfab3cafbb87be60323a
---
M Makefile
A build_tools/download_externals
A build_tools/list_valid_installed_externals
M build_tools/menuselect-deps.in
M codecs/Makefile
A codecs/codecs.xml
M configure
M configure.ac
M makeopts.in
M res/Makefile
A res/res.xml
11 files changed, 327 insertions(+), 3 deletions(-)
git pull ssh://gerrit.asterisk.org:29418/asterisk refs/changes/68/3768/1
diff --git a/Makefile b/Makefile
index bf8f0f8..992e1d3 100644
--- a/Makefile
+++ b/Makefile
@@ -618,9 +618,10 @@
NEWMODS:=$(foreach d,$(MOD_SUBDIRS),$(notdir $(wildcard $(d)/*.so)))
OLDMODS=$(filter-out $(NEWMODS) $(notdir $(DESTDIR)$(ASTMODDIR)),$(notdir $(wildcard $(DESTDIR)$(ASTMODDIR)/*.so)))
+BADMODS=$(strip $(filter-out $(shell ./build_tools/list_valid_installed_externals),$(OLDMODS)))
oldmodcheck:
- @if [ -n "$(OLDMODS)" ]; then \
+ @if [ -n "$(BADMODS)" ]; then \
echo " WARNING WARNING WARNING" ;\
echo "" ;\
echo " Your Asterisk modules directory, located at" ;\
@@ -630,7 +631,7 @@
echo " modules are compatible with this version before" ;\
echo " attempting to run Asterisk." ;\
echo "" ;\
- for f in $(OLDMODS); do \
+ for f in $(BADMODS); do \
echo " $$f" ;\
done ;\
echo "" ;\
@@ -980,7 +981,7 @@
menuselect/makeopts: makeopts .lastclean
+$(MAKE_MENUSELECT) makeopts
-menuselect-tree: $(foreach dir,$(filter-out main,$(MOD_SUBDIRS)),$(wildcard $(dir)/*.c) $(wildcard $(dir)/*.cc)) build_tools/cflags.xml build_tools/cflags-devmode.xml sounds/sounds.xml build_tools/embed_modules.xml utils/utils.xml agi/agi.xml configure makeopts
+menuselect-tree: $(foreach dir,$(filter-out main,$(MOD_SUBDIRS)),$(wildcard $(dir)/*.c) $(wildcard $(dir)/*.cc) $(wildcard $(dir)/*.xml)) build_tools/cflags.xml build_tools/cflags-devmode.xml sounds/sounds.xml build_tools/embed_modules.xml utils/utils.xml agi/agi.xml configure makeopts
@echo "Generating input for menuselect ..."
@echo "<?xml version=\"1.0\"?>" > $@
@echo >> $@
diff --git a/build_tools/download_externals b/build_tools/download_externals
new file mode 100755
index 0000000..7503b67
--- /dev/null
+++ b/build_tools/download_externals
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+
+if [[ ( ${BASH_VERSINFO[0]} == 4 && ${BASH_VERSINFO[1]} > 1 ) || ${BASH_VERSINFO[0]} > 4 ]] ; then
+ shopt -s compat41
+fi
+set -e
+
+script_path=$(readlink -f ${0})
+script_dir=$(dirname ${script_path})
+ASTTOPDIR=$(readlink -f ${script_dir}/.. )
+
+module_name=$1
+
+if [[ -z ${module_name} ]] ; then
+ echo "You must supply a module name."
+ exit 64
+fi
+
+tmpdir=$(mktemp -d)
+if [[ -z "${tmpdir}" ]] ; then
+ echo "${module_name}: Unable to create temporary directory."
+ exit 1
+fi
+trap "rm -rf ${tmpdir}" EXIT
+
+sed -r -e "s/^([^ =]+)\s*=\s*(.*)$/\1=\"\2\"/g" ${ASTTOPDIR}/makeopts >${tmpdir}/makeopts
+source ${tmpdir}/makeopts
+if [[ -z "${ASTMODDIR}" ]] ; then
+ echo "${module_name}: Unable to parse ${ASTTOPDIR}/makeopts."
+ exit 1
+fi
+
+XMLSTARLET=${XMLSTARLET:-xmlstarlet}
+if [[ "${XMLSTARLET}" = ":" ]] ; then
+ echo "${module_name}: The externals downloader requires xmlstarlet to be installed."
+ exit 1
+fi
+
+cache_dir="${EXTERNALS_CACHE_DIR}"
+if [[ -z ${cache_dir} ]] ; then
+ cache_dir=${tmpdir}
+fi
+
+version=$(${ASTTOPDIR}/build_tools/make_version ${ASTTOPDIR})
+if [[ ! ${version} =~ ^(GIT-)?([^.-]+)[.-].* ]] ; then
+ echo "${module_name}: Couldn't parse version ${version}"
+ exit 1
+fi
+major_version=${BASH_REMATCH[2]}
+
+if [[ "${major_version}" == "master" ]] ; then
+ echo "${module_name}: External module downloading is not available in the 'master' git branch. Please disable in menuselect and download manually."
+ exit 1
+fi
+
+major_version=${major_version}.0
+
+if [[ "${HOST_CPU}" = "x86_64" ]] ; then
+ host_bits=64
+else
+ host_bits=32
+fi
+
+directory_name=$(${XMLSTARLET} sel -t -v "/menu/category/member[@name = '${module_name}']/member_data/downloader/@directory_name" ${ASTTOPDIR}/menuselect-tree || :)
+remote_url="http://downloads.digium.com/pub/telephony"
+remote_dir=${remote_url}/${directory_name:-${module_name}}/asterisk-${major_version}/x86-${host_bits}
+
+version_convert() {
+ local v=${1##*_}
+ if [[ ${v} =~ ([0-9]+)[.]([0-9]+)[.]([0-9]+) ]] ; then
+ v=$(( ${BASH_REMATCH[1]}<<18 | ${BASH_REMATCH[2]}<<9 | ${BASH_REMATCH[3]} ))
+ fi
+ echo ${v}
+}
+
+${WGET} -q -O ${tmpdir}/manifest.xml ${remote_dir}/manifest.xml || {
+ echo "${module_name}: Unable to fetch ${remote_dir}/manifest.xml"
+ exit 1
+}
+
+rpv=$(${XMLSTARLET} sel -t -v "/package/@version" ${tmpdir}/manifest.xml)
+rpvi=$(version_convert ${rpv})
+echo "${module_name}: Remote package version ${rpv} (${rpvi})"
+
+module_dir=${module_name}-${rpv}-x86_${host_bits}
+tarball=${module_dir}.tar.gz
+export need_install=0
+
+if [[ -f ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml ]] ; then
+ package_arch=$(${XMLSTARLET} sel -t -v "/package/@arch" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+ ipv=$(${XMLSTARLET} sel -t -v "/package/@version" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+ ipvi=$(version_convert ${ipv})
+ ip_major=${ipv%_*}
+ echo "${module_name}: Installed package version ${ipv} (${ipvi})"
+ if [[ "${ip_major}" != "${major_version}" || "${package_arch}" != "x86_${host_bits}" ]] ; then
+ echo "${module_name}: The installed package is not for this version of Asterisk. Reinstalling."
+ need_install=1
+ elif [[ ${rpvi} > ${ipvi} ]] ; then
+ echo "${module_name}: A newer package is available"
+ need_install=1
+ else
+ sums=$(${XMLSTARLET} sel -t -m "//file" -v "@md5sum" -n ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+ for sum in ${sums} ; do
+ install_path=$(${XMLSTARLET} sel -t -v "//file[@md5sum = '${sum}']/@install_path" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml )
+ f=${DESTDIR}$(eval echo ${install_path})
+ if [[ ! -f ${f} ]] ; then
+ echo Not found: ${f}
+ need_install=1
+ else
+ cs=$(md5sum ${f} | cut -b1-32)
+ if [[ "${cs}" != "${sum}" ]] ; then
+ echo Checksum mismatch: ${f}
+ need_install=1
+ fi
+ fi
+ done
+ fi
+else
+ need_install=1
+fi
+
+if [[ ${need_install} == 1 ]] ; then
+ if [[ ( -n "${ipvi}" ) && ${ipvi} > ${rpvi} ]] ; then
+ echo "${module_name}: Installed package is newer than that available for download."
+ exit 0
+ fi
+else
+ echo "${module_name} is up to date."
+ exit 0;
+fi
+
+need_download=1
+if [[ -f ${cache_dir}/${module_name}.manifest.xml ]] ; then
+ cpv=$(${XMLSTARLET} sel -t -v "/package/@version" ${cache_dir}/${module_name}.manifest.xml)
+ cpvi=$(version_convert ${cpv})
+ echo "${module_name}: Cached package version ${cpv} (${cpvi})"
+ if [[ ${cpvi} == ${rpvi} && ( -f ${cache_dir}/${tarball} ) ]] ; then
+ echo "${module_name}: Cached version is available."
+ need_download=0
+ fi
+fi
+
+if [[ ${need_download} = 1 ]] ; then
+ echo "${module_name}: Downloading ${remote_dir}/${tarball}"
+ ${WGET} -q -O ${cache_dir}/${tarball} ${remote_dir}/${tarball} || {
+ echo "${module_name}: Unable to fetch ${remote_dir}/${tarball}"
+ exit 1
+ }
+ cp ${tmpdir}/manifest.xml ${cache_dir}/${module_name}.manifest.xml
+fi
+
+tar -xzf ${cache_dir}/${tarball} -C ${cache_dir}
+trap "rm -rf ${cache_dir}/${module_dir}" EXIT
+
+echo "${module_name}: Installing."
+
+if [[ $EUID == 0 ]] ; then
+ install_params="--group=0 --owner=0"
+fi
+
+names=$(${XMLSTARLET} sel -t -m "//file" -v "@name" -n ${cache_dir}/${module_dir}/manifest.xml)
+for name in ${names} ; do
+ source_path=${cache_dir}/${module_dir}/${name}
+ install_path=$(${XMLSTARLET} sel -t -v "//file[@name = '${name}']/@install_path" ${cache_dir}/${module_dir}/manifest.xml)
+ install_path=${DESTDIR}$(eval echo ${install_path})
+ executable=$(${XMLSTARLET} sel -t -v "//file[@name = '${name}']/@executable" ${cache_dir}/${module_dir}/manifest.xml || :)
+ if [[ "${executable}" = "yes" ]] ; then
+ mode=0755
+ else
+ mode=0644
+ fi
+
+ ${INSTALL} -Dp ${install_params} --mode=${mode} ${source_path} ${install_path}
+
+done
+${INSTALL} -Dp ${install_params} --mode=0644 ${cache_dir}/${module_dir}/manifest.xml ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml
+
+echo "${module_name}: Installed."
diff --git a/build_tools/list_valid_installed_externals b/build_tools/list_valid_installed_externals
new file mode 100755
index 0000000..0cc07ac
--- /dev/null
+++ b/build_tools/list_valid_installed_externals
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+if [[ ( ${BASH_VERSINFO[0]} == 4 && ${BASH_VERSINFO[1]} > 1 ) || ${BASH_VERSINFO[0]} > 4 ]] ; then
+ shopt -s compat41
+fi
+set -e
+
+script_path=$(readlink -f ${0})
+script_dir=$(dirname ${script_path})
+ASTTOPDIR=$(readlink -f ${script_dir}/.. )
+
+tmpdir=$(mktemp -d)
+if [[ -z "${tmpdir}" ]] ; then
+ echo "${module_name}: Unable to create temporary directory."
+ exit 1
+fi
+trap "rm -rf ${tmpdir}" EXIT
+
+sed -r -e "s/^([^ =]+)\s*=\s*(.*)$/\1=\"\2\"/g" ${ASTTOPDIR}/makeopts >${tmpdir}/makeopts
+source ${tmpdir}/makeopts
+if [[ -z "${ASTMODDIR}" ]] ; then
+ echo "${module_name}: Unable to parse ${ASTTOPDIR}/makeopts."
+ exit 1
+fi
+
+XMLSTARLET=${XMLSTARLET:-xmlstarlet}
+if [[ "${XMLSTARLET}" = ":" ]] ; then
+ echo "${module_name}: The externals downloader requires xmlstarlet to be installed."
+ exit 1
+fi
+
+version=$(${ASTTOPDIR}/build_tools/make_version ${ASTTOPDIR})
+if [[ ! ${version} =~ ^(GIT-)?([^.-]+)[.-].* ]] ; then
+ echo "${module_name}: Couldn't parse version ${version}"
+ exit 1
+fi
+major_version=${BASH_REMATCH[2]}.0
+
+if [[ "${HOST_CPU}" = "x86_64" ]] ; then
+ host_bits=64
+else
+ host_bits=32
+fi
+
+names=""
+for manifest in ${DESTDIR}${ASTMODDIR}/*.manifest.xml ; do
+ if [ ! -f "$manifest" ] ; then
+ break
+ fi
+ package_version=$(${XMLSTARLET} sel -t -v "/package/@version" ${manifest})
+ package_major_version=${package_version%_*}
+ package_arch=$(${XMLSTARLET} sel -t -v "/package/@arch" ${manifest})
+ if [[ "$package_major_version" = "$major_version" && "${package_arch}" = "x86_${host_bits}" ]] ; then
+ names+=$(${XMLSTARLET} sel -t -m "//file[@executable = 'yes']" -v "concat(@name, ' ')" ${manifest})
+ fi
+done
+echo $names
diff --git a/build_tools/menuselect-deps.in b/build_tools/menuselect-deps.in
index f194482..a3afb56 100644
--- a/build_tools/menuselect-deps.in
+++ b/build_tools/menuselect-deps.in
@@ -30,6 +30,7 @@
LDAP=@PBX_LDAP@
LIBEDIT=@PBX_LIBEDIT@
LIBXML2=@PBX_LIBXML2@
+XMLSTARLET=@PBX_XMLSTARLET@
LTDL=@PBX_LTDL@
LUA=@PBX_LUA@
MISDN=@PBX_MISDN@
diff --git a/codecs/Makefile b/codecs/Makefile
index 684ccc6..881a7be 100644
--- a/codecs/Makefile
+++ b/codecs/Makefile
@@ -91,3 +91,10 @@
$(call MOD_ADD_C,codec_resample,speex/resample.c)
speex/resample.o: _ASTCFLAGS+=$(SPEEX_RESAMPLE_CFLAGS)
+
+ifneq ($(XMLSTARLET),:)
+EXTERNAL_MODS := $(shell $(XMLSTARLET) sel -t -v "/category/member[support_level = 'external']/@name" .moduleinfo)
+
+install::
+ $(foreach MODULE,$(filter-out $(MENUSELECT_CODECS),$(EXTERNAL_MODS)),$(ASTTOPDIR)/build_tools/download_externals $(MODULE) || exit 1;)
+endif
diff --git a/codecs/codecs.xml b/codecs/codecs.xml
new file mode 100644
index 0000000..3740775
--- /dev/null
+++ b/codecs/codecs.xml
@@ -0,0 +1,21 @@
+<member name="codec_silk" displayname="Download the SILK codec from Digium. See http://downloads.digium.com/pub/telephony/codec_silk/README.">
+ <support_level>external</support_level>
+ <depend>xmlstarlet</depend>
+ <defaultenabled>no</defaultenabled>
+</member>
+<member name="codec_siren7" displayname="Download the Siren7 codec from Digium. See http://downloads.digium.com/pub/telephony/codec_siren7/README.">
+ <support_level>external</support_level>
+ <depend>xmlstarlet</depend>
+ <defaultenabled>no</defaultenabled>
+</member>
+<member name="codec_siren14" displayname="Download the Siren14 codec from Digium. See http://downloads.digium.com/pub/telephony/codec_siren14/README.">
+ <support_level>external</support_level>
+ <depend>xmlstarlet</depend>
+ <defaultenabled>no</defaultenabled>
+</member>
+<member name="codec_g729a" displayname="Download the g729a codec from Digium. A license must be purchased for this codec. See http://downloads.digium.com/pub/telephony/codec_g729/README.">
+ <support_level>external</support_level>
+ <depend>xmlstarlet</depend>
+ <defaultenabled>no</defaultenabled>
+ <member_data><downloader directory_name="codec_g729"/></member_data>
+</member>
diff --git a/configure b/configure
index 8999e02..b967199 100755
--- a/configure
+++ b/configure
@@ -822,6 +822,7 @@
SPANDSP_DIR
SPANDSP_INCLUDE
SPANDSP_LIB
+EXTERNALS_CACHE_DIR
SOUNDS_CACHE_DIR
PBX_SDL_IMAGE
SDL_IMAGE_DIR
@@ -1198,6 +1199,7 @@
ax_pthread_config
MD5
SOXMIX
+PBX_XMLSTARLET
PBX_FLEX
PBX_BISON
OPENSSL
@@ -1393,6 +1395,7 @@
with_sdl
with_SDL_image
with_sounds_cache
+with_externals_cache
with_spandsp
with_ss7
with_speex
@@ -2138,6 +2141,8 @@
--with-SDL_image=PATH use Sdl Image files in PATH
--with-sounds-cache=PATH
use cached sound tarfiles in PATH
+ --with-externals-cache=PATH
+ use cached external module tarfiles in PATH
--with-spandsp=PATH use SPANDSP files in PATH
--with-ss7=PATH use ISDN SS7 files in PATH
--with-speex=PATH use Speex files in PATH
@@ -7790,6 +7795,14 @@
fi
+if test "x${XMLSTARLET}" = "x:" ; then
+ PBX_XMLSTARLET=0
+else
+ PBX_XMLSTARLET=1
+fi
+
+
+
if test -n "$ac_tool_prefix"; then
# Extract the first word of "${ac_tool_prefix}soxmix", so it can be a program name with args.
set dummy ${ac_tool_prefix}soxmix; ac_word=$2
@@ -11488,6 +11501,30 @@
+
+# Check whether --with-externals-cache was given.
+if test "${with_externals_cache+set}" = set; then :
+ withval=$with_externals_cache;
+ case ${withval} in
+ n|no)
+ unset EXTERNALS_CACHE_DIR
+ ;;
+ *)
+ if test "x${withval}" = "x"; then
+ :
+ else
+ EXTERNALS_CACHE_DIR="${withval}"
+ fi
+ ;;
+ esac
+
+else
+ :
+fi
+
+
+
+
SPANDSP_DESCRIP="SPANDSP"
SPANDSP_OPTION="spandsp"
PBX_SPANDSP=0
diff --git a/configure.ac b/configure.ac
index ba13bf4..031439f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -340,6 +340,14 @@
fi
AC_SUBST(PBX_FLEX)
+if test "x${XMLSTARLET}" = "x:" ; then
+ PBX_XMLSTARLET=0
+else
+ PBX_XMLSTARLET=1
+fi
+AC_SUBST(PBX_XMLSTARLET)
+
+
AC_CHECK_TOOL([SOXMIX], [soxmix], [:])
if test "${SOXMIX}" != ":" ; then
AC_DEFINE([HAVE_SOXMIX], 1, [Define to 1 if your system has soxmix application.])
@@ -520,6 +528,7 @@
AST_EXT_LIB_SETUP([SDL], [Sdl], [sdl])
AST_EXT_LIB_SETUP([SDL_IMAGE], [Sdl Image], [SDL_image])
AST_OPTION_ONLY([sounds-cache], [SOUNDS_CACHE_DIR], [cached sound tarfiles], [])
+AST_OPTION_ONLY([externals-cache], [EXTERNALS_CACHE_DIR], [cached external module tarfiles], [])
AST_EXT_LIB_SETUP([SPANDSP], [SPANDSP], [spandsp])
AST_EXT_LIB_SETUP([SS7], [ISDN SS7], [ss7])
AST_EXT_LIB_SETUP([SPEEX], [Speex], [speex])
diff --git a/makeopts.in b/makeopts.in
index d4347da..38d4258 100644
--- a/makeopts.in
+++ b/makeopts.in
@@ -28,6 +28,7 @@
FETCH=@FETCH@
DOWNLOAD=@DOWNLOAD@
SOUNDS_CACHE_DIR=@SOUNDS_CACHE_DIR@
+EXTERNALS_CACHE_DIR=@EXTERNALS_CACHE_DIR@
RUBBER=@RUBBER@
CATDVI=@CATDVI@
KPATHSEA=@KPATHSEA@
diff --git a/res/Makefile b/res/Makefile
index e56a148..d78baf0 100644
--- a/res/Makefile
+++ b/res/Makefile
@@ -71,3 +71,10 @@
# Dependencies for res_ari_*.so are generated, so they're in this file
include ari.make
+
+ifneq ($(XMLSTARLET),:)
+EXTERNAL_MODS := $(shell $(XMLSTARLET) sel -t -v "/category/member[support_level = 'external']/@name" .moduleinfo)
+
+install::
+ $(foreach MODULE,$(filter-out $(EMBEDDED_MODS),$(EXTERNAL_MODS)),$(ASTTOPDIR)/build_tools/download_externals $(MODULE))
+endif
diff --git a/res/res.xml b/res/res.xml
new file mode 100644
index 0000000..f5d4d6a
--- /dev/null
+++ b/res/res.xml
@@ -0,0 +1,5 @@
+<member name="res_digium_phone" displayname="Download the Digium Phone Module for Asterisk. See http://downloads.digium.com/pub/telephony/res_digium_phone/README.">
+ <support_level>external</support_level>
+ <depend>xmlstarlet</depend>
+ <defaultenabled>no</defaultenabled>
+</member>
--
To view, visit https://gerrit.asterisk.org/3768
To unsubscribe, visit https://gerrit.asterisk.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id3dcf1289ffd3cb0bbd7dfab3cafbb87be60323a
Gerrit-PatchSet: 1
Gerrit-Project: asterisk
Gerrit-Branch: 14
Gerrit-Owner: George Joseph <gjoseph at digium.com>
More information about the asterisk-code-review
mailing list