From 4e6576e7a4d11e717ce6e16121bbb1bdb362bb50 Mon Sep 17 00:00:00 2001 From: Alessio Sergi Date: Sun, 16 Oct 2016 16:46:46 +0200 Subject: [PATCH] xbps-src: add support for python3 pkgs - python_module build style now builds modules for python2/3 by default - new python2_module and python3_module build styles for building python2-only and python3-only packages respectively - no more python_versions - no need to define pycompile_version for Python modules anymore (still needed for non-Python modules though) - Python version and paths are now guessed automatically and a set of useful variables can now be used in templates - #!/usr/bin/python2 and #!/usr/bin/python3 are now the default shebangs - /usr/bin/foo2 and /usr/bin/foo3 are now the default names for bin scripts (for use with alternatives) --- Manual.md | 51 +++++++++++-------- common/build-style/python-module.sh | 16 +++--- common/build-style/python2-module.sh | 37 ++++++++++++++ common/build-style/python3-module.sh | 37 ++++++++++++++ common/environment/setup/python.sh | 18 +++++++ common/environment/setup/sourcepkg.sh | 2 +- .../04-create-xbps-metadata-scripts.sh | 4 ++ .../pre-pkg/03-rewrite-python-shebang.sh | 41 +++++---------- 8 files changed, 149 insertions(+), 57 deletions(-) create mode 100644 common/build-style/python2-module.sh create mode 100644 common/build-style/python3-module.sh create mode 100644 common/environment/setup/python.sh diff --git a/Manual.md b/Manual.md index d677a9c939a..46a994c582e 100644 --- a/Manual.md +++ b/Manual.md @@ -497,9 +497,6 @@ sonames in shared libraries. - `nocross` If set, cross compilation won't be allowed and will exit immediately. -- `python_versions` A white space separated list of python versions which will -be used to build that package. This is only used by the `python-module` build style. - - `subpackages` A white space separated list of subpackages (matching `foo_package()`) to override the guessed list. Only use this if a specific order of subpackages is required, otherwise the default would work in most cases. @@ -657,11 +654,6 @@ Additional install arguments can be specified via `make_install_args`. - `perl-module` For packages that use the Perl [ExtUtils::MakeMaker](http://perldoc.perl.org/ExtUtils/MakeMaker.html) build method. -- `python-module` For packages that use the Python module build method (setup.py). -By default the module will be built for python2. The `python_versions` variable may -be defined to set the allowed python versions to be built, i.e: -`python_versions="2.7 3.3"`. - - `waf3` For packages that use the Python3 `waf` build method with python3. - `waf` For packages that use the Python `waf` method with python2. @@ -675,6 +667,15 @@ be passed in via `make_build_args` and install arguments via `make_install_args` target can be overridden via `make_build_target` and the install target via `make_install_target`. +For packages that use the Python module build method (`setup.py`), you +can choose one of the following: + +- `python-module` to build *both* Python 2.x and 3.x modules + +- `python2-module` to build Python 2.x only modules + +- `python3-module` to build Python 3.x only modules + > If `build_style` is not set, the template must (at least) define a `do_install()` function and optionally more phases via `do_xxx()` functions. @@ -1080,9 +1081,9 @@ be your guidance to decide whether or not to split off a `-doc` subpackage. ### Python packages -Python packages should be built with the `python-module` build style, if possible. This sets -some environment variables required to allow cross compilation. Support to allow building -a python module for multiple versions from a single template is also possible. +Python packages should be built with the `python{,2,3}-module` build style, if possible. +This sets some environment variables required to allow cross compilation. Support to allow +building a python module for multiple versions from a single template is also possible. To allow cross compilation, the `python-devel` package (for python 2.7) must be added to `hostmakedepends` and `makedepends`. If any other python version is also supported, @@ -1091,15 +1092,6 @@ for example python3.4, those must also be added as host and target build depende The following variables may influence how the python packages are built and configured at post-install time: -- `python_versions`: this variable expects the python versions supported by the module. -By default it's always set to `2.7`. If a package for another python version is wanted -you can set all acceptable versions, i.e `python_versions="2.7 3.4"` will build a package -for `python (2.7)` and `python3.4`. - -- `pycompile_version`: this variable expects the python version that is used to -byte-compile the python code (it generates the `.py[co]` files at post-install time). -By default it's set to `2.7` for `python 2.x` packages. - - `pycompile_module`: this variable expects the python modules that should be `byte-compiled` at post-install time. Python modules are those that are installed into the `site-packages` prefix: `usr/lib/pythonX.X/site-packages`. Multiple python modules may be specified separated @@ -1109,6 +1101,25 @@ by blanks, i.e `pycompile_module="foo blah"`. recursively by the target python version. This differs from `pycompile_module` in that any path may be specified, i.e `pycompile_dirs="usr/share/foo"`. +- `pycompile_version`: this variable expects the python version that is used to +byte-compile the python code (it generates the `.py[co]` files at post-install time). +By default it's set to `2.7` for `python 2.x` packages. + +> NOTE: you need to define it *only* for non-Python modules. + +Also, a set of useful variables are defined to use in the templates: + +| Variable | Value | +|-------------|----------------------------------| +| py2_ver | 2.X | +| py2_lib | /usr/lib/python2.X | +| py2_sitelib | /usr/lib/python2.X/site-packages | +| py2_inc | /usr/include/python2.X | +| py3_ver | 3.X | +| py3_lib | /usr/lib/python3.X | +| py3_sitelib | /usr/lib/python3.X/site-packages | +| py3_inc | /usr/include/python3.Xm | + > NOTE: it's expected that additional subpkgs must be generated to allow packaging for multiple python versions. diff --git a/common/build-style/python-module.sh b/common/build-style/python-module.sh index dafdbc591c4..57ec8c73831 100644 --- a/common/build-style/python-module.sh +++ b/common/build-style/python-module.sh @@ -3,7 +3,7 @@ # do_build() { - : ${python_versions:=2.7} + : ${python_versions:="2.7 $py3_ver"} local pyver= pysufx= for pyver in $python_versions; do @@ -27,7 +27,7 @@ do_build() { } do_install() { - : ${python_versions:=2.7} + : ${python_versions:="2.7 $py3_ver"} local pyver= pysufx= for pyver in $python_versions; do @@ -51,13 +51,11 @@ do_install() { fi # Rename unversioned scripts to avoid name conflicts. - if [ "$python_versions" != "2.7" -a "$python_versions" != "${python_versions#2.7}" ]; then - if [ -d ${DESTDIR}/usr/bin ]; then - find ${DESTDIR}/usr/bin -type f ! -name "*[[:digit:]]\.[[:digit:]]" | while IFS= read -r f _; do - mv "${f}" "${f}${pyver}" - echo "[python-module] Unversioned script renamed to '${f#$DESTDIR}${pyver}'" - done - fi + if [ -d ${DESTDIR}/usr/bin ]; then + find ${DESTDIR}/usr/bin -type f ! -name "*[[:digit:]]" | while IFS= read -r f _; do + mv "${f}" "${f}${pyver%.*}" + echo "[python-module] Unversioned script renamed to '${f#$DESTDIR}${pyver%.*}'" + done fi done } diff --git a/common/build-style/python2-module.sh b/common/build-style/python2-module.sh new file mode 100644 index 00000000000..62c5fdd592c --- /dev/null +++ b/common/build-style/python2-module.sh @@ -0,0 +1,37 @@ +# +# This helper is for templates installing python2-only modules. +# + +do_build() { + if [ -n "$CROSS_BUILD" ]; then + PYPREFIX="$XBPS_CROSS_BASE" + CFLAGS+=" -I${XBPS_CROSS_BASE}/${py2_inc} -I${XBPS_CROSS_BASE}/usr/include" + LDFLAGS+=" -L${XBPS_CROSS_BASE}/${py2_lib} -L${XBPS_CROSS_BASE}/usr/lib" + CC="${XBPS_CROSS_TRIPLET}-gcc -pthread $CFLAGS $LDFLAGS" + LDSHARED="${CC} -shared $LDFLAGS" + env CC="$CC" LDSHARED="$LDSHARED" \ + PYPREFIX="$PYPREFIX" CFLAGS="$CFLAGS" \ + LDFLAGS="$LDFLAGS" python2 setup.py \ + build --build-base=build-${py2_ver} ${make_build_args} + else + python2 setup.py build --build-base=build-${py2_ver} ${make_build_args} + fi +} + +do_install() { + if [ -n "$CROSS_BUILD" ]; then + PYPREFIX="$XBPS_CROSS_BASE" + CFLAGS+=" -I${XBPS_CROSS_BASE}/${py2_inc} -I${XBPS_CROSS_BASE}/usr/include" + LDFLAGS+=" -L${XBPS_CROSS_BASE}/${py2_lib} -L${XBPS_CROSS_BASE}/usr/lib" + CC="${XBPS_CROSS_TRIPLET}-gcc -pthread $CFLAGS $LDFLAGS" + LDSHARED="${CC} -shared $LDFLAGS" + env CC="$CC" LDSHARED="$LDSHARED" \ + PYPREFIX="$PYPREFIX" CFLAGS="$CFLAGS" \ + LDFLAGS="$LDFLAGS" python2 setup.py \ + build --build-base=build-${py2_ver} \ + install --prefix=/usr --root=${DESTDIR} ${make_install_args} + else + python2 setup.py build --build-base=build-${py2_ver} \ + install --prefix=/usr --root=${DESTDIR} ${make_install_args} + fi +} diff --git a/common/build-style/python3-module.sh b/common/build-style/python3-module.sh new file mode 100644 index 00000000000..df2bd901564 --- /dev/null +++ b/common/build-style/python3-module.sh @@ -0,0 +1,37 @@ +# +# This helper is for templates installing python3-only modules. +# + +do_build() { + if [ -n "$CROSS_BUILD" ]; then + PYPREFIX="$XBPS_CROSS_BASE" + CFLAGS+=" -I${XBPS_CROSS_BASE}/${py3_inc} -I${XBPS_CROSS_BASE}/usr/include" + LDFLAGS+=" -L${XBPS_CROSS_BASE}/${py3_lib} -L${XBPS_CROSS_BASE}/usr/lib" + CC="${XBPS_CROSS_TRIPLET}-gcc -pthread $CFLAGS $LDFLAGS" + LDSHARED="${CC} -shared $LDFLAGS" + env CC="$CC" LDSHARED="$LDSHARED" \ + PYPREFIX="$PYPREFIX" CFLAGS="$CFLAGS" \ + LDFLAGS="$LDFLAGS" python3 setup.py \ + build --build-base=build-${py3_ver} ${make_build_args} + else + python3 setup.py build --build-base=build-${py3_ver} ${make_build_args} + fi +} + +do_install() { + if [ -n "$CROSS_BUILD" ]; then + PYPREFIX="$XBPS_CROSS_BASE" + CFLAGS+=" -I${XBPS_CROSS_BASE}/${py3_inc} -I${XBPS_CROSS_BASE}/usr/include" + LDFLAGS+=" -L${XBPS_CROSS_BASE}/${py3_lib} -L${XBPS_CROSS_BASE}/usr/lib" + CC="${XBPS_CROSS_TRIPLET}-gcc -pthread $CFLAGS $LDFLAGS" + LDSHARED="${CC} -shared $LDFLAGS" + env CC="$CC" LDSHARED="$LDSHARED" \ + PYPREFIX="$PYPREFIX" CFLAGS="$CFLAGS" \ + LDFLAGS="$LDFLAGS" python3 setup.py \ + build --build-base=build-${py3_ver} \ + install --prefix=/usr --root=${DESTDIR} ${make_install_args} + else + python3 setup.py build --build-base=build-${py3_ver} \ + install --prefix=/usr --root=${DESTDIR} ${make_install_args} + fi +} diff --git a/common/environment/setup/python.sh b/common/environment/setup/python.sh new file mode 100644 index 00000000000..5f3209aa0c4 --- /dev/null +++ b/common/environment/setup/python.sh @@ -0,0 +1,18 @@ +# +# Useful variables for determining Python version and paths. +# +__python2="/usr/bin/python2" +__python3="/usr/bin/python3" + +if [ -x ${__python2} ]; then + py2_ver="$(${__python2} -c 'import sys; print(sys.version[:3])')" + py2_lib="$(${__python2} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib(0, 1))')" + py2_sitelib="$(${__python2} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" + py2_inc="$(${__python2} -c 'from distutils.sysconfig import get_python_inc; print(get_python_inc())')" +fi +if [ -x ${__python3} ]; then + py3_ver="$(${__python3} -c 'import sys; print(sys.version[:3])')" + py3_lib="$(${__python3} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib(0, 1))')" + py3_sitelib="$(${__python3} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" + py3_inc="$(${__python3} -c 'from distutils.sysconfig import get_python_inc; print(get_python_inc())')" +fi diff --git a/common/environment/setup/sourcepkg.sh b/common/environment/setup/sourcepkg.sh index edc408fd122..435e1438c6a 100644 --- a/common/environment/setup/sourcepkg.sh +++ b/common/environment/setup/sourcepkg.sh @@ -5,7 +5,7 @@ unset -v pkgname version revision short_desc homepage license maintainer unset -v only_for_archs distfiles checksum build_style nocross broken unset -v configure_script configure_args wrksrc build_wrksrc create_wrksrc -unset -v make_cmd make_build_args make_install_args make_build_target make_install_target python_versions stackage +unset -v make_cmd make_build_args make_install_args make_build_target make_install_target stackage unset -v patch_args disable_parallel_build keep_libtool_archives unset -v reverts subpackages makedepends hostmakedepends depends restricted unset -v nopie build_options build_options_default bootstrap repository reverts diff --git a/common/hooks/post-install/04-create-xbps-metadata-scripts.sh b/common/hooks/post-install/04-create-xbps-metadata-scripts.sh index 36dbc8c72fa..090e27e4a7c 100644 --- a/common/hooks/post-install/04-create-xbps-metadata-scripts.sh +++ b/common/hooks/post-install/04-create-xbps-metadata-scripts.sh @@ -231,6 +231,10 @@ _EOF # # Handle python bytecode archives with pycompile trigger. # + if [ -d ${PKGDESTDIR}/usr/lib/python* ]; then + pycompile_version="$(find ${PKGDESTDIR}/usr/lib/python* -type d | grep -o '[[:digit:]]\.[[:digit:]]$')" + fi + if [ -n "${pycompile_dirs}" -o -n "${pycompile_module}" ]; then echo "export pycompile_version=\"${pycompile_version:=2.7}\"" >>$tmpf if [ -n "${pycompile_dirs}" ]; then diff --git a/common/hooks/pre-pkg/03-rewrite-python-shebang.sh b/common/hooks/pre-pkg/03-rewrite-python-shebang.sh index e45c6044458..e5fb2157441 100644 --- a/common/hooks/pre-pkg/03-rewrite-python-shebang.sh +++ b/common/hooks/pre-pkg/03-rewrite-python-shebang.sh @@ -2,36 +2,23 @@ # - rewrites python shebangs with the corresponding python version hook() { - local pyver= shebang= warn= off= + local pyver= shebang= off= - case $pkgname in - python-*) - pyver=2.7;; - python3.4-*) - pyver=3.4;; - python3.5-*) - pyver=3.5;; - *) - for i in $pycompile_version $python_versions; do - if [ "$pyver" ]; then - warn=1 - break; - fi - pyver=$i - done - : ${pyver:=2.7} - ;; - esac + : ${pyver:=2} - shebang="#!/usr/bin/python$pyver" + if [ -d ${PKGDESTDIR}/usr/lib/python* ]; then + pycompile_version="$(find ${PKGDESTDIR}/usr/lib/python* -type d | grep -o '[[:digit:]]\.[[:digit:]]$')" + fi + + if [ -n "$pycompile_version" ]; then + pyver="$pycompile_version" + fi + + shebang="#!/usr/bin/python${pyver%.*}" find ${PKGDESTDIR} -type f -print0 | \ - xargs -0 grep -H -b -m 1 "^#!.*\([[:space:]]\|/\)python\([[:space:]]\|$\)" -- | while IFS=: read -r f off _; do + xargs -0 grep -H -b -m 1 "^#!.*\([[:space:]]\|/\)python\([[:space:]]*\|$\)" -- | while IFS=: read -r f off _; do [ -z "$off" ] && continue - if [ "$warn" ]; then - msg_warn "$pkgver: multiple python versions defined! (using $pyver for shebangs)\n" - unset warn - fi - echo " Unversioned shebang replaced by '$shebang': ${f#$PKGDESTDIR}" - sed -i "1s@.*python@${shebang}@" -- "$f" + echo " Shebang converted to '$shebang': ${f#$PKGDESTDIR}" + sed -i "1s@.*python.*@${shebang}@" -- "$f" done }