samba-mirror/bootstrap/config.py

#!/usr/bin/env python3

# Copyright (C) Catalyst.Net Ltd 2019
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Manage dependencies and bootstrap environments for Samba.

Config file for packages and templates.

Author: Joe Guo <joeg@catalyst.net.nz>
"""
import os
from os.path import abspath, dirname, join
HERE = abspath(dirname(__file__))
# output dir for rendered files
OUT = join(HERE, 'generated-dists')


# pkgs with same name in all packaging systems
COMMON = [
    'acl',
    'attr',
    'autoconf',
    'binutils',
    'bison',
    'curl',
    'flex',
    'gcc',
    'gdb',
    'git',
    'gzip',
    'hostname',
    'htop',
    'make',
    'patch',
    'perl',
    'psmisc',  # for pstree in test
    'rng-tools',
    'rsync',
    'sed',
    'sudo',  # docker images has no sudo by default
    'tar',
    'tree',
]


# define pkgs for all packaging systems in parallel
# make it easier to find missing ones
# use latest ubuntu and fedora as defaults
# deb, rpm, ...
PKGS = [
    # NAME1-dev, NAME2-devel
    ('lmdb-utils', 'lmdb'),
    ('nettle-dev', 'nettle-devel'),
    ('zlib1g-dev', 'zlib-devel'),
    ('libbsd-dev', 'libbsd-devel'),
    ('libaio-dev', 'libaio-devel'),
    ('libarchive-dev', 'libarchive-devel'),
    ('libblkid-dev', 'libblkid-devel'),
    ('libcap-dev', 'libcap-devel'),
    ('libacl1-dev', 'libacl-devel'),
    ('libattr1-dev', 'libattr-devel'),

    # libNAME1-dev, NAME2-devel
    ('libpopt-dev', 'popt-devel'),
    ('libreadline-dev', 'readline-devel'),
    ('libjansson-dev', 'jansson-devel'),
    ('liblmdb-dev', 'lmdb-devel'),
    ('libncurses5-dev', 'ncurses-devel'),
    # NOTE: Debian 7+ or Ubuntu 16.04+
    ('libsystemd-dev', 'systemd-devel'),
    ('libkrb5-dev', 'krb5-devel'),
    ('libldap2-dev', 'openldap-devel'),
    ('libcups2-dev', 'cups-devel'),
    ('libpam0g-dev', 'pam-devel'),
    ('libgpgme11-dev', 'gpgme-devel'),
    # NOTE: Debian 8+ and Ubuntu 14.04+
    ('libgnutls28-dev', 'gnutls-devel'),
    ('libtasn1-bin', ''),
    ('libtasn1-dev', 'libtasn1-devel'),
    ('', 'quota-devel'),
    ('uuid-dev', 'libuuid-devel'),
    ('libjs-jquery', ''),
    ('libavahi-common-dev', 'avahi-devel'),
    ('libdbus-1-dev', 'dbus-devel'),
    ('libpcap-dev', 'libpcap-devel'),
    ('libunwind-dev', 'libunwind-devel'),  # for back trace
    ('libglib2.0-dev', 'glib2-devel'),
    ('libicu-dev', 'libicu-devel'),

    # NAME1, NAME2
    # for debian, locales provide locale support with language packs
    # ubuntu split language packs to language-pack-xx
    # for centos, glibc-common provide locale support with language packs
    # fedora split language packs  to glibc-langpack-xx
    ('locales', 'glibc-common'),  # required for locale
    ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
    ('bind9utils', 'bind-utils'),
    ('dnsutils', ''),
    ('xsltproc', 'libxslt'),
    ('krb5-user', ''),
    ('krb5-config', ''),
    ('krb5-kdc', 'krb5-server'),
    ('apt-utils', 'yum-utils'),
    ('pkg-config', 'pkgconfig'),
    ('procps', 'procps-ng'),  # required for the free cmd in tests
    ('lsb-release', 'lsb-release'),  # we need lsb_relase to show info
    ('', 'rpcgen'),  # required for test
    # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
    ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
    ('', 'libnsl2-devel'),  # for <rpcsvc/yp_prot.h> header on fedora
    ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
    ('mawk', 'gawk'),

    # python
    ('python-dev', 'python-devel'),
    ('python-dbg', ''),
    ('python-iso8601', ''),
    ('python-gpg', 'python2-gpg'),  # defaults to ubuntu/fedora latest
    ('python-crypto', 'python-crypto'),
    ('python-markdown', 'python-markdown'),
    ('python-dnspython', 'python-dns'),
    ('python-pexpect', ''),  # for wintest only

    ('python3-dev', 'python3-devel'),
    ('python3-dbg', ''),
    ('python3-iso8601', ''),
    ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
    ('python3-crypto', 'python3-crypto'),
    ('python3-markdown', 'python3-markdown'),
    ('python3-matplotlib', ''),
    ('python3-dnspython', 'python3-dns'),
    ('python3-pexpect', ''),  # for wintest only

    ('', 'libsemanage-python'),
    ('', 'policycoreutils-python'),

    # perl
    ('libparse-yapp-perl', 'perl-Parse-Yapp'),
    ('libjson-perl', 'perl-JSON-Parse'),
    ('perl-modules', ''),
    ('', 'perl-Archive-Tar'),
    ('', 'perl-ExtUtils-MakeMaker'),
    ('', 'perl-Test-Base'),
    ('', 'perl-generators'),
    ('', 'perl-interpreter'),

    # fs
    ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
    ('', 'glusterfs-api-devel'),
    ('glusterfs-common', 'glusterfs-devel'),
    ('libcephfs-dev', 'libcephfs-devel'),

    # misc
    # @ means group for rpm, use fedora as rpm default
    ('build-essential', '@development-tools'),
    ('debhelper', ''),
    # rpm has no pkg for docbook-xml
    ('docbook-xml', 'docbook-dtds'),
    ('docbook-xsl', 'docbook-style-xsl'),
    ('', 'keyutils-libs-devel'),
    ('', 'which'),
]


DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]

GENERATED_MARKER = r"""
#
# This file is generated by 'bootstrap/template.py --render'
# See also bootstrap/config.py
#
"""


APT_BOOTSTRAP = r"""
#!/bin/bash
{GENERATED_MARKER}
set -xueo pipefail

export DEBIAN_FRONTEND=noninteractive
apt-get -y update

apt-get -y install \
    {pkgs}

apt-get -y autoremove
apt-get -y autoclean
apt-get -y clean
"""


YUM_BOOTSTRAP = r"""
#!/bin/bash
{GENERATED_MARKER}
set -xueo pipefail

yum -y -q update
yum -y -q install epel-release
yum -y -q update

yum -y -q --verbose install \
    {pkgs}

yum clean all
"""


DNF_BOOTSTRAP = r"""
#!/bin/bash
{GENERATED_MARKER}
set -xueo pipefail

dnf update -y

dnf install -y \
    --setopt=install_weak_deps=False \
    {pkgs}

dnf clean all
"""

ZYPPER_BOOTSTRAP = r"""
#!/bin/bash
{GENERATED_MARKER}
set -xueo pipefail

zypper --non-interactive refresh
zypper --non-interactive update
zypper --non-interactive install \
    --no-recommends \
    system-user-nobody \
    {pkgs}

zypper --non-interactive clean

if [ -f /usr/lib/mit/bin/krb5-config ]; then
    ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
fi
"""

# A generic shell script to setup locale
LOCALE_SETUP = r"""
#!/bin/bash
{GENERATED_MARKER}
set -xueo pipefail

# refer to /usr/share/i18n/locales
INPUTFILE=en_US
# refer to /usr/share/i18n/charmaps
CHARMAP=UTF-8
# locale to generate in /usr/lib/locale
# glibc/localedef will normalize UTF-8 to utf8, follow the naming style
LOCALE=$INPUTFILE.utf8

# if locale is already correct, exit
( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0

# if locale not available, generate locale into /usr/lib/locale
if ! ( locale --all-locales | grep -i $LOCALE )
then
    # no-archive means create its own dir
    localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
fi

# update locale conf and global env file
# set both LC_ALL and LANG for safe

# update conf for Debian family
FILE=/etc/default/locale
if [ -f $FILE ]
then
    echo LC_ALL="$LOCALE" > $FILE
    echo LANG="$LOCALE" >> $FILE
fi

# update conf for RedHat family
FILE=/etc/locale.conf
if [ -f $FILE ]
then
    # LC_ALL is not valid in this file, set LANG only
    echo LANG="$LOCALE" > $FILE
fi

# update global env file
FILE=/etc/environment
if [ -f $FILE ]
then
    # append LC_ALL if not exist
    grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
    # append LANG if not exist
    grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
fi
"""


DOCKERFILE = r"""
{GENERATED_MARKER}
FROM {docker_image}

# pass in with --build-arg while build
ARG SHA1SUM
RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt

ADD *.sh /tmp/
# need root permission, do it before USER samba
RUN /tmp/bootstrap.sh && /tmp/locale.sh

# if ld.gold exists, force link it to ld
RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"

# make test can not work with root, so we have to create a new user
RUN useradd -m -U -s /bin/bash samba && \
    mkdir -p /etc/sudoers.d && \
    echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba

USER samba
WORKDIR /home/samba
# samba tests rely on this
ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
"""

# Vagrantfile snippet for each dist
VAGRANTFILE_SNIPPET = r"""
    config.vm.define "{name}" do |v|
        v.vm.box = "{vagrant_box}"
        v.vm.hostname = "{name}"
        v.vm.provision :shell, path: "{name}/bootstrap.sh"
        v.vm.provision :shell, path: "{name}/locale.sh"
    end
"""

# global Vagrantfile with snippets for all dists
VAGRANTFILE_GLOBAL = r"""
{GENERATED_MARKER}

Vagrant.configure("2") do |config|
    config.ssh.insert_key = false

{vagrantfile_snippets}

end
"""


DEB_DISTS = {
    'debian7': {
        'docker_image': 'debian:7',
        'vagrant_box': 'debian/wheezy64',
        'replace': {
            'libgnutls28-dev': 'libgnutls-dev',
            'libsystemd-dev': '',  # not available, remove
            'lmdb-utils': '',  # not available, remove
            'liblmdb-dev': '',  # not available, remove
            'python-gpg': 'python-gpgme',
            'python3-gpg': '',  # no python3 gpg pkg available, remove
            'language-pack-en': '',   # included in locales
        }
    },
    'debian8': {
        'docker_image': 'debian:8',
        'vagrant_box': 'debian/jessie64',
        'replace': {
            'python-gpg': 'python-gpgme',
            'python3-gpg': 'python3-gpgme',
            'language-pack-en': '',   # included in locales
        }
    },
    'debian9': {
        'docker_image': 'debian:9',
        'vagrant_box': 'debian/stretch64',
        'replace': {
            'language-pack-en': '',   # included in locales
        }
    },
    'ubuntu1404': {
        'docker_image': 'ubuntu:14.04',
        'vagrant_box': 'ubuntu/trusty64',
        'replace': {
            'libsystemd-dev': '',  # remove
            'libgnutls28-dev': 'libgnutls-dev',
            'python-gpg': 'python-gpgme',
            'python3-gpg': 'python3-gpgme',
            'lmdb-utils': 'lmdb-utils/trusty-backports',
            'liblmdb-dev': 'liblmdb-dev/trusty-backports',
            'libunwind-dev': 'libunwind8-dev',
            'glusterfs-common': '',
            'libcephfs-dev': '',
        }
    },
    'ubuntu1604': {
        'docker_image': 'ubuntu:16.04',
        'vagrant_box': 'ubuntu/xenial64',
        'replace': {
            'python-gpg': 'python-gpgme',
            'python3-gpg': 'python3-gpgme',
            'glusterfs-common': '',
            'libcephfs-dev': '',
        }
    },
    'ubuntu1804': {
        'docker_image': 'ubuntu:18.04',
        'vagrant_box': 'ubuntu/bionic64',
    },
}


RPM_DISTS = {
    'centos6': {
        'docker_image': 'centos:6',
        'vagrant_box': 'centos/6',
        'bootstrap': YUM_BOOTSTRAP,
        'replace': {
            'lsb-release': 'redhat-lsb',
            'python3-devel': 'python34-devel',
            'python2-gpg': 'pygpgme',
            'python3-gpg': '',  # no python3-gpg yet
            '@development-tools': '"@Development Tools"',  # add quotes
            'glibc-langpack-en': '',  # included in glibc-common
            'glibc-locale-source': '',  # included in glibc-common
            'procps-ng': 'procps',  # centos6 still use old name
            # update perl core modules on centos
            # fix: Can't locate Archive/Tar.pm in @INC
            'perl': 'perl-core',
            'rpcsvc-proto-devel': '',
            'glusterfs-api-devel': '',
            'glusterfs-devel': '',
            'libcephfs-devel': '',
        }
    },
    'centos7': {
        'docker_image': 'centos:7',
        'vagrant_box': 'centos/7',
        'bootstrap': YUM_BOOTSTRAP,
        'replace': {
            'lsb-release': 'redhat-lsb',
            'python3-devel': 'python34-devel',
            # although python36-devel is available
            # after epel-release installed
            # however, all other python3 pkgs are still python34-ish
            'python2-gpg': 'pygpgme',
            'python3-gpg': '',  # no python3-gpg yet
            '@development-tools': '"@Development Tools"',  # add quotes
            'glibc-langpack-en': '',  # included in glibc-common
            'glibc-locale-source': '',  # included in glibc-common
            # update perl core modules on centos
            # fix: Can't locate Archive/Tar.pm in @INC
            'perl': 'perl-core',
            'rpcsvc-proto-devel': '',
            'glusterfs-api-devel': '',
            'glusterfs-devel': '',
            'libcephfs-devel': '',
        }
    },
    'fedora28': {
        'docker_image': 'fedora:28',
        'vagrant_box': 'fedora/28-cloud-base',
        'bootstrap': DNF_BOOTSTRAP,
        'replace': {
            'lsb-release': 'redhat-lsb',
        }
    },
    'fedora29': {
        'docker_image': 'fedora:29',
        'vagrant_box': 'fedora/29-cloud-base',
        'bootstrap': DNF_BOOTSTRAP,
        'replace': {
            'lsb-release': 'redhat-lsb',
        }
    },
    'opensuse150': {
        'docker_image': 'opensuse/leap:15.0',
        'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
        'bootstrap': ZYPPER_BOOTSTRAP,
        'replace': {
            '@development-tools': '',
            'dbus-devel': 'dbus-1-devel',
            'docbook-style-xsl': 'docbook-xsl-stylesheets',
            'glibc-common': 'glibc-locale',
            'glibc-locale-source': 'glibc-i18ndata',
            'glibc-langpack-en': '',
            'jansson-devel': 'libjansson-devel',
            'keyutils-libs-devel': 'keyutils-devel',
            'krb5-workstation': 'krb5-client',
            'libnsl2-devel': 'libnsl-devel',
            'libsemanage-python': 'python2-semanage',
            'nettle-devel': 'libnettle-devel',
            'openldap-devel': 'openldap2-devel',
            'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
            'perl-JSON-Parse': 'perl-JSON-XS',
            'perl-generators': '',
            'perl-interpreter': '',
            'procps-ng': 'procps',
            'python-dns': 'python2-dnspython',
            'python3-crypto': 'python3-pycrypto',
            'python3-dns': 'python3-dnspython',
            'python3-markdown': 'python3-Markdown',
            'quota-devel': '',
            'glusterfs-api-devel': '',
        }
    }
}


DEB_FAMILY = {
    'name': 'deb',
    'pkgs': DEB_PKGS,
    'bootstrap': APT_BOOTSTRAP,  # family default
    'dists': DEB_DISTS,
}


RPM_FAMILY = {
    'name': 'rpm',
    'pkgs': RPM_PKGS,
    'bootstrap': YUM_BOOTSTRAP,  # family default
    'dists': RPM_DISTS,
}


YML_HEADER = r"""
---
packages:
"""


def expand_family_dists(family):
    dists = {}
    for name, config in family['dists'].items():
        config = config.copy()
        config['name'] = name
        config['home'] = join(OUT, name)
        config['family'] = family['name']
        config['GENERATED_MARKER'] = GENERATED_MARKER

        # replace dist specific pkgs
        replace = config.get('replace', {})
        pkgs = []
        for pkg in family['pkgs']:
            pkg = replace.get(pkg, pkg)  # replace if exists or get self
            if pkg:
                pkgs.append(pkg)
        pkgs.sort()

        lines = ['  - {}'.format(pkg) for pkg in pkgs]
        config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)

        sep = ' \\' + os.linesep + '    '
        config['pkgs'] = sep.join(pkgs)

        # get dist bootstrap template or fall back to family default
        bootstrap_template = config.get('bootstrap', family['bootstrap'])
        config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
        config['locale.sh'] = LOCALE_SETUP.format(**config).strip()

        config['Dockerfile'] = DOCKERFILE.format(**config).strip()
        # keep the indent, no strip
        config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)

        dists[name] = config
    return dists


# expanded config for dists
DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)

# assemble all together
DISTS = {}
DISTS.update(DEB_DISTS_EXP)
DISTS.update(RPM_DISTS_EXP)


def render_vagrantfile(dists):
    """
    Render all snippets for each dist into global Vagrantfile.

    Vagrant supports multiple vms in one Vagrantfile.
    This make it easier to manage the fleet, e.g:

    start all: vagrant up
    start one: vagrant up ubuntu1804

    All other commands apply to above syntax, e.g.: status, destroy, provision
    """
    # sort dists by name and put all vagrantfile snippets together
    snippets = [
        dists[dist]['vagrantfile_snippet']
        for dist in sorted(dists.keys())]

    return VAGRANTFILE_GLOBAL.format(
            vagrantfile_snippets=''.join(snippets),
            GENERATED_MARKER=GENERATED_MARKER
            )


VAGRANTFILE = render_vagrantfile(DISTS)


# data we need to expose
__all__ = ['DISTS', 'VAGRANTFILE', 'OUT']