"""
.. attention:: **sbuild** (retry-workaround in ``gnupg.py``): Spurious build failures ``can't connect to the agent: IPC connect call failed`` (:debbug:`849551`)

.. attention:: **sbuild**: Fails to build "all" packages with "build archall" flag set to arch "x" in case DSP has >= 1 arch "all" and >=1 arch "y" binary package (:debbug:`706086`)

  This is due to sbuild and in in more detail explained in the Debian bug report.

  A bad one-package workaround would be to change the "build archall" flag to arch "y".

.. attention:: **sbuild** (workaround in ``apt-update`` snippet): ``apt-get update`` "File has unexpected size ... Mirror sync in progress?" failures

  This may happen if by chance there are reprepro updates around the same time ``apt-get update`` runs in the build.

  You may reproduce this quite reliably with the testsuite (it's due to the immediate parent package remove, see ``./devel``):

  1. testsuite: Change ``installed-port`` test to (also) port for running codename (``changelog`` + ``./devel``)
  2. ./devel updatetestall
  3. ./devel testsuite-packages installed-port

  Internal sbuild snippet works around this problem by retrying ``apt-get update`` -- but you still see the error lines
  in the buildlog if that happens.

  Solution could be to lock HTTP delivery on any reprepro change, but for now this seems overkill to me.

.. attention:: **sbuild+apt-cacher-ng** (workaround in ``apt-update`` snippet): ``apt-get update`` stalls on parallel access

  **Phenomena**: A build just always remains ongoing && the buildlog just stalls during ``apt-get update``.

  **Repeat bug**:

  .. literalinclude:: ../../examples/mini-buildd/apt-parallel-bug
    :caption: /usr/share/doc/mini-buildd/examples/apt-parallel-bug

  **Remedy situation at hand**:

  1. Kill corresponding process ``/usr/lib/apt/methods/gpgv`` or ``/usr/lib/apt/methods/http`` (apt-get update may be retried and build continues)
  2. Cancel resp. build(s) (then retry)

  **Related 3rd party findings**:

    * https://bugs.launchpad.net/ubuntu/+source/apt-cacher-ng/+bug/1983856
    * https://bugs.launchpad.net/ubuntu/+source/apt/+bug/2003851
"""
import os
import time
import re
import subprocess
import logging

from mini_buildd import util, config, call, files

LOG = logging.getLogger(__name__)


#: Build dir constants.
CONFIG_DIR = ".config"
CONFIG_APT_KEYS = "apt_keys"
CONFIG_APT_PREFERENCES = "apt_preferences"
CONFIG_APT_SOURCES_LIST = "apt_sources.list"
CONFIG_CHROOT_SETUP_SCRIPT = "chroot_setup_script"
CONFIG_SBUILDRC_SNIPPET = "sbuildrc_snippet"
CONFIG_SSL_CERT = "ssl_cert"

SETUP_D_DIR = ".setup.d"

#: Quiet, non-interactive, least invasive and loggable apt-get call (Dpkg::Use-Pty=false is to avoid https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=539617).
APT_GET = "apt-get --quiet --yes --option=APT::Install-Recommends=false --option=Acquire::Languages=none --option=Dpkg::Use-Pty=false --option=Dpkg::Options::=--force-confdef --option=Dpkg::Options::=--force-confnew"
APT_TRUSTED_GPGD = "/etc/apt/trusted.gpg.d"
APT_CONFD = "/etc/apt/apt.conf.d"

#: Safe canonized paths
BASH_VARS = """\
MBD_SCRIPT="$(realpath "${0}")"  # Note: Older versions of realpath (<=wheezy) did not support options, just avoid them
MBD_BASE="${MBD_SCRIPT%/*/*}"\
"""


class Blocks(files.Dir):
    def __init__(self, type_):
        super().__init__()
        self.type = type_
        self.name = f"Sbuild-{self.type.capitalize()}-Blocks"
        self.default = ""

    def usage(self):
        default = self.default.strip()
        header = (
            f"Ordered sbuild {self.type} blocks (space-separated)\n"
            "\n"
            f"Default if not given: {self.default}\n\n"
            "Available blocks:\n\n"
        )
        ml = max(len(name) for name in self.keys())  # "max length" formatting helper
        return header + "\n".join([f"{name:<{ml}}: [{'default' if name in default else 'extra'}] {block.description}" for name, block in self.items()])

    def validate(self, value):
        """Validate value from user space (string, space separated), return value as list"""
        eol = value.split()
        unknown = set(eol) - set(self.keys())
        if unknown:
            raise util.HTTPBadRequest(f"Unknown sbuild {self.type} block(s): {' '.join(unknown)}")
        return eol

    def line(self, value):
        self.validate(value)
        return f"{self.name}: {value}"


class ConfigBlocks(Blocks):
    def __init__(self):
        super().__init__("config")
        self.default = "set-default-path ccache"

    def configure(self, file_, blocks):
        """Add configured blocks to provided file"""
        for block in self.validate(blocks):
            file_.add(self[block].get(snippet=True))


class SetupBlocks(Blocks):
    def __init__(self):
        super().__init__("setup")
        self.default = "apt-setup apt-update ccache eatmydata"

    def configure(self, dir_, blocks):
        """Add configured blocks to provided dir"""
        for block in self.validate(blocks):
            dir_.add(self[block])


CONFIG_BLOCKS = ConfigBlocks()

CONFIG_BLOCKS.add(files.PerlModule(
    "set-default-path",
    description="Explicitly set $path (to sbuild's default value), so later config blocks may add to it",
    snippet="""\
$path = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games';
"""))

CONFIG_BLOCKS.add(files.PerlModule(
    "ccache",
    description="Update $path and $$build environment to support ccache",
    snippet="""\
$path = "/usr/lib/ccache:$path";
$$build_environment{'CCACHE_DIR'} = '%LIBDIR%/.ccache';
"""))

SETUP_BLOCKS = SetupBlocks()

SETUP_BLOCKS.add(files.BashScript(
    "apt-setup",
    description="Sets up APT for buildrequest (APT keys, optional https, sources and preferences)",
    snippet=f"""\
{BASH_VARS}

mkdir -p -v "{APT_TRUSTED_GPGD}"
cp -v "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_APT_KEYS}" "{APT_TRUSTED_GPGD}/mini-buildd-buildrequest.asc"

if grep 'https://' "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_APT_SOURCES_LIST}"; then
  {APT_GET} install ca-certificates apt-transport-https
  if [ -s "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_SSL_CERT}" ]; then
    cp -v "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_SSL_CERT}" /usr/local/share/ca-certificates/mini-buildd-repo.crt
    /usr/sbin/update-ca-certificates
  fi
fi

cp -v "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_APT_SOURCES_LIST}" /etc/apt/sources.list
cat /etc/apt/sources.list

cp -v "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_APT_PREFERENCES}" /etc/apt/preferences
cat /etc/apt/preferences
"""))

# * Using "timeout" to work around ``apt-parallel-bug```(see administrator.rst)
# * Using retry/sleep for occasional failures due to intermittent reprepro repository updates
SETUP_BLOCKS.add(files.ShScript(
    "apt-update",
    description="Update APT",
    snippet=f"""\
for N in 10 40 80 160 320; do
    if timeout ${{N}}s {APT_GET} update; then
       apt-cache policy
       exit 0
    fi
    printf "W: apt-update: Did not succeed after %s seconds (retrying)\n" "${{N}}" >&2
    sleep 10
done
exit 1
"""))

# All below need explicit opt-in
SETUP_BLOCKS.add(files.ShScript(
    "ccache",
    description="Installs ccache (together with the 'ccache' config block, this enables ccache for builds on the same instance)",
    snippet=f"""\
{APT_GET} install ccache
"""))

SETUP_BLOCKS.add(files.ShScript(
    "eatmydata",
    description="Install eatmydata and activate via LD_PRELOAD when possible (may improve speed)",
    snippet=f"""\
if {APT_GET} install eatmydata; then
    # eatmydata <= 26: Just one 'deb', and not multiarched
    EATMYDATA_SO=$(dpkg -L eatmydata | grep 'libeatmydata.so$') || true
    if [ -z "${{EATMYDATA_SO}}" ]; then
        # eatmydata >= 82: Has a library 'deb', and is multiarched
        EATMYDATA_SO=$(dpkg -L libeatmydata1 | grep 'libeatmydata.so$') || true
    fi
    if [ -n "${{EATMYDATA_SO}}" ]; then
        printf " ${{EATMYDATA_SO}}" >> /etc/ld.so.preload
    else
        printf "W: eatmydata: No *.so found (skipping)\\n" >&2
    fi
else
    printf "W: eatmydata: Not installable (skipping)\\n" >&2
fi
"""))


SETUP_BLOCKS.add(files.BashScript(
    "show-sbuildrc",
    description="Print generated '.sbuildrc' (informational only)",
    snippet=f"""\
{BASH_VARS}

cat "${{MBD_BASE}}/{CONFIG_DIR}/../.sbuildrc"
"""))


SETUP_BLOCKS.add(files.BashScript(
    "apt-key-binary",
    description="Convert APT keys to binary (needed for Debian <= jessie, Ubuntu <= yakketti) -- place between ``apt-setup`` and ``apt-update``",
    snippet=f"""\
{APT_GET} update
{APT_GET} install gnupg
gpg --dearmor "{APT_TRUSTED_GPGD}/mini-buildd-buildrequest.asc"
"""))

SETUP_BLOCKS.add(files.BashScript(
    "apt-key-add",
    description="Add APT keys via deprecated ``apt-key add`` (needed for Debian <= squeeze) -- place between ``apt-setup`` and ``apt-update``",
    snippet=f"""\
{BASH_VARS}

apt-key add "${{MBD_BASE}}/{CONFIG_DIR}/{CONFIG_APT_KEYS}"
"""))


def _setup_blocks_apt_option(name, option, desc):
    """Add a script that sets an APT option"""
    SETUP_BLOCKS.add(files.ShScript(
        name,
        description=f'Set APT option "{option}" ({desc}) -- place before ``apt-setup``',
        snippet=f'printf "%s\\n" "{option};" >"{APT_CONFD}/00mbd-{name}"\n'))


_setup_blocks_apt_option("apt-no-check-valid-until", "Acquire::Check-Valid-Until false", "needed when Release files are expired")
_setup_blocks_apt_option("apt-allow-unauthenticated", "APT::Get::AllowUnauthenticated true", "needed when APT keys are expired")
_setup_blocks_apt_option("apt-force-yes", "APT::Get::Force-Yes true", "compat(urold)")
_setup_blocks_apt_option("apt-no-cache", "Acquire::http::No-Cache true", "compat(urold)")
_setup_blocks_apt_option("apt-no-pipelining", "Acquire::http::Pipeline-Depth 0", "compat(urold)")


SETUP_BLOCKS.add(files.ShScript(
    "apt-clear",
    description="Try to clear any redundant APT data in ``/var/`` (experimental: can make APT work again if stuck)",
    snippet="""\
rm -v -r -f /var/lib/apt/lists/*
mkdir /var/lib/apt/lists/partial  # <= in lenny, apt would fail if missing
apt-get clean
"""))

SETUP_BLOCKS.add(files.ShScript(
    "bash-devices",
    description="Fixup /dev/fd|stdin|stdout|stderr. Needed for *building* bash only. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=327477 (compat(bash < 3.0-17 (2005))",
    snippet="""\
[ -e /dev/fd ] || ln -sv /proc/self/fd /dev/fd
[ -e /dev/stdin ] || ln -sv fd/0 /dev/stdin
[ -e /dev/stdout ] || ln -sv fd/1 /dev/stdout
[ -e /dev/stderr ] || ln -sv fd/2 /dev/stderr
"""))

SETUP_BLOCKS.add(files.BashScript(
    "schroot-shm",
    description="Fixup /dev/shm mount. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=728096 (compat(schroot < 1.6.10-3))",
    snippet="""\
# Pre 1.99.19, this used to live here: /etc/schroot/setup.d/15mini-buildd-workarounds

# '/dev/shm' might be a symlink to '/run/shm' in some
# chroots. Depending on the system mini-buildd/sbuild runs on
# and what chroot is being build for this may or may not lead to
# one or both of these problems:
#
# - shm not mounted properly in build chroot (leading to build errors when shm is used).
# - shm mount on *the host* gets overloaded (leading to an shm mount "leaking" on the host).
#
# This workaround
#
# - [in '/etc/schroot/mini-buildd/fstab-generic'] Removes shm form the generic sbuild mounts (avoids the mount leaking).
# - [here] Fixes /dev/shm to a directory in case it's not.
# - [here] Always mounts /dev/shm itself.
#
# Debian Bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=728096
#
# When this is fixed (and we have added a versioned depends on
# schroot accordingly), all of the above may be reverted again.
#
printf "=> Fixing up /dev/shm mount (Bug #728096):\\n"

if [ -L "/dev/shm" ]; then
  printf "Removing /dev/shm symlink...\\n"
  rm -v "/dev/shm"
fi
mkdir -v -p "/dev/shm"
mount -v -ttmpfs none "/dev/shm"
"""))

SETUP_BLOCKS.add(files.ShScript(
    "sun-java6-license",
    description="Accept sun-java6 licence (debconf) so we can build-depend on it (sun-java is no longer part of Debian since 2011)",
    snippet="""\
echo "sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections --verbose
echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections --verbose
echo "sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections --verbose
"""))


class SBuild():
    BUILDLOG_STATUS_REGEX = re.compile("^(Status|Lintian|Piuparts|Autopkgtest|Build-Time|Package-Time|Space|Build-Space): [^ ]+.*$")

    def __init__(self, breq, extra_args):
        self.breq = breq
        self.extra_args = extra_args
        self.bres = self.breq.gen_buildresult()
        self.call = None

    def run(self):
        """
        Run sbuild && update buildresult from build log

        .. note:: This will iterate all lines of the build log,
                  and parse out the selection of sbuild's summary status
                  we need.  In case the build log above does write the
                  same output like 'Status: xyz', sbuild's correct status
                  at the bottom will override this later.  Best thing,
                  though, would be if sbuild would eventually provide a
                  better way to get these values.
        """
        self.breq.untar(self.breq.dir_path())

        config_path = os.path.join(self.breq.dir_path(), CONFIG_DIR)
        sbuildrc_file_path = os.path.join(self.breq.dir_path(), ".sbuildrc")
        buildlog_file_path = os.path.join(self.breq.dir_path(), self.breq.dfn.buildlog(self.breq["architecture"]))

        #
        # Generate .sbuildrc
        #
        sbuildrc = files.PerlModule(".sbuildrc", placeholders={"LIBDIR": config.ROUTES["libdir"].path.new_sub([str(self.breq.dist.codename), self.breq["architecture"]], create=True).full})

        # From distribution config
        CONFIG_BLOCKS.configure(sbuildrc, self.breq.cget(CONFIG_BLOCKS.name, CONFIG_BLOCKS.default))

        # Custom script from distribution config
        sbuildrc.add_file(os.path.join(config_path, CONFIG_SBUILDRC_SNIPPET), description="Custom sbuild config (from Distribution)")

        sbuildrc.save_as(sbuildrc_file_path)

        #
        # Generate setup.d/
        #
        setup_d = files.Dir(os.path.join(self.breq.dir_path(), SETUP_D_DIR), enumerate_file_names=0)

        # From distribution config
        SETUP_BLOCKS.configure(setup_d, self.breq.cget(SETUP_BLOCKS.name, SETUP_BLOCKS.default))

        # Custom script from distribution config
        custom_buildrequest = os.path.join(config_path, CONFIG_CHROOT_SETUP_SCRIPT)
        if os.path.getsize(custom_buildrequest) > 0:
            setup_d.add(files.AutoScript(custom_buildrequest))

        setup_d.save()

        #
        # Generate command line
        #
        cmdline = [
            "sbuild",
            "--dist", self.breq["distribution"],
            "--arch", self.breq["architecture"],
            "--chroot", self.breq.schroot_name(),
            "--build-dep-resolver", self.breq.cget("Build-Dep-Resolver"),
            "--nolog",
            "--log-external-command-output",
            "--log-external-command-error",
            "--no-apt-update",
        ]

        for name in setup_d.keys():
            script = os.path.join(setup_d.path, name)
            cmdline += ["--chroot-setup-command", script]

        if self.breq.cget("Arch-All"):
            cmdline += ["--arch-all"]
        else:
            cmdline += ["--no-arch-all"]  # Be sure to set explicitly: sbuild >= 0.77 does not seem to use this as default any more?

        for dependency in self.breq.cget("Add-Depends", "").split(","):
            cmdline += ["--add-depends", dependency]

        lintian_check = self.breq.check_mode("lintian")
        if lintian_check.mode.value > 0:
            cmdline += ["--run-lintian"]
            cmdline += ["--lintian-opts", lintian_check.lintian_options(self.breq.dist.codename)]
            cmdline += ["--lintian-opts", self.breq.check_extra_options("lintian")]
        else:
            cmdline += ["--no-run-lintian"]

        piuparts_check = self.breq.check_mode("piuparts")
        if piuparts_check.mode.value > 0:
            cmdline += ["--run-piuparts"]
            cmdline += ["--piuparts-opts", f"--schroot {self.breq.schroot_name()} --dpkg-force-confdef --warn-on-debsums-errors --warn-on-leftovers-after-purge --warn-on-others --keep-sources-list"]
            cmdline += ["--piuparts-opts", self.breq.check_extra_options("piuparts")]
        else:
            cmdline += ["--no-run-piuparts"]

        autopkgtest_check = self.breq.check_mode("autopkgtest")
        if autopkgtest_check.mode.value > 0:
            cmdline += ["--run-autopkgtest"]
            cmdline += ["--autopkgtest-opts", f"-- schroot {self.breq.schroot_name()}"]
            cmdline += ["--autopkgtest-root-args", ""]
        else:
            cmdline += ["--no-run-autopkgtest"]

        if "sbuild" in config.DEBUG:
            cmdline += ["--debug"]

        cmdline += self.extra_args

        cmdline += [self.breq.dfn.dsc()]

        # Generate Environment
        env = {
            "HOME": self.breq.dir_path(),
            "GNUPGHOME": config.ROUTES["home"].path.join(".gnupg"),
            "DEB_BUILD_OPTIONS": self.breq.cget("Deb-Build-Options", ""),
            "DEB_BUILD_PROFILES": self.breq.cget("Deb-Build-Profiles", ""),
        }

        # Run sbuild
        with util.fopen(buildlog_file_path, "w+") as buildlog, subprocess.Popen(
                cmdline,
                cwd=self.breq.dir_path(),
                env=call.taint_env(env),
                stdout=buildlog,
                stderr=subprocess.STDOUT) as self.call:
            LOG.info(f"{self.breq}: Running sbuild from {self.breq.dir_path()} (pid={self.call.pid})")
            retval = self.call.wait()

        # Update bres from buildlog
        self.bres.add_file(buildlog_file_path)
        with util.fopen(buildlog_file_path, errors="replace") as f:
            for line in f:
                if self.BUILDLOG_STATUS_REGEX.match(line):
                    LOG.debug(f"Build log line detected as build status: {line.strip()}")
                    s = line.split(":")
                    self.bres.cset("Sbuild-" + s[0], s[1].strip())

        # Guarantee to set 'error' sbuild status if retval != 0 and 'Sbuild-Status' could not be parsed from buildlog
        if retval != 0 and not self.bres.cget("Sbuild-Status"):
            self.bres.cset("Sbuild-Status", "error")

        LOG.debug(f"{self.breq}: Sbuild finished: {self.bres.cget('Sbuild-Status')}")

        return self.bres

    def cancel(self, who):
        if self.call is not None:
            LOG.info(f"Terminating {self.breq} (sbuild process '{self.call.pid}')...")
            self.bres.cset("Canceled-By", who)
            self.call.terminate()

            # Wait some time (max 6 seconds) to see if terminate() succeeds
            for i in range(4):
                time.sleep(i)
                if self.call.poll() is not None:
                    return

            # I guess we should kill, then
            LOG.warning(f"Killing {self.breq} (sbuild process '{self.call.pid}')...")
            self.call.kill()
