# Common bracket library code

# Copyright (c) 2008-2023 Andreas Gustafsson.  All rights reserved.
# Please refer to the file COPYRIGHT for detailed copyright information.

from __future__ import print_function

import base64
import datetime
import fcntl
import importlib
import os
import shutil
import stat
import string
import subprocess
import sys
import time
import operator
import re
import weakref
import itertools
import gzip
import xml.parsers.expat
import textwrap
import email.utils
import multiprocessing
import ast
import traceback
import json
import random

from functools import reduce
from importlib import import_module

# Find a function for quoting shell commands
try:
    from shlex import quote as sh_quote
except ImportError:
    from pipes import quote as sh_quote

# Not available on Debian 7
try:
    import sysctl
except:
    pass

from bisect import bisect_right, bisect_left
from datetime import datetime
from glob import glob
from collections import defaultdict

from utils import mkdir_p, none2empty, ts2py, py2ts, rmtree_harder

from utils import UTC, totimestamp, ts2rcs, rcs2ts, ts2cvs, \
    read_file, write_file, append_to_file, mid_first

from __main__ import libdir
from __main__ import libpydir

EMAIL_RE = re.compile(r'(.*)<[^>]*@.*>.*')

# Remove the email address part of a hg-style committer string.
# This is intended to reduce the amount of spam received by the
# commmitters by making it slightly less trivial to harvest email
# addresses from the HTML reports.

def remove_email(committer):
    m = EMAIL_RE.match(committer)
    if m:
        return m.group(1)
    return committer

# Read a bracket.conf file and return it as a dictionary.  If the
# optional argument "config" is given, use it as the initial contents.

def read_config(fn, config = None):
    if config == None:
        config = {}
    f = open(fn, "r")
    for line in f:
        line = line.rstrip()
        if line == '':
            continue
        if line[0] == '#':
            continue
        # print line
        if line.startswith('. '):
            # Handle include file
            fn = line[2:]
            config = read_config(fn, config)
            continue
        a = line.split("=", 1)
        k, v = a
        #v = re.sub(r'^"(.*)"$', r'\1', v)
        v = ast.literal_eval(v)
        v = string.Template(v).substitute(config)
        config[k] = v
    f.close()
    return config

# Migrate a configuration item to a new name for backwards
# compatibility

def config_migrate(old_key, new_key):
    o = config.get(old_key)
    n = config.get(new_key)
    if o and not n:
        config[new_key] = o
        del config[old_key]

config_fn = "bracket.conf"

config = read_config(config_fn)
config_migrate('notify_to', 'notify_build_to')
config_migrate('notify_envelope_to', 'notify_build_envelope_to')

def config_get(key, default):
    t = config.get(key)
    if t is None:
        return default
    else:
        return t

def config_or_empty(key):
    return config.get(key, "")

# Quote a shell command.  This is intended to make it possible to
# manually cut and paste logged command into a shell.  From anita.

def quote_shell_command(v):
    s = ''
    for i in range(len(v)):
        if i > 0:
            # Try to keep options and their arguments on the same line
            if v[i - 1].startswith('-') and not v[i].startswith('-'):
                s += ' '
            else:
                s += ' \\\n    '
        s += sh_quote(v[i])
    return s

def run(cmd, verbose = True):
    if verbose:
        print(cmd)
        sys.stdout.flush()
    return subprocess.call(cmd, shell = True)

# Like subprocess.call, but prints the command when verbose.
# If verbose is an integer, use it as a limit on the message length.

def runv(cmdv, verbose = True, **kwargs):
    if verbose:
        text = quote_shell_command(cmdv)
        # Note that isinstance(verbose, int) won't work because
        # bools are ints.
        if not isinstance(verbose, bool) and len(text) > verbose:
            text = text[0:verbose] + "..."
        print(text)
        sys.stdout.flush()
    return subprocess.call(cmdv, **kwargs)

class PrerequisiteFailed(Exception):
    def __init__(self, name):
        self.name = name
    def __str__(self):
        return "prerequisite failed: " + self.name
    pass

def sense2noun(sense):
    if sense:
        return "success"
    else:
        return "failure"

def sense2past(sense):
    if sense:
        return "succeeded"
    else:
        return "failed"

# Search for the point in time when something broke.
# Invariant: it's working at c0, but broken at c1
# Return a pair of commit numbers (c0, c1), being
# the closest bracketing of the breakage point we could
# achieve.

def search(c0, c1, op, sense, level):
    print("searching for", str(op), sense2noun(not sense), "->", \
    sense2noun(sense), "transition between", \
        commit2human(c0), "and", commit2human(c1), \
        "(distance %i)" % (c1 - c0))

    assert(c1 - c0 >= 1)
    if c1 - c0 == 1:
        # These are adjacent commits - we are done
        return (c0, c1)

    i = 0
    for c in generate_test_point(c0, c1):
        try:
            t = cno2ts(c)
            print("running test", op)
            r = not not test(op, t)
            print("test result:", r)
            if r != sense:
                return search(c, c1, op, sense, level + 1)
            else:
                return search(c0, c, op, sense, level + 1)
        except PrerequisiteFailed as exc:
            print("could not run test %s at %s: %s" % \
                (op, ts2rcs(t), str(exc)))
        i = i + 1
        if i == 6:
            print("too many prerequisite failures, giving up")
            break

    return (c0, c1)

def variant_no_leading_dash():
    variant = config.get('variant', '')
    if variant is None or variant == "":
        return None
    if variant[0] == '-':
        variant = variant[1:]
    return variant

# Return a string like i386-debug

def arch_variant():
    s = config['arch']
    variant = variant_no_leading_dash()
    if variant:
        s += '-' + variant
    return s

# Return a string uniquely identifying the build including
# architecture and variant, like 2017.11.28.15.31.33-amd64-debug

def build_id_string(ts):
    parts = [ts2rcs(ts), arch_variant()]
    jobid = config.get('jobid')
    if jobid:
        parts.append(jobid)
    return '-'.join(parts)

def build_dir(ts):
    return os.path.join(config['build_root'], build_id_string(ts))

def test_dir(ts):
    return os.path.join(config['test_root'], ts2rcs(ts))

def release_dir(ts):
    r = config.get('release_root')
    if r:
        keep_n_str = config.get('keep_release_dirs') or config['keep_test_dirs']
        clean_subdirs(r, int(keep_n_str), 'release')
        return(os.path.join(r, ts2rcs(ts)))
    return os.path.join(test_dir(ts), 'release')

# Use a subdirectory per year to avoid running into the limit
# of 32767 subdirectories per directory.  If this is changed,
# the glob pattern in existing_build_dates() must also be changed.
# This can be disabled for backwards compatibility by setting
# use_subdir_per_year=0 in the configuration.

use_subdir_per_year = bool(int(config_get('use_subdir_per_year', '1')))

if use_subdir_per_year:
    date_hieararchy_dir_pattern = '2*/2*'
else:
    date_hieararchy_dir_pattern = '2*'

# Map a timestamp "ts" into a list of directory or URL components.

def date_hierarchy_list(ts):
    rcsdate = ts2rcs(ts)
    if use_subdir_per_year:
        year_part = rcsdate.split('.')[0]
        return [year_part, rcsdate]
    else:
        return [rcsdate]

# Map a timestamp "ts" into a directory under "root".

def date_hierarchy_dir(root, ts):
    return os.path.join(*[root] + date_hierarchy_list(ts))

def results_dir_r(config, ts):
    return date_hierarchy_dir(config['results_root'], ts)

def results_dir(ts):
    return results_dir_r(config, ts)

def status_fn(ts, tag):
    return os.path.join(results_dir(ts), tag)

def db_fn_r(config, ts):
    return os.path.join(results_dir_r(config, ts), "bracket.db")

def db_fn(ts):
    return db_fn_r(config, ts)

def build_log_tail_gz_fn(ts):
    return os.path.join(results_dir(ts), "build.log.tail.gz")

def db_put(ts, tag, value):
    append_to_file(db_fn(ts), tag + "=" + value + "\n", mode = 't')

# Record a configuration setting in the database
def db_put_config(ts, tag):
    value = config.get(tag)
    if value is not None:
       db_put(ts, tag, value)

def cache_status(ts, tag, status):
    # write_file(status_fn(ts, tag), str(status))
    strval = str(status)
    print("caching %s=%s" % (tag, strval))
    db_put(ts, tag, strval)

def bool2status(boolval):
    return 1 - (boolval + 0)

def status2bool(status):
    return status == 0

def cache_bool(ts, tag, boolval):
    cache_status(ts, tag, bool2status(boolval))

def get_db_r(config, ts):
    dbfn = db_fn_r(config, ts)
    try:
        data = read_file(dbfn, mode = 't')
    except IOError as e:
        if e.errno == 2:
            return { }
        else:
            raise
    try:
        lines = data.splitlines()
        r = { }
        for line in lines:
            try:
                left, right = line.split("=", 1)
            except ValueError:
                print("warning: ignoring malformed line in %s: %s", db_fn, line)
                continue
            r[left] = right
        return r
    except Exception as e:
        raise RuntimeError("could not decode %s: %s" % (dbfn, str(e)))

def get_db(ts):
    return get_db_r(config, ts)

def get_cached_status(ts, tag):
    map = get_db(ts)
    return int(map[tag])

# Get a cached value from the database as a string,
# or return None if no value has been cached

def get_cached_if_any_r(config, ts, tag):
    try:
        map = get_db_r(config, ts)
    except IOError:
        return None
    r = map.get(tag)
    if r is None:
        return None
    return r

def get_cached_if_any(ts, tag):
    return get_cached_if_any_r(config, ts, tag)

# Ditto for an integer return status

def get_cached_status_if_any_r(config, ts, tag):
    r = get_cached_if_any_r(config, ts, tag)
    if r is None:
        return None
    return int(r)

def get_cached_status_if_any(ts, tag):
    return get_cached_status_if_any_r(config, ts, tag)

# Ditto converted to a boolean

def get_cached_bool_if_any(ts, tag):
    status = get_cached_status_if_any(ts, tag)
    if status == None:
        return None
    else:
        return status2bool(status)

def have_cached_failure(ts, tag):
     status = get_cached_status_if_any(ts, tag)
     if status is None:
         return False
     return status != 0

def branch_name(config):
    return config.get('branch', vc.default_branch())

ops_by_name = { }

# A TestOp represents a runnable test along with some meta-information:
# its prerequisites (other TestOps that must be successfully run before
# this test can be run) and its status tag, the name under which the test
# result is stored in bracket.db.
#
# The runnable test (do_test) shall return a true value on success
# (as describe under "force" below) and False on failure.
#
# The fastcheck function is usually not needed, as most tests store
# their result in bracket.db under the status_tag, but when they
# don't, as in the case of the ATF tests, it can be provided as a way
# of checking for existing results.

class TestOp:
    def __init__(self, do_test, prereqs, status_tag = None, name = None,
                 report_func = None, fastcheck = None):
        self.do_test = do_test
        self.prereqs = prereqs
        if status_tag is None and name is not None:
            status_tag = name + '_status'
        self.status_tag = status_tag
        self.name = name
        self.report_func = report_func
        self.fastcheck = fastcheck
        ops_by_name[name] = self

    def __str__(self):
        if self.name is not None:
            return self.name
        else:
            return self.do_test.__name__

# There are three operations we can do using a TestOp: fastcheck, test,
# and force.
#
#   "fastcheck": quickly determine whether it is known to have succeeded,
#   known to have failed, has yet to be tested, or is known to be untestable.
#   "quickly" means based on cached information only, not running any actual
#   tests.  Returns True, False, or None, or throws a PrerequisiteFailed
#   exception.
#
#   "test": Determine, using cached information or by running tests
#   test, whether the op succeeds, fails, or can't be tested due to failed
#   prerequisites.  Returns a true value (as described under "force" below),
#   or False, or throws a PrerequisiteFailed exception.
#
#   "force": Like "test", but runs the test even if we already know it
#   will succeed, because we need output other than just the success/failure
#   status.  Returns a true value on success; this is not necessarily
#   the value True, but can be an object reference that acts as a
#   proxy for the "other output" for purposes of cleaning up temporary
#   files; as long as a at least one reference to the object exists, the
#   "other output" will remain accessible in the file system.  Returns
#   False on failure, or raises a PrerequisiteFailed exception if a
#   prerequisite failed.
#
#   "rerun": Like "force", but runs the test even if we already know it wil
#   fail.

def fastcheck(op, ts):
    #print("doing fastcheck for", op)
    if op.status_tag:
        r = get_cached_bool_if_any(ts, op.status_tag)
        #print("r =", r)
        if r is not None:
            return r
    if op.fastcheck:
        r = op.fastcheck(ts)
        if r is not None:
            return r
    for p in op.prereqs:
        if fastcheck(p, ts) == False:
            raise PrerequisiteFailed(str(p))
    return None

# Common code for test() and force()

def op_run(op, ts):
    print("running test for", op)
    r = op.do_test(ts)
    print("done running test for", op)
    if op.status_tag:
        cache_bool(ts, op.status_tag, not not r)
    # Send email notifications if appropriate
    if op.report_func:
        op.report_func(ts)
    return r

def test(op, ts):
    #print("testing", op)
    mkdir_p(results_dir(ts))
    r = fastcheck(op, ts)
    if r != None:
        return r
    #print("doing prereqs for", op)
    for p in op.prereqs:
        result = force(p, ts)
        if not result:
            raise PrerequisiteFailed(str(p))
    #print("doing", op)
    return op_run(op, ts)

def force(op, ts):
    #print("forcing", op)
    r = fastcheck(op, ts)
    if r == False:
        #print("fastcheck false")
        return r
    return rerun(op, ts)

def rerun(op, ts):
    #print("prereqs")
    for p in op.prereqs:
        result = force(p, ts)
        if not result:
            raise PrerequisiteFailed(str(p))
    # Keep prereq_results in scope while running the test
    #print("running test for", op)
    return op_run(op, ts)

# Returns a cno or None

def find_closest_successful_build(target_cno):
    build_cnos = [ts2cno(ts) for ts in existing_build_dates_at_commits()]
    def distance(cno):
        return abs(cno - target_cno)
    build_cnos.sort(key = distance)
    for c in build_cnos:
        ts = cno2ts(c)
        if get_cached_status_if_any(ts, 'build_status') == 0:
            print("closest known good build is %s" % commit2human(c))
            return c
    return None

# Find the latest build satisfying predicate.  Builds not at a commit
# according to the current clustering are ignored.

def find_latest_build(predicate):
    use_current_repository()
    for ts in reversed(existing_build_dates_at_commits()):
        if predicate(ts):
            return ts
    return None

# Find the latest successful build.  Builds not at a commit
# according to the current clustering are ignored.

def find_latest_successful_build():
    def pred(ts):
        return get_cached_status_if_any(ts, 'build_status') == 0
    return find_latest_build(pred)

# Find the latest completed build.  Builds not at a commit
# according to the current clustering are ignored.

def find_latest_completed_build():
    def pred(ts):
        return get_cached_status_if_any(ts, 'build_status') is not None
    return find_latest_build(pred)

def hints():
    hints_file = config.get('hints_file')
    if hints_file is None:
        return []
    try:
        f = open(hints_file, "r")
    except FileNotFoundError:
        return []
    hints = [int(line) for line in f.readlines()]
    f.close()
    return hints

def hint_cnos():
    return [ts2cno(ts) for ts in hints() if ts_at_commit(ts)]

def hint_cnos_twosided():
    # http://weblogs.asp.net/george_v_reilly/archive/2009/03/24/flattening-list-comprehensions-in-python.aspx
    tmp = sum([[cno - 1, cno] for cno in hint_cnos()], [])
    r = [a for a, b in itertools.groupby(tmp)]
#    for c in r:
#        print c, cno2ts(c), ts2rcs(cno2ts(c))
    return r

def find_hint_between(c0, c1):
    tmp = [c for c in hint_cnos_twosided() if c > c0 and c < c1]
    if len(tmp) > 0:
        return tmp[len(tmp) // 2]
    else:
        return None

def find_closest_hint(target_cno):
    hint_cnos = hint_cnos_twosided()
    def distance(cno):
        return abs(cno - target_cno)
    hint_cnos.sort(key = distance)
    if len(hint_cnos):
        r = hint_cnos[0]
    else:
        r = None
    return r

# Generate a sequence of commit numbers between commit numbers c0 and
# c1 (both non-inclusive) where we can run a test.  Prefer commit
# numbers that we know have successfully built in the past.

def generate_test_point(c0, c1):
    if c1 - c0 <= 1:
        # Nothing inbetween
        return

    for c in mid_first(c0 + 1, c1):
        print("midpoint of interval is %s" % ts2rcs(cno2ts(c)))

# This has issues: as successive test points are generated,
# they can all end up being replaced by the same known successful
# build.
#
#        cb = find_closest_successful_build(c)
#        if cb is not None:
#            if cb >= c0 + 1 and cb < c1:
#                print("using nearby time of known successful build: %s" % \
#                      ts2rcs(cno2ts(cb)))
#                c = cb
#            else:
#                print("out of range, not using it")

        yield c

# The build directory lock

def build_dir_lock_fn(ts):
    return dir_lock_fn(build_dir(ts))

def dir_lock_fn(dir):
    head, tail = os.path.split(dir)
    return os.path.join(head, "lock-" + tail)

# Run the test against commit "c", return true iff successful

def run_test(c, func):
    print("running test for %s" % commit2human(c))
    t = cno2ts(c)
    success = func(t)
    print("test %x result: %s" % (t, success))
    return success

def anita_dist_url(ts):
    d = release_dir(ts)
    arch = config['arch']
    if arch in ['sparc', 'sparc64', 'vax', 'hppa']:
        # The name of the ISO is no longer fixed as of 2012-02-16,
        # so we have to resort to globbing to find it.
        # As of 2012-08, the ISOs have moved from relase/iso
        # to release/images, so we need to glob that, too.
        isos = glob(os.path.join(d, 'release', '*', '*.iso'))
        if len(isos) == 0:
            raise RuntimeError("no install ISO")
        if len(isos) != 1:
            raise RuntimeError("will the real install ISO please stand up")
        iso = isos[0]
        url = "file://" + iso
    else:
        url = "file://%s/release/%s/" % (d, arch)
    return url

def anita_workdir(ts):
    return os.path.join(test_dir(ts), 'anita')

# Get the configuration value for key, or an empty string
# if not set.  If returning a nonempty string, store that
# fact in the database.

def config_or_empty_store(ts, key):
    val = config_or_empty(key)
    if val:
        db_put_config(ts, key)
    return val

# Return a string to pass after "--vmm-args=".
# "command" is install, boot...

def anita_vmm_args(ts, command):
    return (
        config_or_empty_store(ts, 'extra_vmm_' + command + '_args') or
        config_or_empty_store(ts, 'extra_qemu_' + command + '_args') or
        config_or_empty_store(ts, 'extra_vmm_args') or
        config_or_empty_store(ts, 'extra_qemu_args')
    )

# Return an array of arguments to pass on the anita command line

def anita_args(ts, command):
    return (config_or_empty_store(ts, 'extra_anita_' + command + '_args') or
         config_or_empty_store(ts, 'extra_anita_args')).split()

# Returns a bool

def anita_run_2(ts, command, logfile, extra_anita_args = []):
    url = anita_dist_url(ts)
    workdir = anita_workdir(ts)
    status = runv(['time', 'anita',
        '--workdir=' + workdir,
        '--structured-log-file=' + os.path.join(test_dir(ts), command + '.slog'),
        '--vmm-args=' + anita_vmm_args(ts, command)] +
        anita_args(ts, command) +
        extra_anita_args +
        [command, url], stdout=logfile, stderr=logfile)
    return status2bool(status)

# Returns a bool

def anita_run_1(ts, command, logfile_name, extra_anita_args = []):
    logfile = open(logfile_name, "w")
    return anita_run_2(ts, command, logfile, extra_anita_args)

# Returns a bool

def anita_run(ts, command):
    logfile = os.path.join(test_dir(ts), command + '.log')
    r = anita_run_1(ts, command, logfile)
    run('gzip <%s >%s' % \
        (logfile, os.path.join(results_dir(ts), command + '.log.gz')))
    return r

class NoCommitAtTimestamp(Exception):
    pass

class NoCommitExactlyAtTimestamp(Exception):
    pass

# Convert a timestamp to a commit cluster number, loosely, that is,
# choosing the last commit cluster at or before the given time

def ts2cno_loose(ts):
    global dates
    r = bisect_right(dates, ts) - 1
    if r < 0:
        raise NoCommitAtTimestamp()
    return r

# Convert a timestamp to a commit cluster number, strictly, that is,
# the timestamp must match the first commit in an existing cluster
# exactly.

def ts2cno(t):
    r = ts2cno_loose(t)
    if dates[r] != t:
        #print "warning: timestamp not at cluster end:", ts2rcs(t), t
        raise NoCommitExactlyAtTimestamp()
    return r

# Return True iff the given timestamp matches the last commit in
# a commit cluster

def ts_at_commit(t):
    try:
       r = ts2cno_loose(t)
    except NoCommitAtTimestamp:
        # This can happen
        return False
    return dates[r] == t

def ts_adjust(ts):
    use_repository_for(ts)
    return cno2ts(ts2cno_loose(ts))

def ts_adjust_verbose(ts):
    new_ts = ts_adjust(ts)
    if new_ts != ts:
        print("no commit cluster ending at %s, adjusting to %s" % (ts2rcs(ts), ts2rcs(new_ts)))
    return new_ts

def get_commit_clusters():
    global dates
    return dates

def cno_valid(c):
    return c >= 0 and c < len(dates)

# Convert a commit cluster number to a timestamp

def cno2ts(c):
    global dates
    return dates[c]

def commit2human(c):
    t = cno2ts(c)
    return "commit #%d at %s" % (c, ts2rcs(t))

# Create installation file sets, or return false if we can't

class TooManyWorkDirs(Exception):
    pass

# directory -> directories, etc.

def pluralize(s):
    return re.sub(r'y$', 'ie', s) + 's'

def clean_subdirs(parent, keep_n, label):
    #print("cleaning subdirs of %s, keeping %d" % (parent, keep_n))
    # First clean out stale lock files, taking care to disable
    # recursive locking so we don't clean out our own locks just
    # because we are able to take them (again).
    locks = glob(os.path.join(parent, 'lock-*'))
    for lockfile in locks:
        lockfd = trylock(lockfile, recursive = False)
        if lockfd:
            print("removing stale lock", lockfile)
            os.unlink(lockfile)
            lockfd.close()
        else:
            # print("keeping active lock", lockfile)
            pass

    dirs = [f for f in glob(os.path.join(parent, '*'))
            if f[-5:] != '.lock' and os.path.isdir(f)]
    tmp = [ { 'dir': d, 'mtime': os.stat(d)[stat.ST_MTIME] } for d in dirs ]
    oldest_first =  sorted(tmp, key = lambda a: a['mtime'])

    have_n = len(oldest_first)
    # The number of directories to remove
    remove_n = have_n - keep_n

    #print("have %d subdirs, will keep %d, attempting to remove %d" %
    #      (have_n, keep_n, remove_n))

    for item in oldest_first:
        d = item['dir']
        if remove_n <= 0:
            break
        lockfile = dir_lock_fn(d)
        lockfd = trylock(lockfile)
        if lockfd:
            t0 = time.time()
            print("removing old %s %s..." % (label, d), end=' ')
            sys.stdout.flush()
            rmtree_harder(d)
            t1 = time.time()
            print("done, took %i seconds" % (t1 - t0))
            remove_n -= 1
            have_n -= 1
            del lockfd
        else:
           #print("%s is locked, not removing" % d)
           pass

    if have_n > keep_n:
        raise TooManyWorkDirs()

def choose_patches_or_blacklists(ts, dir, pattern):
    cvsdate = ts2rcs(ts)
    patch_path = [libdir, '.']
    patches = []
    for parent in patch_path:
        patches += glob(os.path.join(parent, dir, pattern))
    patches.sort()
    r = []
    for p in patches:
        parts = os.path.basename(p).split("-")
        if cvsdate >= parts[0] and cvsdate < parts[1]:
            r.append(p)
    return r

def choose_patches(ts):
    return choose_patches_or_blacklists(ts, 'patches', '*.patch')

def choose_blacklists(ts):
    return choose_patches_or_blacklists(ts, 'blacklists', '*.bl')

# Split an arch (as understood by bracket) into a MACHINE and
# a MACHINE_ARCH

def split_arch(arch):
    parts = arch.split('-')
    if len(parts) == 1:
        return (parts[0], None)
    else:
        return parts

tar_z_exts = ['.tgz', '.tar.xz']

def have_sets(ts):
    reldir = release_dir(ts)
    arch = config['arch']
    if arch == 'evbarm-earmv7hf':
        representative_set = os.path.join(reldir, \
            "release/evbarm-earmv7hf/binary/gzimg/armv7.img.gz")
        if os.path.exists(representative_set):
            return True
    elif arch == 'evbarm-aarch64':
        representative_set = os.path.join(reldir, \
            "release/evbarm-aarch64/binary/gzimg/arm64.img.gz")
        if os.path.exists(representative_set):
            return True
    else:
        for ext in tar_z_exts:
            representative_set = os.path.join(reldir, \
                "release/%s/binary/sets/base%s" % (arch, ext))
            print("looking for", representative_set)
            if os.path.exists(representative_set):
                return True
    return False

# Check out the source for "ts" from the repository and apply patches
# This also creates the build dir as we need it to check out into.

def checkout(ts):
    builddir = build_dir(ts)
    if os.path.exists(builddir):
        rmtree_harder(builddir)
    mkdir_p(builddir)

    repo_lock = lock(repo_lock_fn(), exclusive = False, verbose = True)

    modules = config_get('repo_modules', 'src').split(',')
    for module in modules:
        print("doing checkout of %s for %s..." % (module, ts2rcs(ts)), end=' ')
        sys.stdout.flush()

        logfn = os.path.join(builddir, 'checkout-%s.log' % module)
        logfd = os.open(logfn, os.O_WRONLY | os.O_CREAT)
        if logfd < 0:
            raise RuntimeError("could not open %s", logfn)
        branch = branch_name(config)
        t0 = time.time()
        status = vc.checkout(branch, module, ts, builddir, logfd)
        t1 = time.time()
        os.close(logfd)
        if status != 0:
            raise RuntimeError("checkout of %s failed; see %s" % (module, logfn))
        print("done, took %i seconds" % (t1 - t0))

    del repo_lock

    # Apply special patches specified with patch_file, etc.
    # These are not the usual patches.
    patch_dir = config.get('patch_dir')
    patch_file = config.get('patch_file')
    patch_options = config.get('patch_options', '-f -p0')
    if patch_dir is not None and patch_file is not None:
        cmd = 'cd %s/%s && patch %s <%s' % (build_dir(ts), patch_dir, patch_options, patch_file)
        status = run(cmd)
        if status != 0:
            raise RuntimeError("patch did not apply")

# Patches is a list of patch file names.  If not specified, the
# patches are chosen based on the source date.

def apply_patches(ts, patches = None):
    if patches is None:
        print("choosing patches for", ts2rcs(ts))
        patches = choose_patches(ts)

    # A list each of applied and failed patch names
    patch_result = [[], []]

    srcparent = build_dir(ts)

    for p in patches:
        print("applying patch", p, "...", end=' ')
        sys.stdout.flush()
        patch_fd = os.open(p, os.O_RDONLY)
        status = runv(['patch', '-f', '-p0', '-E'],
                      cwd = srcparent, stdin = patch_fd)
        os.close(patch_fd)
        if status == 0:
            print("applied successfully")
        else:
            print("failed to apply")
            status = 1
        patch_result[status].append(os.path.basename(p))

    for status in (0,1):
        db_put(ts, 'patches_' + \
            ['applied', 'failed'][status], ','.join(patch_result[status]))

    return len(patch_result[1]) == 0

def n_make_jobs():
    n = config.get('njobs')
    if n is not None:
        return int(random.choice(n.split(',')))
    return multiprocessing.cpu_count()

# Create a directory "relname" under "parent", with a purpose
# described by the human readable "label", blocking if needed to
# ensure that no more than max_dirs such directories exist.  Returns a
# lock; the caller should "del" it when the directory is no longer
# needed.

def get_dir_limited(relname, parent, label, max_dirs):
    # First make sure the top-level build dir exists, because
    # it may be on a ramdisk that is lost on reboot.
    mkdir_p(parent)

    # Make sure we have a build directory slot available.  Locking
    # is tricky here; we must make sure two jobs don't try to
    # clean the build directory at once, and also must not deadlock.
    i = 0
    t0 = time.time()
    while True:
        #print("trying to lock build dir set")
        # Lock the set of build directories to make sure we are
        # the only one cleaning them.  Do not block holding the lock.
        # We use recursive = False because recursive = True does not
        # work correctly when closing locks explicitly.
        build_dir_set_lock = lock(parent + '.lock', recursive = False, verbose = True)
        #print("got build dir set lock")
        try:
            #print("cleaning subdirs")
            clean_subdirs(parent, max_dirs - 1, label)
        except TooManyWorkDirs:
            #print("too many %s" % pluralize(label))
            if i == 0:
                print("waiting for %s..." % label, end='')
                sys.stdout.flush()
            else:
                print(".", end='')
                sys.stdout.flush()
            build_dir_set_lock.close()
            time.sleep(60)
            i += 1
            continue
        print("not too many %s" % pluralize(label))
        # otherwise break loop holding build_dir_set_lock
        break
    if i > 0:
        t1 = time.time()
        print(" took %i seconds" % (t1 - t0))

    # Lock the build directory for the source date.  We must also
    # create the actual directory so that clean_subdirs() will count
    # it.
    builddir = os.path.join(parent, relname)
    mkdir_p(builddir)
    build_dir_lock = lock(dir_lock_fn(builddir), verbose = True)

    # Now it is safe to release the build dir set lock
    build_dir_set_lock.close()

    return build_dir_lock

# Create the build directory, waiting for a slot if needed to keep the
# disk usage on the build ramdisk bounded.  Returns the build directory
# lock; the caller should "del" it when the build directory is no longer
# needed.

def get_build_dir(ts):
    relname = build_id_string(ts)
    parent = config['build_root']
    label = 'build directory'
    max_dirs = \
        int(config_get('build_slots', '1')) + \
        int(config_get('keep_build_dirs', '0'))
    return get_dir_limited(relname, parent, label, max_dirs)

# Get a build slot.  This is to limit the CPU load and memory use of
# the build rather than the disk usage.  This abuses machinery
# originally created for limiting the number of build directories as a
# kind of generic semaphore mechanism.  This involves the needless
# creation of directories that are not actually used to store any
# files, which is a bit silly and inefficient, but it happens to have
# the right semantics with respect to things like processes dying with
# the semaphore held, so it gets the job done with a minimum of code.
# The "ts" argument is used only to generate a unique name in
# combination with the arch and variant.

def get_build_slot(ts):
    relname = build_id_string(ts)
    parent = os.path.join(config['db_dir'], 'build_slot')
    label = 'build slot'
    max_dirs = int(config_get('build_slots', '1'))
    return get_dir_limited(relname, parent, label, max_dirs)

def linux_get_cpu_brand():
    pipe = subprocess.Popen(['lscpu', '-J'], stdout=subprocess.PIPE).stdout
    j = json.load(pipe)
    a = j['lscpu']
    return [e['data'] for e in a if e[u'field'] == u'Model name:'][0]

# Edit the kernel configuration file fn, commenting out lines
# matching the regular expressions in options_to_disable
# and appending lines from options_to_enable

def edit_config(fn, options_to_disable, options_to_append):
    print('editing kernel configuration %s:' % fn)
    print("  disabling", str(options_to_disable))
    print("  enabling", str(options_to_append))
    s = read_file(fn, 't')
    for o in options_to_disable:
        pattern = r'(?=^%s)' % o
        print(pattern)
        s = re.sub(pattern, '#', s, flags = re.MULTILINE)
    for line in options_to_append:
        s += line + '\n'
    write_file(fn, s, 't')

def do_build(ts):
    if have_sets(ts):
        print("already have sets")
        return True
    print("no sets, must build")
    sys.stdout.flush()

    build_dir_lock = get_build_dir(ts)

    mkdir_p(results_dir(ts))

    print("checking blacklists for", ts2rcs(ts))
    blacklists = choose_blacklists(ts)
    if len(blacklists) > 0:
        print("blacklisted, not building")
        return False

    checkout(ts)

    clean_subdirs(config['test_root'], int(config['keep_test_dirs']), 'test')

    testdir = test_dir(ts)
    print("creating directory %s" % testdir)
    test_dir_lock = lock(dir_lock_fn(testdir), verbose = True)
    if os.path.exists(testdir):
        shutil.rmtree(testdir)
    mkdir_p(testdir)

    build_lock = get_build_slot(ts)

    # Begin code originally in do-build.sh

    print("doing build for", ts2rcs(ts))

    njobs = n_make_jobs()

    builddir = build_dir(ts)
    srcparent = builddir
    srcdir = os.path.join(srcparent, "src")

    # PR 49535
    mkdir_p(os.path.join(builddir, "obj"))

    mkdir_p(results_dir(ts))

    if not apply_patches(ts):
        raise RuntimeError("one or more patches failed to apply")

    arch = config['arch']
    machine, machine_arch = split_arch(arch)

    print("looking for config_enable")
    # Optionally disable kernel configuration lines listed in config_disable
    config_disable = config_or_empty_store(ts, 'config_disable')
    config_enable = config_or_empty_store(ts, 'config_enable')
    # Like str.split(delim), but returning an empty list for an empty string.
    # Without this, an empty config_disable would disable every option.
    def split_or_empty(s, delim):
        if s == '':
            return []
        return s.split(delim)
    disable_items = [re.sub(r' ', r'\\s+', e) for e in split_or_empty(config_disable, ',')]
    enable_items = split_or_empty(config_enable, ',')
    if disable_items or enable_items:
        edit_config(os.path.join(srcdir, 'sys', 'arch', machine, 'conf', 'GENERIC'),
                    disable_items, enable_items)

    build_target_list = config.get('build_target', 'release iso-image').split()

    debug_build_sh_flags = []

    # Support for debug builds with -fdebug-prefix-map.
    # Setting "-V MKDEBUG=yes -V MKREPRO=yes" in build_sh_flags
    # is usually preferred as that may match the unpublished
    # [PR 54544] build options of the official releases,
    # but if you specifically want gdb to find the sources
    # without MKREPRO, you can set debug_build="1".
    if int(config.get('debug_build', '0')):
       # '-V', 'MKKDEBUG=yes'
       debug_build_sh_flags = [ '-V', 'MKDEBUG=yes' ]
       # This is just a guess as to when -fdebug-prefix-map support was imported
       if ts > rcs2ts('2015.01.01.00.00.00'):
           debug_build_sh_flags.extend(['-V', 'COPTS=-g -fdebug-prefix-map=%s=/usr/src' % srcdir])

    build_sh_flags = config_or_empty_store(ts, 'build_sh_flags').split()

    releasedir = os.path.join(release_dir(ts), 'release')
    mkdir_p(releasedir)

    arch_build_sh_flags = ['-m', machine]
    if machine_arch:
        arch_build_sh_flags += ['-a', machine_arch]

    objdir = os.path.join(builddir, 'obj')

    build_sh_cmdv = \
        [ 'time', 'nice', './build.sh'] + \
        [ '-j', str(njobs) ] + \
        [ '-D', os.path.join(builddir, 'destdir') ] + \
        [ '-R', releasedir ] + \
        [ '-T', os.path.join(builddir, 'tools') ] + \
        [ '-O', objdir ] + \
        arch_build_sh_flags + \
        debug_build_sh_flags + \
        build_sh_flags + \
        [ '-U' ] + \
        build_target_list

    build_sh_env = {
        'PATH': '/bin:/usr/bin:/sbin:/usr/sbin',
        'HOST_CPPFLAGS': config_get('host_cppflags', ''),
        'MKREPRO_TIMESTAMP': str(ts),
        #'HOST_CFLAGS': '-O -g',
        #'HOST_LDFLAGS': '-g',
    }
    for l, r in build_sh_env.items():
        db_put(ts, 'build_sh_env_' + l.lower(), r)

    buildlog = os.path.join(testdir, 'build.log')
    buildlog_fd = os.open(buildlog, os.O_WRONLY | os.O_CREAT)

    db_put(ts, "build_sh_cmd", " ".join([repr(word) for word in build_sh_cmdv]))

    # Run the build
    status = runv(build_sh_cmdv, cwd = srcdir,
                  stdout = buildlog_fd, stderr = buildlog_fd,
                  env = build_sh_env)

    os.close(buildlog_fd)

    # Deal with inconsistent releasedir naming for arm
    if arch in ['evbarm-earmv7hf','evbarm-aarch64']:
        # The symlink can fail either because the version built
        # is one where the directory naming has already been fixed,
        # or because the build failed and the directory does not
        # exist.  In either case, the error should be silently
        # ignored.
        try:
            os.symlink('evbarm', os.path.join(releasedir, arch))
        except OSError as e:
            #print("symlink:", e)
            pass

    # Save a copy of the kernel debug symbols for GENERIC, if any
    try:
        # Note "machine", not "arch"; for example, the evbarm-earmv7hf
        # GENERIC debug kernel is in
        # obj/sys/arch/evbarm/compile/GENERIC/netbsd.gdb
        debug_kernel_fn = os.path.join(objdir, 'sys', 'arch', machine,
                                       'compile', 'GENERIC', 'netbsd.gdb')
        shutil.copy(debug_kernel_fn, testdir)
        print("saved debug kernel %s" % debug_kernel_fn)
    except Exception as e:
        print("no debug kernel %s" % debug_kernel_fn)

    # Keep more of the build log if the build failed
    if status:
        keep_lines = 2000
    else:
        keep_lines = 1000

    buildlog_tail = os.path.join(builddir, 'build.log.tail')
    buildlog_tail_fd = os.open(buildlog_tail, os.O_WRONLY | os.O_CREAT)
    runv(['tail', '-%d' % keep_lines, buildlog], cwd = builddir, stdout = buildlog_tail_fd)
    os.close(buildlog_tail_fd)
    runv(['gzip', '-f', buildlog_tail])
    shutil.move(os.path.join(builddir, 'build.log.tail.gz'), build_log_tail_gz_fn(ts))

    # Measure size of build log, ignoring the --- lines from make
    nlines = 0
    with open(buildlog, "rb") as f:
        for line in f:
            if line.startswith(b'---'):
                continue
            nlines += 1

    db_put(ts, 'build_log_lines', str(nlines))
    (sysname, nodename, release, version, machine) = os.uname()
    db_put(ts, 'build_host', nodename)
    db_put(ts, 'build_host_os', sysname)
    db_put(ts, 'build_host_osrel', release)
    db_put(ts, 'build_host_machine', machine)
    db_put(ts, 'build_date', str(int(time.time())))
    db_put(ts, 'build_njobs', str(njobs))

    # To help debug tmpfs performance issues
    db_put(ts, 'build_root', config['build_root'])

    # Store information about the type and number of CPUs,
    # if available
    def db_put_sysctl(name):
        try:
            value = sysctl.read(name)
        except:
            return False
        db_put(ts, 'build_' + name.replace('.', '_'), str(value))
        return True

    db_put_sysctl('hw.ncpu')
    if not db_put_sysctl('machdep.cpu_brand'):
        try:
            db_put(ts, 'build_machdep_cpu_brand', linux_get_cpu_brand())
        except Exception as e:
            print("warning: could not get cpu brand:", e)
    if using_noemu():
        # To debug possible DADdy issues
        db_put_sysctl('net.inet.ip.dad_count')
    # In case sysctl hw.ncpu is not available
    db_put(ts, 'ncpu', str(multiprocessing.cpu_count()))

    # End code originally in do-build.sh

    print("build exit status", status)
    del build_lock

    del build_dir_lock

    # We need to store the status before updating reports,
    # even if this means it gets stored twice.
    cache_bool(ts, 'build_status', status == 0)

    update_reports()

    if status == 0:
        maybe_archive_release(ts, 'built')

    if status == 0:
       return test_dir_lock
    else:
       return False

def make_report_func(name):
    def f(ts):
        return maybe_report_status_change(ts, name)
    return f

build_op = TestOp(do_build, [], name = 'build',
                  report_func = make_report_func('build'))

def noemu_build_fn(config):
    return os.path.join(config['db_dir'], 'noemu_build.' +
                        config_get('noemu_lock', None))

# See if we already have an installed machine image for ts.  This
# needs to be different between qemu and noemu; in qemu, the
# machine image is a file we can check for, but with noemu, we
# need to keep track of whether we have done an install and not
# released the noemu lock since then.

def have_image(ts):
    global g_noemu
    if using_noemu():
        if g_noemu is not None and g_noemu.installed == ts:
            print("using noemu image installed by this process")
            return True
        build_id_fn = noemu_build_fn(config)
        if os.path.exists(build_id_fn):
            build_id = read_file(build_id_fn, 't')
            if build_id == build_id_string(ts):
                print("using previously installed noemu image")
                return True
        print("no matching noemu image")
        return False
    else:
        # qemu
        # To avoid using an image still being installed, check for
        # the install_complete file in addition to wd0.img itself.
        if all([os.path.exists(os.path.join(test_dir(ts), fn))
                for fn in ('anita/wd0.img', 'install_complete')]):
            return True
    return False

# Create a virtual machine image for Anita testing

def run_anita_install(ts):
    global g_noemu

    if have_image(ts):
        print("already have installed image for %s" % ts2rcs(ts))
        return True

    print("no VM image, must install")

    testdir = test_dir(ts)
    mkdir_p(testdir)
    test_dir_lock = lock(dir_lock_fn(testdir), verbose = True, recursive = True)

    # If we are using noemu, hold a lock across the install and test to ensure
    # that the installed image is not overwritten inbetween.
    if using_noemu():
        g_noemu = Noemu()

    r = anita_run(ts, 'install')

    # If the install was unsucessful, save the packet capture, if any,
    # to debug possible netboot issues.
    if not r:
        pcap_fn_base = 'install.pcap'
        pcap_fn = os.path.join(test_dir(ts), 'anita', pcap_fn_base)
        if os.path.exists(pcap_fn):
            shutil.copyfile(pcap_fn,
                            os.path.join(results_dir(ts), pcap_fn_base))

    if r:
        # Install successful
        # Mark the install as complete for the benefit of 'latest_image',
        # so that we don't accidentally start using a wd0.img still being
        # installed
        write_file(os.path.join(test_dir(ts), 'install_complete'), '', mode = 't')
        if using_noemu():
           g_noemu.installed = ts
           # Also store a string identifying the version installed.
           # This can be useful for subsequent manual testing,
           # and could potentially also be used to avoid a needless
           # automated reinstall of the same version.
           write_file(noemu_build_fn(config), build_id_string(ts), mode = 't')
        return test_dir_lock
    else:
        # Install unsuccessful
        # Save a copy of the install.slog file for debugging
        run('gzip <%s >%s' % (
            os.path.join(test_dir(ts), 'install.slog'),
            os.path.join(results_dir(ts), 'install.slog.gz')))
        return False

install_op = TestOp(run_anita_install, [build_op], name = 'install',
                    report_func = make_report_func('install'))

def run_anita_boot(ts):
    return anita_run(ts, 'boot')
boot_op = TestOp(run_anita_boot, [install_op], name = 'boot')

def maybe_archive_release(ts, type):
    archive_root = config.get(type + '_archive_root')
    if not archive_root:
        return
    archive_dir = os.path.join(archive_root, ts2rcs(ts))
    if os.path.exists(archive_dir):
        # This can happen when we build the same version more than once
        return
    clean_subdirs(archive_root, int(config['keep_' + type + '_archive_dirs']), type + ' archive')
    shutil.copytree(test_dir(ts), archive_dir)

def run_anita_atf_test(ts):
    # This is needed when we run tests without installing first
    global g_noemu
    if using_noemu() and not g_noemu:
        g_noemu = Noemu()

    result = anita_run(ts, 'test')

    # Save the ATF results if we have any
    def atf_path(basename):
        return os.path.join(test_dir(ts), 'anita', 'atf', basename)
    for fn in ['test.tps', 'test.xml']:
        path = atf_path(fn)
        if os.path.exists(path):
            run('gzip <%s >%s' % \
                (path, os.path.join(results_dir(ts), fn + '.gz')))

    # Generate HTML report if possible
    xml_path = atf_path('test.xml')
    xsl_path = atf_path('tests-results.xsl')
    css_path = atf_path('tests-results.css')
    html_path = atf_path('test.html')
    if reduce(operator.__and__, [os.path.exists(path)
            for path in [xml_path, xsl_path, css_path]]):
        status = run('xsltproc --nonet --novalid %s %s >%s' %
            (xsl_path, xml_path, html_path))
        # ATF sometimes fails to complete the tests and leaves
        # a malformed XML file, causing xltproc to fail.  Don't
        # export an empty HTML file when that happens.
        if status == 0:
            # Ugly hack to turn PR references in the HTML report into links
            html = read_file(html_path, 't')
            html = re.sub(r'(PR ([a-z][a-z0-9-]+)?/(\d+))',
                          r'<a href="http://gnats.netbsd.org/\3">\1</a>', html)
            tmpfn = html_path + '.prsub'
            write_file(tmpfn, html, 't')
            run('gzip <%s >%s/test.html.gz' % (tmpfn, results_dir(ts)))
            os.unlink(tmpfn)
            # Must keep the silly file name because that's how it's referenced
            # in the HTML
            run('gzip <%s >%s/tests-results.css.gz' % \
                (css_path, results_dir(ts)))

    # Extract success/fail counts and store in database
    runv(['/bin/sh', os.path.join(libdir, 'extract-test-results.sh'), results_dir(ts)])

    # If we passed the test, archive the release
    if result == True:
        maybe_archive_release(ts, 'golden')

    # If we got a crash dump, save it
    status = run("grep 'savecore: writing' %s" % os.path.join(test_dir(ts), 'test.log'))
    if status == 0:
        maybe_archive_release(ts, 'panic')

    # updating reports here is no good because test_status has not yet been stored
    return result

# As above, but only require the test run to complete, not pass

def run_anita_atf_test_completed(ts):
    # If we already have cached results, we don't need to rerun the tests.
    # XXX should also cache information about attempted tests that did not complete
    # XXX with noemu, install will happen anyway
    print("checking for tests already run...", end=' ')
    if (get_cached_if_any(ts, 'passed_tests') is not None):
        print("already run, not rerunning")
        return True
    print("not yet run")
    result = run_anita_atf_test(ts)
    # This won't get cached under this name otherwise
    cache_bool(ts, 'test_status', not not result)
    return (get_cached_if_any(ts, 'passed_tests') is not None)

# test_completed_op succeeds if the tests ran to completion

test_completed_op = TestOp(run_anita_atf_test_completed,
                           [install_op], name = 'test_completed',
                           report_func = make_report_func('test_completed'))

# test_op succeeds only if all tests passed

def all_tests_passed(ts):
    return get_cached_status(ts, 'test_status') == 0

test_op = TestOp(all_tests_passed, [test_completed_op], name = 'test')

# As find_failure below, but locate either a break (when sense = False)
# or a fix (when sense = True)

def find_change(t0, t1, op, sense, confirm = True):
    use_repository_for(t1)

    c0 = ts2cno(t0)
    c1 = ts2cno(t1)

    if confirm:
        print("confirming precondition: %s at %s" % (sense2noun(not sense), commit2human(c0)))
        if (not not test(op, cno2ts(c0))) == (not sense):
            print("precondition confirmed")
        else:
            raise RuntimeError(("precondition failed: %s should have %s " +
                "at %s but didn't") % (str(op), sense2past(not sense), commit2human(c0)))

        print("confirming postcondition: %s at %s" % (sense2noun(sense), commit2human(c1)))
        # The outer parentheses below are needed because the precedence of the
        # python "not" operator is very different from that of the C "!"
        # operator.
        if (not not test(op, cno2ts(c1))) == sense:
            print("postcondition confirmed")
        else:
            raise RuntimeError(("postcondition failed: %s should have %s at %s " +
                "but didn't") % (str(op), sense2past(sense), commit2human(c1)))
    else:
        print("presumed %s after %s\npresumed %s after %s\ncommits inbetween: %d" % \
            (sense2past(not sense), commit2human(c0),
             sense2past(sense), commit2human(c1),
             c1 - c0))

    (c0, c1) = search(c0, c1, op, sense, 0)

    print("Results:")
    print("%s after %s\n%s after %s\ncommits inbetween: %d" % \
        (sense2noun(not sense), commit2human(c0),
         sense2noun(sense), commit2human(c1),
         c1 - c0))

    return (c0, c1)

# Locate the point of a build break or other failure by binary search.
# The failure is tested by running the TestOp "op".
#
# The caller must satisfy the precondition that he break is known to
# be between the times "t0" and "t1", such that the function "func"
# returns true when called with t0 and false when called with t1.  If
# "confirm" is true, this precondition will be verified by explicitly
# evaluating func at t0 and t1.
#
# If a PrerequisiteFailed exception is raised by the tst, we try a few
# other points of time between t0 and t1 before giving up, but will
# not do an exhaustive search.
#
# Return a pair of commit numbers bracketing the change.

def find_failure(t0, t1, op, confirm = True):
    return find_change(t0, t1, op, False, confirm)

# As above, but finds when breakage was fixed instead of when it was introduced

def find_fix(t0, t1, op, confirm = True):
    return find_change(t0, t1, op, True, confirm)

# Populate the "dates" array from the repo or an index.
# May also cache other repo data privately to the vc.

def read_dates(do_lock = True):
    if do_lock:
        repo_lock = lock(repo_lock_fn(), exclusive = False, verbose = True)
    else:
        repo_lock = None
    vc.read_dates()
    del repo_lock

# Main program

# Global variables

# Sorted array of commit cluster timestamps, searchable by
# binary serach
dates = None

# Map from timestamp to commit hash, for DVCSes like git and hg only.
# Empty for CVS.
ts2sha = { }

# Map of lock file names to open file descriptors.  This
# facilitates recursive locks.
locks = weakref.WeakValueDictionary()

# Current noemu object, if any
g_noemu = None

def get_dates():
    global dates
    return dates

# Get the commit number of the last commit in the respository

def last_commit_cno():
    global dates
    return len(dates) - 1

# Get the timestamp of the last commit in the repository

def last_commit_ts():
    return cno2ts(last_commit_cno())

def format_time_delta(delta):
    if delta < 0:
        return "%i seconds before" % -delta
    else:
        return "%i seconds after" % delta

def last_safe_commit_ts():
    return vc.last_safe_commit_ts()

# Get the timestamps of all existing builds

def existing_build_dates_r(config):
    dirs = glob(os.path.join(config['results_root'], date_hieararchy_dir_pattern))
    if len(dirs) == 0:
        print("warning: found no existing builds - check use_subdir_per_year setting")
    dirs.sort()
    return [rcs2ts(os.path.basename(d)) for d in dirs]

def existing_build_dates():
    return existing_build_dates_r(config)

# As above, but only those whose timestamp corresponds to a commit cluster

def existing_build_dates_at_commits():
    return [ts for ts in existing_build_dates() if ts_at_commit(ts)]

class RepositoryUpdateFailed(Exception):
    pass

def repo_lock_fn():
    return os.path.join(config['repo_root'], 'lock')

# A script that makes use of the repository must call
# one of the three following functions to set it up.

# Use the local repository copy without updating it from the master

def use_current_repository():
    global dates
    if dates is None:
        read_dates()

def time_func(f, desc):
    print(desc + "...", end=' ')
    sys.stdout.flush()
    t0 = time.time()
    r = f()
    t1 = time.time()
    print("done, took %i seconds" % (t1 - t0))
    return r

# Update our local copy of the repository

def update_repository():
    modules = config_get('repo_modules', 'src').split(',')
    repo_lock = lock(repo_lock_fn(), exclusive = True, verbose = True)

    for module in modules:
        def f():
            vc.update_repo_module(module)
        time_func(f, "updating repository module '%s'" % module)

    def f():
        vc.index_repo()
    time_func(f, "indexing repository")

    del repo_lock
    # Update dates[] whether already set or not
    read_dates()

# Make sure the repository covers the time "ts", updating
# it from the master if necessary

def use_repository_for(ts):
    use_current_repository()
    if last_commit_ts() < ts:
        update_repository()
    if last_commit_ts() < ts:
        raise RuntimeError("master repository does not yet contain the date %s" % ts2rcs(ts))

class Commit:
    def __repr__(self):
        return "<commit at %s by %s>" % (ts2rcs(self.timestamp), remove_email(self.committer))

#  See specification in INTERNALS

def get_commits(ts0, ts1):
    return vc.get_commits(ts0, ts1)

# Locking

def global_lock_fn():
    return os.path.join(config['db_dir'], 'lock')

# Lock the lock file "fn".
# If "verbose", print log messages when a locking attempt blocks
# (but not when the lock succeeds without blocking, nor when a trylock fails)
# If verbose > 1, be even more verbose.

def lock(fn, trylock = False, verbose = False, recursive = True, exclusive = True):
    global locks
    if verbose > 1:
        print("LOCK", fn, "try=" + str(trylock), "recursive=" + str(recursive), "exclusive=" + str(exclusive))
    if recursive:
        existing = locks.get(fn)
        if existing:
            if verbose > 1:
                print("returning existing lock")
            return existing

    if exclusive:
        flags = fcntl.LOCK_EX
    else:
        flags = fcntl.LOCK_SH

    fd = open(fn, 'a+')
    try:
        fcntl.flock(fd, flags | fcntl.LOCK_NB)
    except IOError as e:
        # On Linux, it's EWOULDBLOCK = 11
        # On NetBSD, it's EAGAIN = 35
        if e.errno == 11 or e.errno == 35:
            if trylock:
                fd.close()
                if verbose > 1:
                    print("failed to get lock")
                return None
            else:
                # Block
                if verbose:
                    print("waiting for", ['read', 'write'][exclusive], "lock on", fn, "...", end=' ')
                    sys.stdout.flush()
                    t0 = time.time()
                fcntl.flock(fd, flags)
                if verbose:
                    t1 = time.time()
                    print("got lock after %i seconds" % (t1 - t0))
        else:
            # Unexpected error
            fd.close()
            raise
    # It would tempting to store the pid in the file for debugging here,
    # but we can't because this is used to lock files with other content,
    # such as the jobno file.
    if verbose > 1:
        print("got lock")
    locks[fn] = fd
    return fd

# Backwards compat

def trylock(fn, **kwargs):
    return lock(fn, trylock = True, **kwargs)

# This may be used as the test function when locating breakage
# that can't be tested automatically

def ask_user_for_status(ts):
    print("A release is now in %s." % os.path.join(release_dir(ts), 'release'))
    fd = open('/dev/tty', 'r', 0)
    while True:
        print("Please test it manually and enter 'success' or 'failure' >", end=' ')
        sys.stdout.flush()
        line = fd.readline()
        print(line)
        if line == 'success\n':
            fd.close()
            return True
        elif line == 'failure\n':
            fd.close()
            return False
        print("Please enter 'success' or 'failure'")

def lock_results(ts):
    mkdir_p(results_dir(ts))
    fd = open(db_fn(ts), "a+")
    fd.close()
    db_lock = trylock(db_fn(ts))
    if not db_lock:
        raise RuntimeError("database is locked - build already in progress?")
    return db_lock

# Return true iff both builds failed but the build log length
# changed significantly

def log_size_changed_significantly(tsp):
    try:
        #print([(ts2rcs(ts), get_cached_status_if_any(ts, 'build_status') != 0) for ts in tsp])
        if not all([get_cached_status_if_any(ts, 'build_status') != 0 for ts in tsp]):
           return False
        diff = abs(get_cached_status(tsp[0], 'build_log_lines') -
                   get_cached_status(tsp[1], 'build_log_lines'))
        return diff > 8000
    except:
        return False

def test_failure_count_changed_significantly(ts):
    try:
        diff = abs(get_cached_status(tsp[0], 'failed_tests') -
                   get_cached_status(tsp[1], 'failed_tests'))
        return diff >= 5
    except:
        return False

def status_changed(tsp, tag):
    a = get_cached_status_if_any(tsp[0], tag)
    b = get_cached_status_if_any(tsp[1], tag)
    return a is not None and b is not None and a != b

# Return true iff both tests were run but only one of them completed

def test_completion_changed(tsp):
    return \
        get_cached_status_if_any(tsp[0], 'test_status') is not None and \
        get_cached_status_if_any(tsp[1], 'test_status') is not None and \
        ((get_cached_status_if_any(tsp[0], 'passed_tests') is not None) != \
         ((get_cached_status_if_any(tsp[1], 'passed_tests') is not None)))

def build_status_changed(tsp):
    return status_changed(tsp, 'build_status')

def install_status_changed(tsp):
    return status_changed(tsp, 'install_status')

def boot_status_changed(tsp):
    status_changed(tsp, 'boot_status')

reasons = [
    build_status_changed,
    install_status_changed,
    boot_status_changed,
    test_completion_changed,
    log_size_changed_significantly,
    test_failure_count_changed_significantly,
]

# Given a pair of timestamps, if something changed between them
# (e.g,., the build or install was broken or fixed, see "reasons"
# above), return a string identifying the type of change.

def any_status_changed(tsp):
    for reason in reasons:
        if reason(tsp):
            return reason.__name__
    return False

# Find the month of a timestamp.  Return a string like "2010.05".

def ts_month(ts):
    rcsdate = ts2rcs(ts)
    parts = rcsdate.split('.')
    return '.'.join(parts[0:2])

# Find the year of a timestamp.  Return a string like "2010".

def ts_year(ts):
    rcsdate = ts2rcs(ts)
    parts = rcsdate.split('.')
    return '.'.join(parts[0:1])

# Find the month of a commit.  Return a string like "2010.05".

def commit_month(c):
    return ts_month(c.timestamp)

# Extract the ATF test results for the given source date from the ATF
# XML output.  Returns a dictionary mapping (testprogram, testcase)
# tuples to booleans.
#
# This is rather slow, taking about 60 ms on guido (Intel i5).

def atf_test_results_from_xml(ts):
    xmlfile = gzip.open(os.path.join(results_dir(ts), 'test.xml.gz'), 'r')
    state = { 'tp': None, 'tc': None }
    results = {}
    def start_element(name, attrs):
        if name == 'tp' or name == 'tc':
            state[name] = attrs['id']
        elif name == 'passed':
            results[(state['tp'], state['tc'])] = True
        elif name == 'failed':
            results[(state['tp'], state['tc'])] = False
    p = xml.parsers.expat.ParserCreate('ISO-8859-1')
    p.StartElementHandler = start_element
    p.ParseFile(xmlfile)
    return results

# Ditto but using the console output.  This is inherently unreliable
# due to things like kernel messages interspersed with the ATF output.

def atf_test_results_from_console(ts):
    # Open in binary mode as there may be random junk from
    # serial line glitches
    f = gzip.open(os.path.join(results_dir(ts), 'test.log.gz'), 'rb')
    results = {}
    tp = None
    for line in f.readlines():
        # Get rid of carriage returns
        line = line.rstrip()
        m = re.match(br'^(.+) \(.*\): \d+ test case', line)
        if m:
            tp = m.group(1).decode('ASCII', errors = 'ignore')
        m = re.match(br' +(.*): \[.*\] (.*$)', line)
        if tp and m:
            tc = m.group(1).decode('ASCII', errors = 'ignore')
            rest = m.group(2).decode('ASCII', errors = 'ignore')
            outcome = (rest == 'Passed.')
            results[(tp, tc)] = outcome
    return results

# Ditto, using either XML and console output per configuration.

def atf_test_results(ts):
    if bool(int(config_get('have_atf_xml', '1'))):
        f = atf_test_results_from_xml
    else:
        f = atf_test_results_from_console
    return f(ts)

# Ditto, returning None instead of raising an exception if no test
# results could be read

def atf_test_results_or_none(ts):
    try:
        return atf_test_results(ts)
    except Exception as e:
        #print("could not get atf test results for %s: " % ts2rcs(ts), e)
        return None

def init_test_result_cache():
    global test_result_cache
    global test_result_recent
    test_result_cache = {}
    test_result_recent = []

# Get test results for "ts", or None if not available.
# Memoize a few recently used results, and all None results

def atf_test_results_or_none_memoize(ts):
    global test_result_cache
    global test_result_recent
    assert(len(test_result_recent) <= 4)
    if ts in test_result_cache:
        return test_result_cache[ts]
    results = atf_test_results_or_none(ts)
    test_result_cache[ts] = results
    if results is not None:
        test_result_recent.append(ts)
        if len(test_result_recent) > 4:
            victim = test_result_recent[0]
            del test_result_cache[victim]
            test_result_recent = test_result_recent[1:]
    return results

# Find the newest test results older than "ts".
# Return their timestamp and a dictionary mapping (testprog, testcase)
# tuples to booleans (True for success, False for failure).

def prev_test_results(build_dates, ts):
    i = bisect_left(build_dates, ts) - 1
    while i >= 0:
        ts = build_dates[i]
        dic = atf_test_results_or_none_memoize(ts)
        if dic is not None:
            return ts, dic
        i -= 1
    return None, None

# Helper for make_sh_test_func

def make_sh_test_temp_disk(ts, sh_commands):
    tmp = os.path.join(test_dir(ts), "test_disk_temp")
    try:
        shutil.rmtree(tmp)
    except:
        pass
    os.mkdir(tmp)
    f = open(os.path.join(tmp, "script"), "w")
    print(sh_commands, file=f)
    f.close()
    img_fn = os.path.join(test_dir(ts), "test_disk.img")
    run("makefs -M 1m %s %s" % (img_fn, tmp))
    return img_fn

# Implementation of make_sh_test_func() (see below) for qemu.
# This passes the script to the guest using temporary disk image.

def make_sh_test_func_qemu(name, sh_commands):
    def test_func(ts):
        img_fn = make_sh_test_temp_disk(ts, sh_commands)
        return anita_run_1(ts, "boot",
            os.path.join(results_dir(ts), name + ".log"),
            extra_anita_args = ["--run", "mount /dev/wd1a /mnt && sh /mnt/script", "--vmm-args", "-hdb %s" % img_fn])
    return test_func

# Read a noemu configuration file

def read_noemu_conf(fn):
    config = {}
    f = open(fn, "r")
    for line in f:
        s = line.rstrip()
        if s == '':
            continue
        l, r = s.split("=")
        config[l] = r
    f.close()
    return config

# Implementation of make_sh_test_func() (see below) for noemu.
# This passes the script to the guest using tftp.

def make_sh_test_func_noemu(name, sh_commands):
    noemu_conf = read_noemu_conf('noemu.conf')
    def test_func(ts):
        script_fn = os.path.join(anita_workdir(ts), 'script')
        write_file(script_fn, sh_commands, 't')
        return anita_run_1(ts, "boot",
            os.path.join(results_dir(ts), name + ".log"),
            extra_anita_args = [
                "--no-install", "--run", "(echo blksize 8192; echo get script) | tftp %s && sh script" %
                noemu_conf['serveraddr']])
    return test_func

# This passes the script to the guest over the serial console
# as a base64 encoded string.  Not tested or used.

def make_sh_test_func_b64(name, sh_commands):
    def test_func(ts):
        b64 = base64.b64encode(sh_commands)
        return anita_run_1(ts, "boot",
            os.path.join(results_dir(ts), name + ".log"),
            extra_anita_args = ["--run", "echo '%s' | base64 -d >script && cat script && sh script" % b64])
    return test_func

# Construct a test function that runs sh_commands using /bin/sh on
# the target and succeeds iff they return a zero exit status.
# sh_commands can be anything from a single sh command to a
# multi-line shell script.

def make_sh_test_func(name, sh_commands):
   if using_noemu():
       f = make_sh_test_func_noemu
   else:
       f = make_sh_test_func_qemu
   return f(name, sh_commands)

def make_sh_test_op(name, sh_commands):
    func = make_sh_test_func(name, sh_commands)
    return TestOp(func, [install_op], name = name)

def do_not_report(ts, success):
    pass

# Publish all reports to the web server.  We no longer use --size-only
# because there have been cases where a file was updated and the
# compressed length did not change.  Trying -t instead: rsync man page
# says "Note that if this option is not used, the optimization that
# excludes files that have not been modified cannot be effective"
# Parts of rsync -a we don't use: -l (symlinks), -p (permissions), -g
# (group), -o (owner), -D (devices, specials)

def publish_reports():
    if config.get('jobid'):
        print("not updating reports for precommit job")
        return
    target = config.get('webserver_rsync_target', '')
    if target == '':
        return
    argv = ['rsync']
    rsync_rsh = config.get('webserver_rsync_rsh')
    if rsync_rsh:
        argv += ['-e', rsync_rsh]
    files = glob(os.path.join(config['htmldir'], '*'))
    argv += ['--recursive', '-t'] + files + [target]
    # '-v',
    runv(argv, verbose = 150)

def update_reports(args = None):
    if args is None:
        args = []
    # Clear the test result cache as the test results may have
    # changed since this function was last invoked in this process
    init_test_result_cache()
    htmldir = config_get('htmldir', '')
    if htmldir == '':
        return
    import report
    lockfd = report.main(htmldir, args)
    import update_event_list
    update_event_list.main()
    # Generate list of files for mirroring
    run("( cd \"%s\" && find . -print | sed -n 's!^\./!!p' >files.dat )" % config['htmldir'])
    publish_reports()

# Create a lock for keeping concurrent noemu tests from stepping one
# each other's toes.  db_key can be noemu_lock (to lock a serial
# port) or noemu_netboot_lock (to lock a network interface).

def lock_noemu(db_key):
    lockname = config_get(db_key, None)
    if lockname is None:
        return None
    print("locking", lockname)
    lk = lock(os.path.join(config['db_dir'], db_key + '.' + lockname),
              verbose = True, recursive = True)
    return lk

# An object of class Noemu represents the state of a noemu-attached
# target machine.

class Noemu:
    def __init__(self):
        self.netboot_lock = lock_noemu('noemu_netboot_lock')
        self.lock = lock_noemu('noemu_lock')
        self.installed = False

def using_noemu():
    return (config_get('noemu_lock', None) is not None)

# Build version "ts" in the foreground, then test it (maybe in the
# background).

def build_and_test(ts, test_in_background = False):
    # Make sure we don't start a new build when one is already
    # in progress, by holding a lock on bracket.db.
    results_lock = lock_results(ts)

    success = test(build_op, ts)

    if success:
        # Build succeeded.  Install, archive results (if configured to do so)
        # and run tests, possibly in the background.
        if test_in_background:
            print("forking")
            sys.stdout.flush()
            pid = os.fork()
            if pid != 0:
                # Parent
                print("parent %d returning" % os.getpid())
                sys.stdout.flush()
                return
            # Child
            print("child %d running tests" % os.getpid())
            sys.stdout.flush()
        try:
            success = test(test_op, ts)
        except Exception as e:
            print("could not run anita tests:", type(e), str(e))
            traceback.print_exc()
        finally:
            update_reports()
            if have_newer_result(config, ts, 'test_completed'):
                print('not the newest atf test run, not reporting changes')
            elif ts < py2ts(datetime.now()) - 7 * 24 * 3600:
                print('atf test run results are too old, not reporting changes')
            else:
                summarize_atf_changes(ts, 30, do_mail = True, do_print = True)
            print("child %d exiting" % os.getpid())
            sys.exit(0)

def opt_callback(option, opt, value, parser, key):
    config[key] = value

def add_bracket_options(optparser):
    optparser.add_option("--update-repo", action="callback", type="string",
                         callback = opt_callback,
                         callback_kwargs = { 'key': 'update_repo' })

# Find the message ID of the previous "test_type" failure report
# before "ts", if any.

def get_failure_msgid(ts, test_type):
    build_dates = existing_build_dates()
    i = bisect_left(build_dates, ts) - 1
    while i >= 0:
        ts = build_dates[i]
        msgid = get_cached_if_any(ts, "%s_%s_msgid" % (test_type, "failure"))
        if msgid:
            return msgid
        i -= 1
    return None

# Generate a URL pointing to the commit at ts.  This points into
# the bracket-generated reports, not an external site such as cvsweb.
#
# This returns a relative URL; caller may add a prefix if needed.

def commit_url(ts):
    pydate = ts2py(ts).date()
    return pydate.strftime("commits-%Y.%m.html#") + ts2rcs(ts)

# Construct an email message reporting a build or test failure or success.
# stage is "build", "install", "test_completed", or "test" (for failures
# of individual ATF tests).
# Returns a tuple of the message and the envelope To: address,
# or None if emails are disabled in the configuration.

def make_email(new_ts, old_ts, stage, report_type, body, success = False):
    event_names = ['failure', 'success']
    new_event = event_names[success]
    eventful_names = ['failed', 'successful']
    old_eventful = eventful_names[not success]
    new_eventful = eventful_names[success]
    notify_from = none2empty(config.get('notify_from'))
    if notify_from == '':
        return None
    envelope_to = config.get('notify_%s_envelope_to' % stage)
    if envelope_to is None:
        envelope_to = config.get('notify_%s_to' % stage)
    if envelope_to is None:
        return None

    committers = { }
    commits = get_commits(old_ts, new_ts)
    for c in commits:
        committers[c.committer] = True

    committer_emails = [f'<{user}@NetBSD.org>' if user.find('@') == -1 else user
                        for user in sorted(committers.keys())]

    conf_notify_to = none2empty(config.get('notify_%s_to' % stage))
    if conf_notify_to == '':
       return None
    elif conf_notify_to == 'committers':
       to = ", ".join(["%s" % m for m in committer_emails])
    else:
       to = conf_notify_to

    reply_to = none2empty(config.get('notify_reply_to'))

    variant = variant_no_leading_dash()
    if variant is None:
        variant_extra = ''
    else:
        variant_extra = " (%s)" % variant

    subject = "Automated report: NetBSD-%s/%s%s %s" % \
        (branch_description(config), config['arch'], variant_extra, report_type)

    msgid = email.utils.make_msgid()
    db_put(new_ts, "%s_%s_msgid" % (stage, new_event), msgid)

    ref_msgid = None
    if success:
        ref_msgid = get_failure_msgid(new_ts, stage)

    report = ''
    report += "From: %s\n" % config['notify_from']
    report += "To: %s\n" % to
    if reply_to != '':
        report += "Reply-To: %s\n" % reply_to
    report += "Subject: " + subject + "\n"
    report += "Message-ID: " + msgid + "\n"
    if ref_msgid:
        report += "In-Reply_To: " + ref_msgid + "\n"
        report += "References: " + ref_msgid + "\n"
    report += "\n"

    report += body

    if stage == 'test_completed':
        stage_desc = 'run'
    else:
        stage_desc = stage

    if len(commits) < 250:
        report += textwrap.fill("The following commits were made between the last " + \
            "%s %s and the first %s %s:" % (old_eventful, stage_desc, new_eventful, stage_desc), 70)
        report += "\n\n"
        for c in commits:
            report += vc.format_commit_email(c)
    else:
        committers = sorted(set(c.committer for c in commits))
        plural = (len(committers) != 1)
        report += textwrap.fill(("Between the last " + \
            "%s %s and the first %s %s, a total of %d revisions were committed, " + \
            "by the following developer%s:") % \
                (old_eventful, stage_desc, new_eventful, stage_desc, len(commits), ["", "s"][plural]), 70)
        report += "\n\n"
        for committer in committers:
            report += "    " + remove_email(committer) + "\n"

        report +="\nThe first of these commits was made on CVS date %s,\n" % ts2rcs(commits[0].timestamp)
        report +="and the last on %s.\n" % ts2rcs(commits[-1].timestamp)

    url_base = config.get('url_base')
    if url_base:
        url = url_base + commit_url(new_ts)
        report += "\nLogs can be found at:\n\n"
        report += "    " + url
        report += "\n"

    return report, envelope_to

class NoImage(Exception):
    pass

def split_path(p):
    a, b = os.path.split(p)
    return (split_path(a) if len(a) and len(b) else []) + [b]

def looks_like_rcsdate(s):
    return not not re.match(r'(\d+\.){5}\d+$', s)

# Parse a date like '4.5 weeks ago' and return a ts, or None if
# the input does not look like that at all.

def parse_ago(s):
    m = re.match(r'([\d\.]+) (\w+?)s? ago$', s)
    if not m:
        return None
    amount, unit = m.groups()
    return py2ts(datetime.now()) - \
         int(float(amount) * {
            'day': 86400,
            'week': 7 * 86400,
            'month': 30 * 86400,
            'year': 365 * 86400,
         }[unit])

# Parse a human-readable source date, including the symbolic
# forms 'latest_image' and 'latest_successful_build' and
# forms like '2 days ago', and return a ts.

def sourcedate2ts(s):
    if s == 'latest_image':
        # XXX Bug: may choose a build whose installation has not yet completed
        dir = config['test_root']
        pattern = os.path.join(dir, '*', 'anita', 'wd0.img')
        image_paths = sorted(glob(pattern))
        if len(image_paths) == 0:
            raise NoImage()
        date_part = split_path(image_paths[-1])[-3]
        return rcs2ts(date_part)
    elif s == 'latest_successful_build':
        return find_latest_successful_build()
    elif s == 'current':
        update_repository()
        return last_commit_ts()
    elif s == 'current_local':
        use_current_repository()
        return last_commit_ts()
    elif looks_like_rcsdate(s):
        return ts_adjust_verbose(rcs2ts(s))
    else:
        ts = parse_ago(s)
        if ts:
            return ts_adjust_verbose(ts)
        raise RuntimeError('could not parse date "%s"' % s)

def send_mail(message, envelope_to):
    print("sending mail to", envelope_to)
    # Archive the mail
    write_file(os.path.join(config['logdir'], 'mail',
                            datetime.now().strftime("%Y%m%d-%H%M")),
               message, mode = 't')
    if sys.version_info[0] >= 3:
        kwargs = { 'encoding' :'utf-8' }
    else:
        kwargs = { }
    pipe = subprocess.Popen(['sendmail'] + [envelope_to],
                            stdin = subprocess.PIPE,
                            **kwargs).stdin
    pipe.write(message)
    pipe.close()
    # Sleep a bit to ensure the archived mail file names get unique
    # timestamps.
    time.sleep(90)

# Get the full test names and runtimes for the tests of "ts"
# as a list of tuples.  Used by testcase_time.py and
# test_time_cum.py.

def get_test_runtimes(ts):
    fn = os.path.join(results_dir(ts), 'test.log.gz')
    f = gzip.open(fn, 'r')
    ofn = ts2rcs(ts) + '.time'
    of = open(ofn, 'w')
    r = []
    testname = ''
    for line in f:
       s = line.rstrip('\r\n')
       m = re.match(r'(\S+) .*test cases', s)
       if m:
           testname = m.group(1)
       m = re.match(r'\s*([^:]+): \[([\d\.]*)s\]', s)
       if m:
           fullname = testname + ':' + m.group(1)
           time = float(m.group(2))
           r.append((fullname, time))
    return r

# Turn a test key into a human-readable name
def test_name(k):
    return k[0] + ':' + k[1]

# Given a test key, construct the file name for registering
# a reported failure
def get_reported_fn(k):
    return os.path.join(config['db_dir'], 'reported_test_failures',
                        re.sub(r'[^A-Za-z0-9]', '_', test_name(k)))

# Generate a plain-text tabular "punch card" report

def render_punchcard_plaintext(ts_list, data):
    def outcome_symbol(outcome):
        if outcome is True:
            return '-'
        elif outcome is False:
            return 'X'
        else:
            return '/'
    s = ''
    for line in data:
        # Construct a string of X:es and dashes indicating failure and success, respectively
        s += ''.join([outcome_symbol(r) for r in line['outcomes']]) + ' ' + \
            ' *'[line['new_failure']] + ' ' + line['test_name'] + '\n'
    return s

# Generate a summary of changes in the last n_runs ATF test results as
# of "ts".  If "ts" is None, use the time of the most recent build.
#
# If do_mail, send an email notification if configured to do so.
# If do_print, print a tabular report on standard output.

def summarize_atf_changes(ts, n_runs, do_mail, do_print):
    init_test_result_cache()
    build_dates = existing_build_dates()
    if len(build_dates) == 0:
        return

    if ts == None:
        ts = build_dates[-1]

    ts += 1

    # List of timestamps for the N most recent runs, newest first
    ts_list = []

    # Test key tuples (testprog, testcase) for all tests that have failed
    # during the last N runs
    test_set = set()

    # Map from (timestamp, test_key) tuple to outcome (True, False)
    # for all tests that have failed at least once during
    # the last N runs
    outcomes = {}

    while len(ts_list) < n_runs:
        ts, results = prev_test_results(build_dates, ts)
        if ts is None:
            break
        # Ignore runs with a huge number of failures
        #if len([v for k, v in results.iteritems() if v == False]) > 100:
        #    continue
        for k, v in results.items():
            if v is False:
                # Test failed
                test_set.add(k)
            outcomes[(ts, k)] = v
        ts_list.append(ts)

    # Unreported failures indexed by the (n_failures, n_successes) tuple
    unreported_failures = defaultdict(set)

    punchcard_data = []
    for k in sorted(test_set, key = test_name):
        # Construct a list of booleans indicating the outcomes of this test
        # in each test run.  Elements may also be None indicating that the
        # test was not run.
        outcome_list = [outcomes.get((ts, k)) for ts in ts_list]
        #print outcome_list

        # If we have four failures in a row after 20 successful runs
        # in a row, consider it a new failure.  First count the number
        # of recent failures
        i = 0
        n_failures = 0
        n_successes = 0
        while i < len(outcome_list) and outcome_list[i] is False:
            n_failures += 1
            i += 1
        while i < len(outcome_list) and outcome_list[i] is True:
            n_successes += 1
            i += 1
        new_failure = (n_failures >= 4 and n_successes >= 20)

        punchcard_data.append({
            'test_tuple': k,
            'test_name': test_name(k),
            'new_failure': new_failure,
            'outcomes': list(reversed(outcome_list)),
        })

        reported_fn = get_reported_fn(k)
        if new_failure:
            if os.path.exists(reported_fn):
                # Already reported
                continue
            unreported_failures[(n_failures, n_successes)].add(k)
            # There is a race condition between the check for
            # reported_fn above and its creation after sending
            # the mail below.

        # If the test succeeded, we can clear the "already reported" status
        # so that the failure can be reported again if it recurs, but
        # only if this is the same arch/variant.
        if do_mail and len(outcome_list) > 0 and outcome_list[0] is True:
            #print('test', test_name(k), 'passed, checking', reported_fn)
            try:
                content = read_file(reported_fn, 't')
                if content == arch_variant():
                    print('removing', reported_fn)
                    rm_f(reported_fn)
                else:
                    #print('different arch/variant, not removing', reported_fn)
                    pass
            except:
                pass

    if do_print:
        from report import render_punchcard_html
        f = render_punchcard_plaintext
        #f = render_punchcard_html
        print(f(list(reversed(ts_list)), punchcard_data))

    if do_mail and unreported_failures:
        read_dates()

        for k, v in unreported_failures.items():
            n_failures, n_successes = k
            plural = (len(v) != 1)
            report = textwrap.fill("This is an automatically generated notice of %s of the NetBSD test suite." % \
                                   ["a new failure", "new failures"][plural], 70)
            report += "\n\n"
            report += textwrap.fill("The newly failing test %s:" % ["case is", "cases are"][plural], 70)
            report += "\n\n"
            vsorted = sorted(v, key = test_name)
            # Show at most this many test cases
            show_max = 100
            show_n = min(len(vsorted), show_max)
            hide_n = len(vsorted) - show_n
            for i in range(show_n):
                report += "    %s\n" % test_name(vsorted[i])
            if hide_n:
                report += "    and %d more.\n" % hide_n
            report += "\n"

            report += textwrap.fill(["The above test failed", "The above tests failed"][plural] +
                " in each of the last %d test runs, and passed in at least %d consecutive runs before that." % \
                                    (n_failures, n_successes), 70)
            report += "\n\n"

            r = make_email(ts_list[n_failures - 1], ts_list[n_failures], "test", "test failure", report)
            if r is not None:
                message, envelope_to = r
                # print message
                send_mail(message, envelope_to)

                # Keep track of failures that have already been
                # reported by creating a file containing the name of
                # the reporting architecture/variant pair
                for tup in v:
                   reported_fn = get_reported_fn(tup)
                   write_file(reported_fn, arch_variant(), 't')


def test_args2op(args):
    testname = args[0]
    op = ops_by_name.get(testname)
    if op:
        if len(args[1:]):
            raise RuntimeError('op "%s" does not take arguments' % testname)
        return op
    module_name = "tests." + testname
    module = importlib.import_module(module_name)
    return module.op(*args[1:])

def testname2op(testname):
    return test_args2op([testname])

# Run one or more external test scripts
# nee test_more.py

def run_external_tests(sourcedate, testnames):
    req_ts = sourcedate2ts(sourcedate)
    ts = ts_adjust_verbose(req_ts)

    overall_success = True

    for testname in testnames:
        print("*** Testing", testname)
        op = testname2op(testname)
        result = test(op, ts)
        overall_success = overall_success and result
        print(testname + ":", ['FAIL', 'PASS'][result])
        print("\n")

    return overall_success

# Run a single external test script taking arguments

def run_external_test(sourcedate, args):
    req_ts = sourcedate2ts(sourcedate)
    ts = ts_adjust_verbose(req_ts)
    print("*** Testing", args)
    testname = args[0]
    op = test_args2op(args)
    result = test(op, ts)
    print(testname + ":", ['FAIL', 'PASS'][result])
    return result

def setup():
    run(os.path.join(libdir, 'setup.sh'), verbose = False)
    htmldir = config_get('htmldir', '')
    if htmldir != '':
        mkdir_p(htmldir)
        mkdir_p(os.path.join(htmldir, 'build'))
    vc.setup()

# Attempt to extract the relevant part of a build log, i.e., the last
# couple dozen lines leading up to the error that caused the build
# failure (if any).  Return that part as a string.

def build_log_summary(ts):
    pipe = os.popen("zcat %s" % build_log_tail_gz_fn(ts))
    lines = pipe.readlines()
    pipe.close()

    # Find the first error reported by make.
    first_make_error_line = None
    # Post-2020.06.19.21.17.48
    for i in range(len(lines)):
        if re.search(r'make.*stopped', lines[i]):
            first_make_error_line = i
    if first_make_error_line is None:
        # Pre-2020.06.19.21.17.48
        for i in range(len(lines)):
            if re.search(r'Error code [0-9]+$', lines[i]):
                first_make_error_line = i
                break
    if first_make_error_line is None:
        return None

    # Search backwards for something that could be a compiler error, but
    # no more than max_lines_back lines back
    # 500 was not enough in the case of
    # http://releng.netbsd.org/b5reports/i386/2021/2021.08.29.10.18.17/build.log.tail
    max_lines_back = 1000
    last_compiler_error_line = None
    i = first_make_error_line - 1
    while True:
        if i == 0:
            break
        if i == first_make_error_line - max_lines_back:
            break
        # Most error messages include the string "error:", but not
        # all, so we also look for some common ones that don't.
        if re.search(r'error:|'
                     r'No such file or directory|'
                     r'undefined reference|'
                     r'don.t know how to make',
                     lines[i]):
            last_compiler_error_line = i
            break
        i = i - 1

    # Decide how many lines to keep
    if last_compiler_error_line:
        keep_from = last_compiler_error_line - 5
        keep_to = last_compiler_error_line + 6
    else:
        # No obvious compiler error, show the make error
        keep_from = first_make_error_line - 20
        keep_to = first_make_error_line + 3

    keep_from = max(keep_from, 0)
    keep_to = min(keep_to, len(lines))

    # Strip excess verbiage
    lines = lines[keep_from:keep_to]

    # Strip empty lines
    lines = [line for line in lines if not re.match(r'^$', line)]

    return ''.join(["    " + line for line in lines])

# Return a human readable name for the configured branch.

def branch_description(config):
    branch = branch_name(config)
    if branch == 'HEAD':
        return 'current'
    else:
        return re.sub(r'^netbsd-', '', branch)

# Report a new failure or success.
# "stage" is "build" or "install".

def report_status_change(old_ts, new_ts, stage, mail = True, success = False):
    event = ['failure', 'success'][success]

    if success:
        # A more succinct message will do.
        if stage == 'test_completed':
            desc = 'ATF tests are running to completion'
        else:
            desc = '%s is working' % stage
        report = 'The NetBSD-%s/%s %s again.\n\n' % \
            (branch_description(config), config['arch'], desc)
    else:
        (sysname, nodename, release, version, machine) = os.uname()

        version_desc = 'NetBSD-%s/%s' % (branch_description(config), config['arch'])

        if stage == 'test_completed':
            if success:
                event_desc = 'successful completion of the ATF test suite '
                'on ' + version_desc
            else:
                event_desc = 'failure to run the ATF test suite ' \
                'to completion on ' + version_desc
        else:
            event_desc = version_desc + " " + stage + " " + event

        if stage == 'build':
            inner_system = ''
        else:
            inner_system = 'a system built on '
        report = textwrap.fill("This is an automatically generated notice of a %s." \
                               % (event_desc), 70) + "\n\n"
        report += textwrap.fill(("The %s occurred on %s%s, a %s/%s host, " + \
                                "using sources from CVS date %s.") % \
                                (event, inner_system, nodename, sysname, machine,
                                 ts2rcs(new_ts)), 70) + "\n\n"

        if stage == 'build':
            summary = build_log_summary(new_ts)
            if summary:
                report += "An extract from the build.sh output follows:\n\n"
                report += summary
                report += "\n"

                if re.match(r'bconfig.h: no such file or directory', summary):
                    print("not reporting PR48914 failure again")
                    return

    if stage == 'test_completed':
        stage_desc = 'ATF test suite completion'
    else:
        stage_desc = stage
    r = make_email(new_ts, old_ts, stage, "%s %s" % (stage_desc, event),
                   report, success)
    if r is None:
        return
    message, envelope_to = r
    if message:
         if mail:
             send_mail(message, envelope_to)
         else:
             print(message, end=' ')

# Find the newest build/install before "ts" that has produced a
# success/failure result for "name"

def previous_build_with_result(config, ts, name):
    for bs in reversed(existing_build_dates_r(config)):
        if bs < ts and \
            get_cached_status_if_any_r(config, bs, name + '_status') \
                is not None:
            return bs
    return None

# Return true iff we have a result of test "name" newer than "ts"

def have_newer_result(config, ts, name):
    for bs in reversed(existing_build_dates_r(config)):
        if bs > ts and \
            get_cached_status_if_any_r(config, bs, name + '_status') \
                is not None:
            return True
    return False

# If this is a new failure (or perhaps a new success), send mail.
# ts1 is the timestamp of a new build/install.
# "name" is one of "build", "install", etc.

def maybe_report_status_change(ts1, name):
    print("maybe_report_status_change", ts2rcs(ts1), name)
    ts0 = previous_build_with_result(config, ts1, name)
    if ts0 is None:
        print("no previous %s, not reporting" % name)
        return

    # Create a pair of boolean indicating the success/failure at ts0,ts1
    success = [get_cached_status_if_any(ts, name + '_status') == 0
               for ts in (ts0, ts1)]

    # There must be a change
    event = ['failure', 'success'][success[1]]
    if success[1] == success[0]:
        print("%s is not new, not reporting" % event)
        return

    # dates[] may have 0 to 2 elements:
    # dates[0] is the time of the last successful test before a
    # reported failure
    # dates[1] is the time of the first successful test after a
    # closed failure report
    dir = os.path.join(config['arch_root'], 'reported')
    mkdir_p(dir)
    fn = os.path.join(dir, name)
    try:
        with open(fn, 'r') as f:
            dates = [rcs2ts(line) for line in f.readlines()]
    except:
        dates = []

    # Don't report a failure unless this is the newest result.
    # Otherwise, refine runs can cause error reports for bugs
    # that have already been fixed, such as message-id
    # <160025082802.18178.3227525592611292626@babylon5.NetBSD.org>.
    # But do report a sucess even if it's not the newest result.
    if not success[1] and have_newer_result(config, ts1, name):
        print("not the newest result")
        return

    # Don't report ancient events
    if ts1 < py2ts(datetime.now()) - 7 * 24 * 3600:
        print("event is too old")
        return

    # Cases where we should send a report:
    # - this is a failure, and no previous report has been sent, or
    # - this is a failure, and the previous report has been closed,
    #     and this is about a version newer than the closing date
    if not success[1]:
        # Failure
        if len(dates) == 1:
            print("open report already exists")
            return
        if len(dates) == 2 and ts1 <= dates[1]:
            print("failure is older than latest report")
            return
        print("opening new report")
        with open(fn, 'w') as f:
            print(ts2rcs(ts0), file = f)
    else:
        # Success
        if len(dates) != 1:
            print("no open report")
            return
        # Close the report
        print("closing report")
        with open(fn, 'a') as f:
            print(ts2rcs(ts1), file = f)


    print("reporting")
    report_status_change(ts0, ts1, name, mail = True, success = success[1])

# Find the set of committers for commit cno.  This is a singleton set
# unless multiple people committed to CVS in the same second.

def committer_set(cno):
   commits = get_commits(cno2ts(cno - 1), cno2ts(cno))
   return set([c.committer for c in commits])

# Common to notify and build-all commands

def maybe_update_repo():
    if config.get('update_repo') == '0':
        use_current_repository()
    else:
        update_repository()

def reap_zombies():
    try:
        while True:
            os.waitpid(0, os.WNOHANG)
    except:
        pass

# Build each commit since the latest commit for which we already have
# a build result (successful or failed), or for which a build is in
# progress.
#
# Todo: skip to a commit by a different developer when fallen behind

def build_all(test_in_background = True):
    maybe_update_repo()
    ts = find_latest_completed_build()
    if not ts:
        return
    print("latest completed build is %s" % ts2rcs(ts))
    cno = ts2cno(ts)
    while True:
        cno += 1
        if not cno_valid(cno):
            print("no more commits")
            break
        ts = cno2ts(cno)
        cset = committer_set(cno)
        print("next commit is %s by %s" % (ts2rcs(ts), str(cset)))
        # Look ahead to see if there are further commits by the same
        # committer.  If so, skip forward.
        while True:
            cno_next = cno + 1
            if not cno_valid(cno_next):
                break
            ts_next = cno2ts(cno_next)
            cset_next = committer_set(cno_next)
            if not (len(cset) == 1 and cset_next == cset):
                break;
            print("%s has same committers %s" % (ts2rcs(ts_next), str(cset_next)))
            cno = cno_next
            ts = ts_next
            cset = cset_next
            # Since we're spawing background tests in a loop,
            # zombies may accumulate.
            reap_zombies()

        print("testing %s by %s" % (ts2rcs(ts), str(cset)))
        try:
            build_and_test(ts, test_in_background)
        except Exception as e:
            # Perhaps in progress
            print(e)
            print("failed, trying the next commit")

# When an ATF test run has failed to complete, attempt to determine
# why and return a string describing the failure reason, or None

def noncompletion_reason(ts):
    try:
        f = gzip.open(os.path.join(results_dir(ts), 'test.log.gz'), 'rb')
    except:
        return None
    for line in f.readlines():
        # Get rid of carriage returns
        line = line.rstrip()
        #print line
        m = re.search(br'Fatal kernel mode data abort|'
                      br'panic: kernel|'
                      br'root device: Traceback|'
                      br'CPU [0-9]+: fatal kernel trap',
                      line)
        if m:
            return line.decode('unicode-escape')
    return None

# Call f(ts), which must return a string not containing newlines, and
# cache its return value under "tag", or if a value has already been
# cahced, return it instead.

def db_memoize(config, f, ts, tag):
    v = get_cached_if_any_r(config, ts, tag)
    if v is None:
        v = f(ts)
        db_put(ts, tag, v)
    return v

# Find the last change in the outcome of "op", using only cached data
# (without running any actual tests).  Returns triplet sense, ts0,
# ts1, where sense is the outcome after the change, and ts0 and ts1
# are the closest known timestamps before and after the change,
# respectively.

def find_last_change_fast(op):
    # Like fastcheck, but doesn't throw
    def c(op, ts):
        try:
            #print('fastcheck', op, ts2rcs(ts))
            return fastcheck(op, ts)
        except PrerequisiteFailed:
            return None
    # XXX use generator
    dates = list(reversed(existing_build_dates_r(config)))
    i = 0
    ts0, ts1 = None, None
    # Search backwards for a result, any result
    while i < len(dates):
        ts = dates[i]
        r = c(op, ts)
        #print("search1", ts2rcs(ts), r)
        if r is not None:
            sense = r
            ts1 = ts
            break
        i += 1
    # Search further backwards for an opposite result
    while i < len(dates):
        ts = dates[i]
        r = c(op, ts)
        #print("search2", ts2rcs(ts), r)
        if r is not None and not not r is (not sense):
            ts0 = ts
            break
        i += 1
    if ts0 is None:
        raise RuntimeError("find_last_change_fast failed")
    return sense, ts0, ts1

# Import support for the chosen version control system.
# Do this last so that everything else in the bracket module
# is defined and can be imported by the vc module.

vc_name = config_get('vc', 'cvs')
vc_dirname = vc_name
if vc_dirname == 'cvs':
    vc_dirname = 'xcvs'
# Make sure the name vc is imported into the vc module
# even though it does not yet have its final value.
vc = None
vc = import_module('vc.' + vc_dirname)
