path: root/devel/ckmake

#!/usr/bin/env python

# ncurses Linux kernel cross kernel compilation utility

# Copyright (C) 2012 Luis R. Rodriguez <mcgrof@do-not-panic.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

import argparse
import locale
import curses
import time
import os
import re
import random
import tempfile
import subprocess
import sys
import signal

from Queue import Queue
from threading import Thread, Lock
from shutil import copytree, ignore_patterns, rmtree, copyfileobj
from time import sleep

# find self
source_dir = os.path.abspath(os.path.dirname(__file__))
source_dir = os.path.dirname(source_dir)
# add parent directory to path to get to lib/
sys.path.append(source_dir)
# import libraries we need
from lib import bpreqs as reqs

releases_processed = []
releases_baking = []
processed_lock = Lock()
baking_lock = Lock()
my_cwd = os.getcwd()
ckmake_return = 0

tmp_path = my_cwd + "/.tmp.ckmake"
ckmake_log = my_cwd + "/ckmake.log"
ckmake_report = my_cwd + "/ckmake-report.log"

home = os.getenv("HOME")
ksrc = home + "/" + "ksrc-backports"
modules = ksrc + "/lib/modules"


def clean():
    if os.path.exists(tmp_path):
        rmtree(tmp_path)


def get_processed_rel(i):
    "Because... list.pop(i) didn't work to keep order..."
    processed_lock.acquire()
    for rel in releases_processed:
        if (rel['idx'] == i):
            processed_lock.release()
            return rel
    processed_lock.release()

    baking_lock.acquire()
    for rel in releases_baking:
        if (rel['idx'] == i):
            releases_baking.remove(rel)
            baking_lock.release()
            return rel
    baking_lock.release()


def get_status_name(status):
    if (status == 0):
        return 'OK'
    elif (status == 130):
        return 'TERM'
    elif (status == 1234):
        return 'INIT'
    elif (status == 1235):
        return 'LINK'
    elif (status == -2):
        return 'TERM'
    else:
        return 'FAIL'


def get_stat_pos(status):
    if (status == 0):
        return 34
    elif (status == 130):
        return 33
    elif (status == 1234):
        return 33
    elif (status == 1235):
        return 33
    elif (status == -2):
        return 33
    else:
        return 33


def get_status_color(status):
    if (status == 0):
        return curses.color_pair(1)
    elif (status == 130):
        return curses.color_pair(2)
    elif (status == 1234):
        return curses.color_pair(2)
    elif (status == -2):
        return curses.color_pair(2)
    else:
        return curses.color_pair(3)


def print_report():
    os.system("clear")
    sys.stdout.write("\n\n\n")
    sys.stdout.write("== ckmake-report.log ==\n\n")
    report = open(ckmake_report, 'r+')
    copyfileobj(report, sys.stdout)
    report.close()


def process_logs():
    os.system('clear')
    log = open(ckmake_log, 'w+')
    report = open(ckmake_report, 'w+')
    for i in range(0, len(releases_processed)):
        rel = get_processed_rel(i)
        rel_log = open(rel['log'], 'r+')
        log.write(rel_log.read())
        status = get_status_name(rel['status'])
        rel_report = "%-4s%-20s[  %s  ]\n" % \
                     (rel['idx'] + 1, rel['version'], status)
        report.write(rel_report)
        if (rel['status'] != 0 and
                rel['status'] != 2):
            ckmake_return = -1
    report.close()
    log.close()
    print_report()


def process_kernel(num, kset, cmdline_args):
    while True:
        rel = kset.queue.get()
        work_dir = tmp_path + '/' + rel['version']
        copytree(my_cwd,
                 work_dir,
                 ignore=ignore_patterns('.tmp*', ".git"))
        build = '%s/build/' % rel['full_path']
        jobs = '-j%d' % kset.build_jobs
        load = '-l%d' % 50
        make_args = ['KLIB=%s' % build,
                     'KLIB_BUILD=%s' % build]
        nice = ['ionice', '-c', '3', 'nice', '-n', '20']

        log_file_name = work_dir + '/' + 'ckmake.n.log'
        rel['log'] = log_file_name
        log_file = open(log_file_name, 'w')
        null_file = open('/dev/null', 'r')

        all_config_name = os.path.join(work_dir, 'all.config')
        all_config = open(all_config_name, 'w')
        all_config.write("CPTCFG_CFG80211_INTERNAL_REGDB=n\n")
        config_name = 'allnoconfig'
        if cmdline_args.allyesconfig:
            config_name = 'allyesconfig'
        elif cmdline_args.defconfig:
            all_config.write(
                open(os.path.join(work_dir, 'defconfigs', cmdline_args.defconfig)).read())
        else:
            all_config.write("CPTCFG_BACKPORT_USERSEL_BUILD_ALL=y\n")
        all_config.close()

        all_config_env = os.environ.copy()
        all_config_env["KCONFIG_ALLCONFIG"] = all_config_name

        kset.baking(rel)

        p = subprocess.Popen(nice + ['make', jobs, load] + make_args + [config_name],
                             cwd=work_dir, env=all_config_env,
                             stdin=null_file,
                             stdout=log_file,
                             stderr=log_file)
        p.wait()

        if p.returncode == 0:
            p = subprocess.Popen(nice + ['make', jobs, load] + make_args,
                                 cwd=work_dir,
                                 stdin=null_file,
                                 stdout=log_file,
                                 stderr=log_file)
            p.wait()

        log_file.close()
        status = p.returncode
        if status == 0:
            warn = re.compile('^WARNING:.*undefined.*')
            log_file = open(log_file_name, 'r')
            for l in log_file:
                if warn.match(l):
                    status = 1235
                    break
        null_file.close()

        kset.update_status(rel, status)
        kset.queue.task_done()
        kset.completed(rel)


def cpu_info_build_jobs():
    if not os.path.exists('/proc/cpuinfo'):
        return 1
    f = open('/proc/cpuinfo', 'r')
    max_cpu = 1
    for line in f:
        m = re.match(r"(?P<PROC>processor\s*:)\s*"
                     "(?P<NUM>\d+)",
                     line)
        if not m:
            continue
        proc_specs = m.groupdict()
        cpu_num = int(proc_specs['NUM'])
        if cpu_num > max_cpu:
            max_cpu = cpu_num
    return max_cpu + 1


def kill_curses():
    curses.endwin()
    sys.stdout = os.fdopen(0, 'w', 0)


def sig_handler(signal, frame):
    kill_curses()
    process_logs()
    clean()
    sys.exit(-2)


def compute_rel_weight_base_2(rel_specs):
    weight = 0
    sublevel = 0

    if (rel_specs['SUBLEVEL'] != ''):
        sublevel = int(rel_specs['SUBLEVEL'].lstrip(".")) * 20
    else:
        sublevel = 5

    weight = (int(rel_specs['VERSION'])    << 32) + \
             (int(rel_specs['PATCHLEVEL']) << 16) + \
             (sublevel << 8)

    return weight


def compute_rel_weight_base_3(rel_specs):
    weight = 0
    extra = 0
    sublevel = 0

    weight = (int(rel_specs['VERSION'])    << 32) + \
             (int(rel_specs['PATCHLEVEL']) << 16)

    return weight


def compute_rel_weight_base(rel_specs):

    if (int(rel_specs['VERSION']) == 2):
        return compute_rel_weight_base_2(rel_specs)
    if (int(rel_specs['VERSION']) == 3):
        return compute_rel_weight_base_3(rel_specs)
    return 0


def compute_rel_weight(rel_specs):
    weight = 0
    extra = 0
    sublevel = 0

    if (rel_specs['EXTRAVERSION'] != ''):
        if ("." in rel_specs['EXTRAVERSION'] or
                "rc" in rel_specs['EXTRAVERSION']):
            rc = rel_specs['EXTRAVERSION'].lstrip("-rc")
            if (rc == ""):
                rc = 0
            else:
                rc = int(rc) - 20
            extra = int(rc)
        else:
            extra = int(rel_specs['EXTRAVERSION']) + 10

    if (rel_specs['SUBLEVEL'] != ''):
        sublevel = int(rel_specs['SUBLEVEL'].lstrip(".")) * 20
    else:
        sublevel = 5

    weight = (int(rel_specs['VERSION'])    << 32) + \
             (int(rel_specs['PATCHLEVEL']) << 16) + \
             (sublevel   		       << 8 ) + \
             (extra * 60)

    return weight


def get_rel_spec_base(rel):
    m = re.match(r"v*(?P<VERSION>\d+)\.+"
                 "(?P<PATCHLEVEL>\d+)[.]*"
                 "(?P<SUBLEVEL>\d*)",
                 rel)
    if (not m):
        return m
    rel_specs = m.groupdict()
    return rel_specs


def get_base_spec(rel_specs):
    if (int(rel_specs['VERSION']) == 2):
        rel = rel_specs['VERSION'] + "." + rel_specs['PATCHLEVEL'] + \
            "." + rel_specs['SUBLEVEL']
    else:
        rel = rel_specs['VERSION'] + "." + rel_specs['PATCHLEVEL']
    return get_rel_spec_base(rel)


def get_rel_spec_ubuntu(rel):
    if ("rc" in rel):
        m = re.match(r"v*(?P<VERSION>\d+)\.+"
                     "(?P<PATCHLEVEL>\d+)[.]+"
                     "(?P<SUBLEVEL>\d+)[-]+"
                     "\d+rc(?P<EXTRAVERSION>\d+)\-*",
                     rel)
    else:
        m = re.match(r"v*(?P<VERSION>\d+)\.+"
                     "(?P<PATCHLEVEL>\d+)[.]+"
                     "(?P<SUBLEVEL>\d+)[-]+"
                     "(?P<EXTRAVERSION>\d+)\-*",
                     rel)
    if (not m):
        return m
    rel_specs = m.groupdict()
    return rel_specs


def krel_same_base(new_rel, rel):
    if (int(new_rel['ver']) != int(rel['ver'])):
        return False
    if (int(new_rel['pat']) != int(rel['pat'])):
        return False
    if (int(new_rel['ver']) == 3):
        return True
    if (int(new_rel['ver']) != 2):
        return False
    if (int(new_rel['sub']) == int(rel['sub'])):
        return True
    return False


def krel_base_update(new_rel, rel):
    if (not krel_same_base(new_rel, rel)):
        return False
    if (int(new_rel['sub']) > int(rel['sub'])):
        return True
    if (int(new_rel['sub']) < int(rel['sub'])):
        return False

    # Too lazy to deal with 2.x kernels,
    if (not new_rel['is_rc']):
        return False

    if (int(new_rel['ext']) <= int(rel['ext'])):
        return False
    return True


def krel_base_smaller(new_rel, rel):
    if (not krel_same_base(new_rel, rel)):
        return False
    if (int(new_rel['sub']) > int(rel['sub'])):
        return False
    if (int(new_rel['sub']) < int(rel['sub'])):
        return True
    return False


class kernel_set():

    def __init__(self, stdscr):
        self.queue = Queue()
        self.target_krevs = []
        self.releases = []
        self.target_kranges = []
        self.stdscr = stdscr
        self.lock = Lock()
        signal.signal(signal.SIGINT, sig_handler)
        self.build_jobs = cpu_info_build_jobs()
        if (curses.has_colors()):
            curses.init_pair(1,
                             curses.COLOR_GREEN,
                             curses.COLOR_BLACK)
            curses.init_pair(2,
                             curses.COLOR_CYAN,
                             curses.COLOR_BLACK)
            curses.init_pair(3,
                             curses.COLOR_RED,
                             curses.COLOR_BLACK)
            curses.init_pair(4,
                             curses.COLOR_BLUE,
                             curses.COLOR_BLACK)

    def baking(self, rel):
        baking_lock.acquire()
        releases_baking.append(rel)
        baking_lock.release()

    def completed(self, rel):
        processed_lock.acquire()
        releases_processed.insert(rel['idx'], rel)
        processed_lock.release()

    def set_locale(self):
        locale.setlocale(locale.LC_ALL, '')
        code = locale.getpreferredencoding()

    def refresh(self):
        self.lock.acquire()
        self.stdscr.refresh()
        self.lock.release()

    def wight_in_target_kranges(self, rel_base_weight):
        for krange in self.target_kranges:
            if (krange['is_range']):
                if (krange['weight_lower'] <= rel_base_weight and
                        rel_base_weight <= krange['weight_upper']):
                    return True
            else:
                if (rel_base_weight == krange['weight']):
                    return True
        return False

    def evaluate_new_rel(self, new_rel):
        for rel in self.releases:
            if (krel_base_update(new_rel, rel)):
                new_rel['idx'] = rel['idx']
                self.releases.remove(rel)
                break
            if (krel_base_smaller(new_rel, rel)):
                return
        if (self.target_kranges):
            if (not self.wight_in_target_kranges(new_rel['base_weight'])):
                return
        self.releases.insert(new_rel['idx'], new_rel)

    def parse_releases(self, target_kranges, args):
        self.target_kranges = target_kranges
        rels = []
        if args.develdebug:
            sys.stdout.write("Paths for kernels:\n")
        for path in os.listdir(os.path.abspath(modules)):
            full_dir = os.path.abspath(os.path.join(modules, path))
            if not os.path.isdir(full_dir):
                continue
            if args.develdebug:
                sys.stdout.write("%s\n" % (path))
            specifics = get_rel_spec_ubuntu(path)
            if (not specifics):
                continue
            base_specs = get_base_spec(specifics)
            if (not base_specs):
                continue
            rc = False

            ver = specifics['VERSION'] + '.' + \
                specifics['PATCHLEVEL']

            if ("rc" in path):
                rc = True
                ver = ver + '-rc' + specifics['EXTRAVERSION']
            else:
                ver = ver + '.' + specifics['SUBLEVEL']

            get_rel_spec_base(path)
            rel = dict(name=path,
                       full_path = full_dir,
                       version=ver,
                       is_rc=rc,
                       ver=specifics['VERSION'],
                       pat=specifics['PATCHLEVEL'],
                       sub=specifics['SUBLEVEL'],
                       ext=specifics['EXTRAVERSION'],
                       base_weight=compute_rel_weight_base(base_specs),
                       weight=compute_rel_weight(specifics),
                       processed=False,
                       log='',
                       status=1234)
            rels.append(rel)

        def relsort(rel):
            return int(rel['ver']), int(rel['pat']), int(rel['sub'])
        rels.sort(key=relsort)
        for rel in rels:
            rel['idx'] = len(self.releases)
            self.evaluate_new_rel(rel)
        if not args.develdebug:
            self.refresh()

    def setup_screen(self):
        for i in range(0, len(self.releases)):
            rel = self.releases[i]
            self.lock.acquire()
            self.stdscr.addstr(rel['idx'], 0,
                               "%-4d" % (rel['idx'] + 1))
            if (curses.has_colors()):
                self.stdscr.addstr(rel['idx'], 5,
                                   "%-20s" % (rel['version']),
                                   curses.color_pair(4))
            else:
                self.stdscr.addstr(rel['idx'], 0,
                                   "%-20s" % (rel['version']))
            self.stdscr.addstr(rel['idx'], 30, "[        ]")
            self.stdscr.refresh()
            self.lock.release()

    def setup_screen_debug(self):
        sys.stdout.write("%-4s" % ("IDX"))
        sys.stdout.write("%-20s" % ("VERSION"))
        sys.stdout.write("%-60s\n" % ("DIRECTORY"))
        for i in range(0, len(self.releases)):
            rel = self.releases[i]
            sys.stdout.write("%-4d" % (rel['idx'] + 1))
            sys.stdout.write("%-20s" % (rel['version']))
            sys.stdout.write("%-60s\n" % (rel['full_path']))

    def create_threads(self, cmdline_args):
        for rel in self.releases:
            th = Thread(target=process_kernel, args=(0, self, cmdline_args))
            th.setDaemon(True)
            th.start()

    def kick_threads(self):
        for rel in self.releases:
            self.queue.put(rel)
            sleep(1)

    def wait_threads(self):
        self.queue.join()

    def update_status(self, rel, status):
        self.lock.acquire()
        stat_name = get_status_name(status)
        stat_pos = get_stat_pos(status)
        if (curses.has_colors()):
            stat_color = get_status_color(status)
            self.stdscr.addstr(rel['idx'], stat_pos, stat_name,
                               get_status_color(status))
        else:
            self.stdscr.addstr(rel['idx'], stat_pos, stat_name)
        rel['processed'] = True
        rel['status'] = status
        self.stdscr.refresh()
        self.lock.release()


def main(stdscr, args, target_kranges):
    kset = kernel_set(stdscr)

    if args.develdebug:
        kill_curses()
    else:
        kset.set_locale()

    kset.parse_releases(target_kranges, args)

    if args.develdebug:
        kset.setup_screen_debug()
        sys.exit(0)

    kset.setup_screen()

    kset.create_threads(args)
    kset.kick_threads()
    kset.wait_threads()
    kset.refresh()


def build_krange(krange_list):
    if (len(krange_list) == 2):
        lower = krange_list.pop(0)
        upper = krange_list.pop(0)
        specifics_lower = get_rel_spec_base(lower)
        if (not specifics_lower):
            return {}
        specifics_upper = get_rel_spec_base(upper)
        if (not specifics_upper):
            return {}
        krange = dict(is_range=True,
                      weight_lower=compute_rel_weight_base(specifics_lower),
                      weight_upper=compute_rel_weight_base(specifics_upper))
        return krange
    elif (len(krange_list) == 1):
        rel = krange_list.pop(0)
        specifics = get_rel_spec_base(rel)
        if (not specifics):
            return {}
        krange = dict(is_range=False,
                      weight=compute_rel_weight_base(specifics))
        return krange
    else:
        return {}
    return {}

if __name__ == "__main__":
    req = reqs.Req()
    req.require('make')
    req.require('gcc')
    if not req.reqs_match():
        sys.exit(1)
    parser = argparse.ArgumentParser(
        description='compile against all kernels you have')
    parser.add_argument(
        '--allyesconfig', const=True, default=False, action="store_const",
        help='Build allyesconfig rather than only backport code.')
    parser.add_argument('--defconfig', metavar='<name>', type=str,
                        help='Build this defconfig rather than only backport code.')
    parser.add_argument('--revs', metavar='<revision-list>', type=str,
                        help='Optional list of kernel revisions to test for, example: 2.6.24,2.6.30,2.6.32..3.2,3.4')
    parser.add_argument(
        '--develdebug', const=True, default=False, action="store_const",
        help='Development debug - use to debug which kernels we are going to process')
    args = parser.parse_args()

    target_kranges = []
    if (args.revs):
        klists = args.revs.split(",")
        for klist in klists:
            krange_list = klist.split("..")
            krange = build_krange(krange_list)
            target_kranges.append(krange)

    if not os.path.exists(modules):
        print "%s does not exist" % (modules)
        sys.exit(1)
    if not os.path.exists(my_cwd + '/Makefile'):
        print "%s does not exist" % (my_cwd + '/Makefile')
        sys.exit(1)
    if os.path.exists(ckmake_log):
        os.remove(ckmake_log)
    if os.path.exists(ckmake_report):
        os.remove(ckmake_report)
    if os.path.exists(tmp_path):
        rmtree(tmp_path)

    if not args.develdebug:
        os.makedirs(tmp_path)

    curses.wrapper(main, args, target_kranges)

    if args.develdebug:
        sys.exit(ckmake_return)

    kill_curses()
    process_logs()
    clean()
    sys.exit(ckmake_return)