diff options
Diffstat (limited to 'crosperf/machine_image_manager.py')
| -rw-r--r-- | crosperf/machine_image_manager.py | 344 |
1 files changed, 175 insertions, 169 deletions
diff --git a/crosperf/machine_image_manager.py b/crosperf/machine_image_manager.py index ffdd6436..74379bff 100644 --- a/crosperf/machine_image_manager.py +++ b/crosperf/machine_image_manager.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- -# Copyright 2015 The Chromium OS Authors. All rights reserved. +# Copyright 2015 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """MachineImageManager allocates images to duts.""" -from __future__ import print_function import functools class MachineImageManager(object): - """Management of allocating images to duts. + """Management of allocating images to duts. * Data structure we have - @@ -137,173 +136,180 @@ class MachineImageManager(object): * Special / common case to handle seperately We have only 1 dut or if we have only 1 label, that's simple enough. - """ - - def __init__(self, labels, duts): - self.labels_ = labels - self.duts_ = duts - self.n_labels_ = len(labels) - self.n_duts_ = len(duts) - self.dut_name_ordinal_ = dict() - for idx, dut in enumerate(self.duts_): - self.dut_name_ordinal_[dut.name] = idx - - # Generate initial matrix containg 'X' or ' '. - self.matrix_ = [['X' if l.remote else ' ' - for _ in range(self.n_duts_)] - for l in self.labels_] - for ol, l in enumerate(self.labels_): - if l.remote: - for r in l.remote: - self.matrix_[ol][self.dut_name_ordinal_[r]] = ' ' - - self.label_duts_ = [[] for _ in range(self.n_labels_)] - self.allocate_log_ = [] - - def compute_initial_allocation(self): - """Compute the initial label-dut allocation. - - This method finds the most efficient way that every label gets imaged at - least once. - - Returns: - False, only if not all labels could be imaged to a certain machine, - otherwise True. """ - if self.n_duts_ == 1: - for i, v in self.matrix_vertical_generator(0): - if v != 'X': - self.matrix_[i][0] = 'Y' - return - - if self.n_labels_ == 1: - for j, v in self.matrix_horizontal_generator(0): - if v != 'X': - self.matrix_[0][j] = 'Y' - return - - if self.n_duts_ >= self.n_labels_: - n = 1 - else: - n = self.n_labels_ - self.n_duts_ + 1 - while n <= self.n_labels_: - if self._compute_initial_allocation_internal(0, n): - break - n += 1 - - return n <= self.n_labels_ - - def _record_allocate_log(self, label_i, dut_j): - self.allocate_log_.append((label_i, dut_j)) - self.label_duts_[label_i].append(dut_j) - - def allocate(self, dut, schedv2=None): - """Allocate a label for dut. - - Args: - dut: the dut that asks for a new image. - schedv2: the scheduling instance, we need the benchmark run - information with schedv2 for a better allocation. - - Returns: - a label to image onto the dut or None if no more available images for - the dut. - """ - j = self.dut_name_ordinal_[dut.name] - # 'can_' prefix means candidate label's. - can_reimage_number = 999 - can_i = 999 - can_label = None - can_pending_br_num = 0 - for i, v in self.matrix_vertical_generator(j): - label = self.labels_[i] - - # 2 optimizations here regarding allocating label to dut. - # Note schedv2 might be None in case we do not need this - # optimization or we are in testing mode. - if schedv2 is not None: - pending_br_num = len(schedv2.get_label_map()[label]) - if pending_br_num == 0: - # (A) - we have finished all br of this label, - # apparently, we do not want to reimaeg dut to - # this label. - continue - else: - # In case we do not have a schedv2 instance, mark - # pending_br_num as 0, so pending_br_num >= - # can_pending_br_num is always True. - pending_br_num = 0 - - # For this time being, I just comment this out until we have a - # better estimation how long each benchmarkrun takes. - # if (pending_br_num <= 5 and - # len(self.label_duts_[i]) >= 1): - # # (B) this is heuristic - if there are just a few test cases - # # (say <5) left undone for this label, and there is at least - # # 1 other machine working on this lable, we probably not want - # # to bother to reimage this dut to help with these 5 test - # # cases - # continue - - if v == 'Y': - self.matrix_[i][j] = '_' - self._record_allocate_log(i, j) - return label - if v == ' ': - label_reimage_number = len(self.label_duts_[i]) - if ((can_label is None) or - (label_reimage_number < can_reimage_number or - (label_reimage_number == can_reimage_number and - pending_br_num >= can_pending_br_num))): - can_reimage_number = label_reimage_number - can_i = i - can_label = label - can_pending_br_num = pending_br_num - - # All labels are marked either '_' (already taken) or 'X' (not - # compatible), so return None to notify machine thread to quit. - if can_label is None: - return None - - # At this point, we don't find any 'Y' for the machine, so we go the - # 'min' approach. - self.matrix_[can_i][j] = '_' - self._record_allocate_log(can_i, j) - return can_label - - def matrix_vertical_generator(self, col): - """Iterate matrix vertically at column 'col'. - - Yield row number i and value at matrix_[i][col]. - """ - for i, _ in enumerate(self.labels_): - yield i, self.matrix_[i][col] - - def matrix_horizontal_generator(self, row): - """Iterate matrix horizontally at row 'row'. - - Yield col number j and value at matrix_[row][j]. - """ - for j, _ in enumerate(self.duts_): - yield j, self.matrix_[row][j] - - def _compute_initial_allocation_internal(self, level, N): - """Search matrix for d with N.""" - - if level == self.n_labels_: - return True - - for j, v in self.matrix_horizontal_generator(level): - if v == ' ': - # Before we put a 'Y', we check how many Y column 'j' has. - # Note y[0] is row idx, y[1] is the cell value. - ny = functools.reduce(lambda x, y: x + 1 if (y[1] == 'Y') else x, - self.matrix_vertical_generator(j), 0) - if ny < N: - self.matrix_[level][j] = 'Y' - if self._compute_initial_allocation_internal(level + 1, N): + def __init__(self, labels, duts): + self.labels_ = labels + self.duts_ = duts + self.n_labels_ = len(labels) + self.n_duts_ = len(duts) + self.dut_name_ordinal_ = dict() + for idx, dut in enumerate(self.duts_): + self.dut_name_ordinal_[dut.name] = idx + + # Generate initial matrix containg 'X' or ' '. + self.matrix_ = [ + ["X" if l.remote else " " for _ in range(self.n_duts_)] + for l in self.labels_ + ] + for ol, l in enumerate(self.labels_): + if l.remote: + for r in l.remote: + self.matrix_[ol][self.dut_name_ordinal_[r]] = " " + + self.label_duts_ = [[] for _ in range(self.n_labels_)] + self.allocate_log_ = [] + + def compute_initial_allocation(self): + """Compute the initial label-dut allocation. + + This method finds the most efficient way that every label gets imaged at + least once. + + Returns: + False, only if not all labels could be imaged to a certain machine, + otherwise True. + """ + + if self.n_duts_ == 1: + for i, v in self.matrix_vertical_generator(0): + if v != "X": + self.matrix_[i][0] = "Y" + return + + if self.n_labels_ == 1: + for j, v in self.matrix_horizontal_generator(0): + if v != "X": + self.matrix_[0][j] = "Y" + return + + if self.n_duts_ >= self.n_labels_: + n = 1 + else: + n = self.n_labels_ - self.n_duts_ + 1 + while n <= self.n_labels_: + if self._compute_initial_allocation_internal(0, n): + break + n += 1 + + return n <= self.n_labels_ + + def _record_allocate_log(self, label_i, dut_j): + self.allocate_log_.append((label_i, dut_j)) + self.label_duts_[label_i].append(dut_j) + + def allocate(self, dut, schedv2=None): + """Allocate a label for dut. + + Args: + dut: the dut that asks for a new image. + schedv2: the scheduling instance, we need the benchmark run + information with schedv2 for a better allocation. + + Returns: + a label to image onto the dut or None if no more available images for + the dut. + """ + j = self.dut_name_ordinal_[dut.name] + # 'can_' prefix means candidate label's. + can_reimage_number = 999 + can_i = 999 + can_label = None + can_pending_br_num = 0 + for i, v in self.matrix_vertical_generator(j): + label = self.labels_[i] + + # 2 optimizations here regarding allocating label to dut. + # Note schedv2 might be None in case we do not need this + # optimization or we are in testing mode. + if schedv2 is not None: + pending_br_num = len(schedv2.get_label_map()[label]) + if pending_br_num == 0: + # (A) - we have finished all br of this label, + # apparently, we do not want to reimaeg dut to + # this label. + continue + else: + # In case we do not have a schedv2 instance, mark + # pending_br_num as 0, so pending_br_num >= + # can_pending_br_num is always True. + pending_br_num = 0 + + # For this time being, I just comment this out until we have a + # better estimation how long each benchmarkrun takes. + # if (pending_br_num <= 5 and + # len(self.label_duts_[i]) >= 1): + # # (B) this is heuristic - if there are just a few test cases + # # (say <5) left undone for this label, and there is at least + # # 1 other machine working on this lable, we probably not want + # # to bother to reimage this dut to help with these 5 test + # # cases + # continue + + if v == "Y": + self.matrix_[i][j] = "_" + self._record_allocate_log(i, j) + return label + if v == " ": + label_reimage_number = len(self.label_duts_[i]) + if (can_label is None) or ( + label_reimage_number < can_reimage_number + or ( + label_reimage_number == can_reimage_number + and pending_br_num >= can_pending_br_num + ) + ): + can_reimage_number = label_reimage_number + can_i = i + can_label = label + can_pending_br_num = pending_br_num + + # All labels are marked either '_' (already taken) or 'X' (not + # compatible), so return None to notify machine thread to quit. + if can_label is None: + return None + + # At this point, we don't find any 'Y' for the machine, so we go the + # 'min' approach. + self.matrix_[can_i][j] = "_" + self._record_allocate_log(can_i, j) + return can_label + + def matrix_vertical_generator(self, col): + """Iterate matrix vertically at column 'col'. + + Yield row number i and value at matrix_[i][col]. + """ + for i, _ in enumerate(self.labels_): + yield i, self.matrix_[i][col] + + def matrix_horizontal_generator(self, row): + """Iterate matrix horizontally at row 'row'. + + Yield col number j and value at matrix_[row][j]. + """ + for j, _ in enumerate(self.duts_): + yield j, self.matrix_[row][j] + + def _compute_initial_allocation_internal(self, level, N): + """Search matrix for d with N.""" + + if level == self.n_labels_: return True - self.matrix_[level][j] = ' ' - return False + for j, v in self.matrix_horizontal_generator(level): + if v == " ": + # Before we put a 'Y', we check how many Y column 'j' has. + # Note y[0] is row idx, y[1] is the cell value. + ny = functools.reduce( + lambda x, y: x + 1 if (y[1] == "Y") else x, + self.matrix_vertical_generator(j), + 0, + ) + if ny < N: + self.matrix_[level][j] = "Y" + if self._compute_initial_allocation_internal(level + 1, N): + return True + self.matrix_[level][j] = " " + + return False |