Add slow mode to quick search (#510)

* Add slow mode support

* next door -> adjacent

* remove redundant comment

* update some sorting code

Author: tgerdesnv

model_analyzer/config/generate/coordinate.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from copy import copy
+from copy import deepcopy
 
 
 class Coordinate:
@@ -25,7 +25,7 @@ def __init__(self, val):
         val: list
             List of floats or integers cooresponding to the location in space
         """
-        self._values = copy(val)
+        self._values = deepcopy(val)
 
     def __getitem__(self, idx):
         return self._values[idx]

model_analyzer/config/generate/neighborhood.py

@@ -49,6 +49,8 @@ def __init__(self, neighborhood_config: NeighborhoodConfig,
         self._radius = self._config.get_radius()
         self._neighborhood = self._create_neighborhood()
 
+        self._force_slow_mode = False
+
     @property
     def coordinate_data(self):
         return self._coordinate_data
@@ -71,31 +73,75 @@ def enough_coordinates_initialized(self) -> bool:
         """
         Returns true if enough coordinates inside of the neighborhood
         have been initialized. Else false
+
+        If the neighborhood is in slow mode, this means all adjacent neighbors
+        must be visited
+        """
+        if self._is_slow_mode():
+            return self._are_all_adjacent_neighbors_visited()
+        else:
+            min_initialized = self._config.get_min_initialized()
+            num_initialized = len(self._get_initialized_coordinates())
+            return num_initialized >= min_initialized
+
+    def force_slow_mode(self):
         """
-        min_initialized = self._config.get_min_initialized()
-        num_initialized = len(self._get_initialized_coordinates())
-        return num_initialized >= min_initialized
+        When called, forces the neighborhood into slow mode
+        """
+        self._force_slow_mode = True
 
     def calculate_new_coordinate(self,
                                  magnitude: int,
                                  enable_clipping: bool = True,
                                  clip_value: int = 2) -> Coordinate:
         """
         Based on the measurements in the neighborhood, determine where
-        the next location should be
+        the next location should be.
+
+        If the neighborhood is in slow mode, return the best found measurement
+        Otherwise calculate a new coordinate from the measurements
 
         Parameters
         ----------
         magnitude
             How large of a step to take
-        disable_clipping
+        enable_clipping
             Determines whether or not to clip the final step vector.
+        clip_value
+            What value to clip the vector at, if it is enabled
 
         Returns
         -------
         new_coordinate
             The new coordinate computed based on the neighborhood measurements.
         """
+
+        if self._is_slow_mode():
+            return self._get_best_coordinate_found()
+        else:
+            return self._calculate_new_coordinate(magnitude, enable_clipping,
+                                                  clip_value)
+
+    def _get_best_coordinate_found(self) -> Coordinate:
+        vectors, measurements = self._get_measurements_passing_constraints()
+
+        if len(vectors) == 0:
+            return self._home_coordinate
+
+        home_measurement = self._get_home_measurement()
+
+        if home_measurement.is_passing_constraints():
+            vectors.append(Coordinate([0] * self._config.get_num_dimensions()))
+            measurements.append(home_measurement)
+
+        _, best_vector = sorted(zip(measurements, vectors))[-1]
+
+        best_coordinate = self._home_coordinate + best_vector
+        return best_coordinate
+
+    def _calculate_new_coordinate(self, magnitude, enable_clipping,
+                                  clip_value) -> Coordinate:
+
         step_vector = self._get_step_vector() * magnitude
 
         if enable_clipping:
@@ -133,14 +179,30 @@ def _clip_vector_values(self, vector: Coordinate,
 
         if max_value > clip_value and max_value != 0:
             for i in range(len(vector)):
-                vector[i] = clip_value * vector[i]/max_value
+                vector[i] = clip_value * vector[i] / max_value
         return vector
 
     def pick_coordinate_to_initialize(self) -> Optional[Coordinate]:
         """
         Based on the initialized coordinate values, pick an unvisited
         coordinate to initialize next.
+
+        If the neighborhood is in slow mode, only pick from within the adjacent neighbors
         """
+
+        if self._is_slow_mode():
+            return self._pick_slow_mode_coordinate_to_initialize()
+        else:
+            return self._pick_fast_mode_coordinate_to_initialize()
+
+    def _pick_slow_mode_coordinate_to_initialize(self):
+        for neighbor in self._get_all_adjacent_neighbors():
+            if not self._is_coordinate_visited(neighbor):
+                return neighbor
+
+        raise Exception("Picking slow mode coordinate, but none are unvisited")
+
+    def _pick_fast_mode_coordinate_to_initialize(self):
         covered_values_per_dimension = self._get_covered_values_per_dimension()
 
         max_num_uncovered = -1
@@ -250,12 +312,11 @@ def _get_step_vector(self) -> Coordinate:
         step_vector
             a coordinate that tells the direction to move.
         """
-        home_measurement = self._coordinate_data.get_measurement(
-            coordinate=self._home_coordinate)
+        home_measurement = self._get_home_measurement()
 
         assert home_measurement is not None, "Home measurement cannot be NoneType."
 
-        if home_measurement.is_passing_constraints():
+        if self._is_home_passing_constraints():
             return self._optimize_for_objectives(home_measurement)
 
         return self._optimize_for_constraints(home_measurement)
@@ -428,3 +489,58 @@ def _get_num_uncovered_values(
                 num_uncovered += 1
 
         return num_uncovered
+
+    def _is_slow_mode(self):
+        if self._force_slow_mode:
+            return True
+
+        if not self._is_home_measured():
+            return False
+
+        passing_vectors, _ = self._get_measurements_passing_constraints()
+        all_vectors, _ = self._get_all_visited_measurements()
+
+        any_failing = len(all_vectors) != len(passing_vectors)
+        any_passing = len(passing_vectors) != 0
+        home_passing = self._is_home_passing_constraints()
+
+        return (home_passing and any_failing) or (not home_passing and
+                                                  any_passing)
+
+    def _are_all_adjacent_neighbors_visited(self):
+        for neighbor in self._get_all_adjacent_neighbors():
+            if not self._is_coordinate_visited(neighbor):
+                return False
+        return True
+
+    def _get_all_adjacent_neighbors(self):
+        adjacent_neighbors = []
+
+        for dim in range(self._config.get_num_dimensions()):
+            dimension = self._config.get_dimension(dim)
+
+            down_neighbor = Coordinate(self._home_coordinate)
+            down_neighbor[dim] -= 1
+            if down_neighbor[dim] >= dimension.get_min_idx():
+                adjacent_neighbors.append(down_neighbor)
+
+            up_neighbor = Coordinate(self._home_coordinate)
+            up_neighbor[dim] += 1
+            if up_neighbor[dim] <= dimension.get_max_idx():
+                adjacent_neighbors.append(up_neighbor)
+
+        return adjacent_neighbors
+
+    def _get_home_measurement(self):
+        return self._coordinate_data.get_measurement(
+            coordinate=self._home_coordinate)
+
+    def _is_home_measured(self):
+        return self._get_home_measurement() is not None
+
+    def _is_home_passing_constraints(self):
+        if not self._is_home_measured():
+            raise Exception("Can't check home passing if it isn't measured yet")
+
+        home_measurement = self._get_home_measurement()
+        return home_measurement.is_passing_constraints()

model_analyzer/config/generate/quick_run_config_generator.py

@@ -95,6 +95,9 @@ def __init__(self, search_config: SearchConfig,
             self._search_config.get_neighborhood_config(),
             self._home_coordinate)
 
+        # Sticky bit. Once true, we should never stay at a home that is failing or None
+        self._home_has_passed = False
+
         self._done = False
 
     def _is_done(self) -> bool:
@@ -121,8 +124,7 @@ def _step(self):
         Determine self._coordinate_to_measure, which is what is used to
         create the next RunConfig
         """
-        if self._measuring_home_coordinate(
-        ) and self._get_last_results() is None:
+        if self._should_step_back():
             self._take_step_back()
         elif self._neighborhood.enough_coordinates_initialized():
             self._take_step()
@@ -153,6 +155,10 @@ def set_last_results(self, measurements: List[Union[RunConfigMeasurement,
         if measurements[0] is not None:
             self._update_best_measurement(measurement=measurements[0])
 
+            if self._measuring_home_coordinate(
+            ) and measurements[0].is_passing_constraints():
+                self._home_has_passed = True
+
         self._print_debug_logs(measurements)
 
     def _update_best_measurement(self, measurement: RunConfigMeasurement):
@@ -197,20 +203,38 @@ def _take_step(self):
         logger.debug(f"Stepping {self._home_coordinate}->{new_coordinate}")
         self._home_coordinate = new_coordinate
         self._coordinate_to_measure = new_coordinate
-        self._recreate_neighborhood()
+        self._recreate_neighborhood(force_slow_mode=False)
 
     def _take_step_back(self):
         new_coordinate = self._neighborhood.get_nearest_neighbor(
             coordinate_in=self._best_coordinate)
 
+        # TODO: TMA-871: handle back-off (and its termination) better.
+        if new_coordinate == self._home_coordinate:
+            self._done = True
+
         logger.debug(
             f"Stepping back: {self._home_coordinate}->{new_coordinate}")
         self._home_coordinate = new_coordinate
         self._coordinate_to_measure = new_coordinate
-        self._recreate_neighborhood()
+        self._recreate_neighborhood(force_slow_mode=True)
 
         self._magnitude_scaler *= MAGNITUDE_DECAY_RATE
 
+    def _should_step_back(self):
+        """
+        Step back if take any of the following steps:
+          - Step from a passing home to a failing home
+          - Step from any home to home with a None measurement
+        """
+        if self._measuring_home_coordinate():
+            if self._get_last_results() is None:
+                return True
+            last_results_passed = self._get_last_results(
+            ).is_passing_constraints()
+            if not last_results_passed and self._home_has_passed:
+                return True
+
     def _measuring_home_coordinate(self):
         return self._coordinate_to_measure == self._home_coordinate
 
@@ -224,11 +248,13 @@ def _determine_if_done(self, new_coordinate: Coordinate):
         if self._coordinate_data.get_visit_count(new_coordinate) >= 2:
             self._done = True
 
-    def _recreate_neighborhood(self):
+    def _recreate_neighborhood(self, force_slow_mode: bool):
         neighborhood_config = self._search_config.get_neighborhood_config()
 
         self._neighborhood = Neighborhood(neighborhood_config,
                                           self._home_coordinate)
+        if force_slow_mode:
+            self._neighborhood.force_slow_mode()
 
     def _pick_coordinate_to_initialize(self):
         self._coordinate_to_measure = self._neighborhood.pick_coordinate_to_initialize(