only use one coordinate data (#512)

* only use one coordinate data

* improve/fix tests

* minor tweak

Author: tgerdesnv

model_analyzer/config/generate/coordinate_data.py

@@ -26,6 +26,7 @@ class CoordinateData:
     def __init__(self):
         self._measurements = {}
         self._visit_counts = {}
+        self._is_measured = {}
 
     def get_measurement(
             self, coordinate: Coordinate) -> Optional[RunConfigMeasurement]:
@@ -42,6 +43,20 @@ def set_measurement(self, coordinate: Coordinate,
         """
         key: Tuple[Coordinate, ...] = tuple(coordinate)
         self._measurements[key] = measurement
+        self._is_measured[key] = True
+
+    def is_measured(self, coordinate: Coordinate) -> bool:
+        """
+        Returns true if a measurement has been set for the given Coordinate
+        """
+        key: Tuple[Coordinate, ...] = tuple(coordinate)
+        return self._is_measured.get(key, False)
+
+    def has_valid_measurement(self, coordinate: Coordinate) -> bool:
+        """
+        Returns true if there is a valid measurement for the given Coordinate
+        """
+        return self.get_measurement(coordinate) is not None
 
     def reset_measurements(self):
         """

model_analyzer/config/generate/neighborhood.py

@@ -42,7 +42,7 @@ class Neighborhood:
     TRANSLATION_LIST = [0.09, 0.3, 1.0]
 
     def __init__(self, neighborhood_config: NeighborhoodConfig,
-                 home_coordinate: Coordinate):
+                 home_coordinate: Coordinate, coordinate_data: CoordinateData):
         """
         Parameters
         ----------
@@ -55,17 +55,13 @@ def __init__(self, neighborhood_config: NeighborhoodConfig,
 
         self._config = neighborhood_config
         self._home_coordinate = home_coordinate
-        self._coordinate_data = CoordinateData()
+        self._coordinate_data = coordinate_data
 
         self._radius = self._config.get_radius()
         self._neighborhood = self._create_neighborhood()
 
         self._force_slow_mode = False
 
-    @property
-    def coordinate_data(self):
-        return self._coordinate_data
-
     @classmethod
     def calc_distance(cls, coordinate1: Coordinate,
                       coordinate2: Coordinate) -> float:
@@ -83,16 +79,17 @@ def calc_distance(cls, coordinate1: Coordinate,
     def enough_coordinates_initialized(self) -> bool:
         """
         Returns true if enough coordinates inside of the neighborhood
-        have been initialized. Else false
+        have been initialized with valid measurements. Else false
 
         If the neighborhood is in slow mode, this means all adjacent neighbors
         must be visited
         """
         if self._is_slow_mode():
-            return self._are_all_adjacent_neighbors_visited()
+            return self._are_all_adjacent_neighbors_measured()
         else:
             min_initialized = self._config.get_min_initialized()
-            num_initialized = len(self._get_initialized_coordinates())
+            num_initialized = len(
+                self._get_coordinates_with_valid_measurements())
             return num_initialized >= min_initialized
 
     def force_slow_mode(self):
@@ -178,7 +175,7 @@ def pick_coordinate_to_initialize(self) -> Optional[Coordinate]:
 
     def _pick_slow_mode_coordinate_to_initialize(self):
         for neighbor in self._get_all_adjacent_neighbors():
-            if not self._is_coordinate_visited(neighbor):
+            if not self._is_coordinate_measured(neighbor):
                 return neighbor
 
         raise Exception("Picking slow mode coordinate, but none are unvisited")
@@ -189,7 +186,7 @@ def _pick_fast_mode_coordinate_to_initialize(self):
         max_num_uncovered = -1
         best_coordinate = None
         for coordinate in self._neighborhood:
-            if not self._is_coordinate_visited(coordinate):
+            if not self._is_coordinate_measured(coordinate):
                 num_uncovered = self._get_num_uncovered_values(
                     coordinate, covered_values_per_dimension)
 
@@ -257,23 +254,11 @@ def _enumerate_all_values_in_bounds(
         tuples = list(product(*possible_index_values))
         return [list(x) for x in tuples]
 
-    def _get_visited_coordinates(self) -> List[Coordinate]:
-        """
-        Returns the list of coordinates in the neighborhood that have been
-        visited (except the home coordinate).
-        """
-        visited_coordinates = []
-        for coordinate in self._neighborhood:
-            if coordinate != self._home_coordinate \
-                    and self._is_coordinate_visited(coordinate):
-                visited_coordinates.append(deepcopy(coordinate))
-        return visited_coordinates
-
-    def _get_initialized_coordinates(self) -> List[Coordinate]:
+    def _get_coordinates_with_valid_measurements(self) -> List[Coordinate]:
         initialized_coordinates = []
         for coordinate in self._neighborhood:
-            if coordinate != self._home_coordinate \
-                    and self._is_coordinate_initialized(coordinate):
+            if coordinate != self._home_coordinate and self._coordinate_data.has_valid_measurement(
+                    coordinate):
                 initialized_coordinates.append(deepcopy(coordinate))
         return initialized_coordinates
 
@@ -296,7 +281,7 @@ def _calculate_step_vector_from_measurements(
             self, compare_constraints: bool) -> Coordinate:
 
         home_measurement = self._get_home_measurement()
-        vectors, measurements = self._get_all_visited_measurements()
+        vectors, measurements = self._get_all_measurements()
 
         # This function should only ever be called if all are passing or none are passing
         _, p = self._get_measurements_passing_constraints()
@@ -340,7 +325,7 @@ def _calculate_step_vector_from_vectors_and_weights(self, vectors, weights):
 
         return step_vector
 
-    def _get_all_visited_measurements(
+    def _get_all_measurements(
             self) -> Tuple[List[Coordinate], List[RunConfigMeasurement]]:
         """
         Gather all the visited vectors (directions from the home coordinate)
@@ -351,10 +336,11 @@ def _get_all_visited_measurements(
         (vectors, measurements)
             collection of vectors and their measurements.
         """
-        visited_coordinates = self._get_visited_coordinates()
+        coordinates = self._get_coordinates_with_valid_measurements()
+
         vectors = []
         measurements = []
-        for coordinate in visited_coordinates:
+        for coordinate in coordinates:
             measurement = self._coordinate_data.get_measurement(coordinate)
             if measurement:
                 vectors.append(coordinate - self._home_coordinate)
@@ -372,22 +358,19 @@ def _get_measurements_passing_constraints(
         (vectors, measurements)
             collection of vectors and their measurements.
         """
-        visited_coordinates = self._get_visited_coordinates()
+        coordinates = self._get_coordinates_with_valid_measurements()
 
         vectors = []
         measurements = []
-        for coordinate in visited_coordinates:
+        for coordinate in coordinates:
             measurement = self._coordinate_data.get_measurement(coordinate)
             if measurement and measurement.is_passing_constraints():
                 vectors.append(coordinate - self._home_coordinate)
                 measurements.append(measurement)
         return vectors, measurements
 
-    def _is_coordinate_visited(self, coordinate: Coordinate) -> bool:
-        return self._coordinate_data.get_visit_count(coordinate) > 0
-
-    def _is_coordinate_initialized(self, coordinate: Coordinate) -> bool:
-        return self._coordinate_data.get_measurement(coordinate) is not None
+    def _is_coordinate_measured(self, coordinate: Coordinate) -> bool:
+        return self._coordinate_data.is_measured(coordinate)
 
     def _clamp_coordinate_to_bounds(self, coordinate: Coordinate) -> Coordinate:
 
@@ -409,13 +392,13 @@ def _get_covered_values_per_dimension(self) -> List[Dict[Coordinate, bool]]:
         (e.g.)
             covered_values_per_dimension[dimension][value] = bool
         """
-        visited_coordinates = self._get_visited_coordinates()
+        measured_coordinates = self._get_coordinates_with_valid_measurements()
 
         covered_values_per_dimension: List[Dict[Coordinate, bool]] = [
             {} for _ in range(self._config.get_num_dimensions())
         ]
 
-        for coordinate in visited_coordinates:
+        for coordinate in measured_coordinates:
             for i, v in enumerate(coordinate):
                 covered_values_per_dimension[i][v] = True
 
@@ -444,7 +427,7 @@ def _is_slow_mode(self):
             return False
 
         passing_vectors, _ = self._get_measurements_passing_constraints()
-        all_vectors, _ = self._get_all_visited_measurements()
+        all_vectors, _ = self._get_all_measurements()
 
         any_failing = len(all_vectors) != len(passing_vectors)
         any_passing = len(passing_vectors) != 0
@@ -453,9 +436,9 @@ def _is_slow_mode(self):
         return (home_passing and any_failing) or (not home_passing and
                                                   any_passing)
 
-    def _are_all_adjacent_neighbors_visited(self):
+    def _are_all_adjacent_neighbors_measured(self):
         for neighbor in self._get_all_adjacent_neighbors():
-            if not self._is_coordinate_visited(neighbor):
+            if not self._is_coordinate_measured(neighbor):
                 return False
         return True
 

model_analyzer/config/generate/quick_run_config_generator.py

@@ -90,7 +90,7 @@ def __init__(self, search_config: SearchConfig,
 
         self._neighborhood = Neighborhood(
             self._search_config.get_neighborhood_config(),
-            self._home_coordinate)
+            self._home_coordinate, self._coordinate_data)
 
         # Sticky bit. Once true, we should never stay at a home that is failing or None
         self._home_has_passed = False
@@ -142,11 +142,7 @@ def set_last_results(self, measurements: List[Union[RunConfigMeasurement,
         ----------
         measurements: List of Measurements from the last run(s)
         """
-        self._coordinate_data.increment_visit_count(self._coordinate_to_measure)
-        self._neighborhood.coordinate_data.increment_visit_count(
-            coordinate=self._coordinate_to_measure)
-
-        self._neighborhood.coordinate_data.set_measurement(
+        self._coordinate_data.set_measurement(
             coordinate=self._coordinate_to_measure, measurement=measurements[0])
 
         if measurements[0] is not None:
@@ -187,8 +183,8 @@ def _update_best_measurement(self, measurement: RunConfigMeasurement):
                 self._best_coordinate = self._coordinate_to_measure
                 self._best_measurement = measurement
 
-    def _get_last_results(self) -> RunConfigMeasurement:
-        return self._neighborhood.coordinate_data.get_measurement(
+    def _get_last_results(self) -> Optional[RunConfigMeasurement]:
+        return self._coordinate_data.get_measurement(
             coordinate=self._coordinate_to_measure)
 
     def _take_step(self):
@@ -245,7 +241,11 @@ def _recreate_neighborhood(self, force_slow_mode: bool):
         neighborhood_config = self._search_config.get_neighborhood_config()
 
         self._neighborhood = Neighborhood(neighborhood_config,
-                                          self._home_coordinate)
+                                          self._home_coordinate,
+                                          self._coordinate_data)
+
+        self._coordinate_data.increment_visit_count(self._home_coordinate)
+
         if force_slow_mode:
             self._neighborhood.force_slow_mode()
 

tests/test_coordinate_data.py

@@ -24,9 +24,7 @@
 
 class TestCoordinateData(trc.TestResultCollector):
 
-    def _construct_rcm(self,
-                       throughput: float,
-                       latency: float,
+    def _construct_rcm(self, throughput: float, latency: float,
                        config_name: str):
         model_config_name = [config_name]
 
@@ -47,8 +45,7 @@ def _construct_rcm(self,
             gpu_metric_values={},
             non_gpu_metric_values=non_gpu_metric_values,
             metric_objectives=metric_objectives,
-            model_config_weights=weights
-        )
+            model_config_weights=weights)
         return rcm
 
     def test_basic(self):
@@ -57,6 +54,8 @@ def test_basic(self):
         coordinate = Coordinate([0, 0, 0])
         self.assertEqual(result_data.get_measurement(coordinate), None)
         self.assertEqual(result_data.get_visit_count(coordinate), 0)
+        self.assertEqual(result_data.is_measured(coordinate), False)
+        self.assertEqual(result_data.has_valid_measurement(coordinate), False)
 
     def test_visit_count(self):
         result_data = CoordinateData()
@@ -77,24 +76,49 @@ def test_visit_count(self):
 
     def test_measurement(self):
         """
-        Test if CoordinateData can properly set and get the measurements.
+        Test if CoordinateData can properly set and get measurements
+
+        Also confirm that is_measured() and has_valid_measurement() work properly
         """
         coordinate_data = CoordinateData()
 
         coordinate0 = Coordinate([0, 0, 0])
         coordinate1 = Coordinate([0, 4, 1])
+        coordinate2 = Coordinate([1, 2, 3])
 
         rcm0 = self._construct_rcm(10, 5, config_name="modelA_config_0")
         rcm1 = self._construct_rcm(20, 8, config_name="modelB_config_0")
+        rcm2 = None
 
         coordinate_data.set_measurement(coordinate0, rcm0)
+        coordinate_data.set_measurement(coordinate1, rcm1)
+        coordinate_data.set_measurement(coordinate2, rcm2)
+
+        self.assertEqual(coordinate_data.is_measured(coordinate0), True)
+        self.assertEqual(coordinate_data.is_measured(coordinate1), True)
+        self.assertEqual(coordinate_data.is_measured(coordinate2), True)
+
+        self.assertEqual(coordinate_data.has_valid_measurement(coordinate0),
+                         True)
+        self.assertEqual(coordinate_data.has_valid_measurement(coordinate1),
+                         True)
+        self.assertEqual(coordinate_data.has_valid_measurement(coordinate2),
+                         False)
+
         measurement0 = coordinate_data.get_measurement(coordinate0)
         self.assertEqual("modelA_config_0", measurement0.model_variants_name())
-        self.assertEqual(10, measurement0.get_non_gpu_metric_value("perf_throughput"))
-        self.assertEqual(5, measurement0.get_non_gpu_metric_value("perf_latency_avg"))
+        self.assertEqual(
+            10, measurement0.get_non_gpu_metric_value("perf_throughput"))
+        self.assertEqual(
+            5, measurement0.get_non_gpu_metric_value("perf_latency_avg"))
+        self.assertTrue(coordinate_data.is_measured(coordinate0))
 
-        coordinate_data.set_measurement(coordinate1, rcm1)
         measurement1 = coordinate_data.get_measurement(coordinate1)
         self.assertEqual("modelB_config_0", measurement1.model_variants_name())
-        self.assertEqual(20, measurement1.get_non_gpu_metric_value("perf_throughput"))
-        self.assertEqual(8, measurement1.get_non_gpu_metric_value("perf_latency_avg"))
+        self.assertEqual(
+            20, measurement1.get_non_gpu_metric_value("perf_throughput"))
+        self.assertEqual(
+            8, measurement1.get_non_gpu_metric_value("perf_latency_avg"))
+
+        measurement2 = coordinate_data.get_measurement(coordinate2)
+        self.assertEqual(measurement2, None)