Handle race condition when a test has errored but is not finalized. (#983)

Add partial test record upload with a background uploader thread.

Cleanup Add missing typing.Optional type annotations to function parameters.

Fixed parsing of default arguments provided to an OpenHTF phase.

Print the OpenHTF result instead of logging it.

Replace marginal field in mfg-i facing protos with new MARGINAL_PASS test status.

Fix bug causing data to not be uploaded when a ThreadTerminationError occurs.

Marginal pass - Propagate marginal determination to the test run level.

Fix a few bugs with AllInRange and add unit tests

Fix bug with is_marginal check on AllInRange validator

Remove circular dependency between diagnoses_lib and phase_descriptor

Move check_for_duplicate_results to phase_descriptor

Fix and update type annotations in diagnoses_lib

Update protos to output marginal determination upstream and add console coloring for marginal output cases.

Add marginal measurements. See go/openhtf-marginal-pass.

Added useful debugging messages to OpenHTF unit test results when they don't pass.

Add a typing.overload to execute_phase_or_test

Move openhtf.measures to the phase_descriptor module

Added a library to convert OpenHTF objects to strings.

Update built in validators to display the spec limits in the database.

Fix bug where plugs were being updated twice, resulting in tearDown being called.

Update unit test docs to cover TestCase.execute_phase_or_test.

Retry setsockopt when starting up the multicast thread

Add a decorator-free way to write unit tests.

Add capturing of instantiated plugs as an attribute on the test case.

Add get_attachment_or_die method to TestApi

Regenerate units with the latest UNECE publication (rec20_Rev15e-2020.xls).

Raise a clear Error message when a DeviceWrappingPlug is not fully initialized

Fix DUT input phase hung w/ ctrl+c (sigint).

Timeout when getting multicast.send() responses from queue

Add force_repeat option to force a repeat of phase up to the repeat_limit.

Adding the phase name to the phase outcome logging statements.

Fix type of conf when accessed as openhtf.conf

Give 3 retries for timeout phase as default; Add repeat_on_timeout option for phase

Replace phase_group member with either phase_sequence or phases when appropriate.
Add workaround for when AbortTest plug is not initialized (this happens sometimes, but is not easily reproducible).

PiperOrigin-RevId: 381093144

Author: cricdecyan

CONTRIBUTORS

@@ -17,3 +17,4 @@ Joe Ethier <[email protected]>
 John Hawley <[email protected]>
 Keith Suda-Cederquist <[email protected]>
 Kenneth Schiller <[email protected]>
+Christian Paulin <[email protected]>

examples/all_the_things.py

@@ -112,6 +112,23 @@ def measures_with_args(test, minimum, maximum):
   test.measurements.replaced_min_max = 1
 
 
+@htf.measures(
+    htf.Measurement('replaced_marginal_min_only').in_range(
+        0, 10, '{marginal_minimum}', 8, type=int),
+    htf.Measurement('replaced_marginal_max_only').in_range(
+        0, 10, 2, '{marginal_maximum}', type=int),
+    htf.Measurement('replaced_marginal_min_max').in_range(
+        0, 10, '{marginal_minimum}', '{marginal_maximum}', type=int),
+)
+def measures_with_marginal_args(test, marginal_minimum, marginal_maximum):
+  """Phase with measurement with marginal arguments."""
+  del marginal_minimum  # Unused.
+  del marginal_maximum  # Unused.
+  test.measurements.replaced_marginal_min_only = 3
+  test.measurements.replaced_marginal_max_only = 3
+  test.measurements.replaced_marginal_min_max = 3
+
+
 def attachments(test):
   test.attach('test_attachment',
               'This is test attachment data.'.encode('utf-8'))
@@ -156,6 +173,8 @@ def main():
           attachments,
           skip_phase,
           measures_with_args.with_args(minimum=1, maximum=4),
+          measures_with_marginal_args.with_args(
+              marginal_minimum=4, marginal_maximum=6),
           analysis,
       ),
       # Some metadata fields, these in particular are used by mfg-inspector,

examples/measurements.py

@@ -162,6 +162,18 @@ def multdim_measurements(test):
       test.measurements['average_current'])
 
 
+# Marginal measurements can be used to obtain a finer granularity of by how much
+# a measurement is passing. Marginal measurements have stricter minimum and
+# maximum limits, which are used to flag measurements/phase/test records as
+# marginal without affecting the overall phase outcome.
+@htf.measures(
+    htf.Measurement('resistance').with_units('ohm').in_range(
+        minimum=5, maximum=17, marginal_minimum=9, marginal_maximum=11))
+def marginal_measurements(test):
+  """Phase with a marginal measurement."""
+  test.measurements.resistance = 13
+
+
 def main():
   # We instantiate our OpenHTF test with the phases we want to run as args.
   test = htf.Test(hello_phase, again_phase, lots_of_measurements,

openhtf/__init__.py

@@ -14,12 +14,12 @@
 """The main OpenHTF entry point."""
 
 import signal
+import typing
 
 from openhtf import plugs
 from openhtf.core import phase_executor
 from openhtf.core import test_record
 from openhtf.core.base_plugs import BasePlug
-from openhtf.core.diagnoses_lib import diagnose
 from openhtf.core.diagnoses_lib import DiagnosesStore
 from openhtf.core.diagnoses_lib import Diagnosis
 from openhtf.core.diagnoses_lib import DiagnosisComponent
@@ -30,14 +30,15 @@
 
 from openhtf.core.measurements import Dimension
 from openhtf.core.measurements import Measurement
-from openhtf.core.measurements import measures
 from openhtf.core.monitors import monitors
 from openhtf.core.phase_branches import BranchSequence
 from openhtf.core.phase_branches import DiagnosisCheckpoint
 from openhtf.core.phase_branches import DiagnosisCondition
 from openhtf.core.phase_branches import PhaseFailureCheckpoint
 from openhtf.core.phase_collections import PhaseSequence
 from openhtf.core.phase_collections import Subtest
+from openhtf.core.phase_descriptor import diagnose
+from openhtf.core.phase_descriptor import measures
 from openhtf.core.phase_descriptor import PhaseDescriptor
 from openhtf.core.phase_descriptor import PhaseOptions
 from openhtf.core.phase_descriptor import PhaseResult
@@ -57,6 +58,9 @@
 from openhtf.util import units
 import pkg_resources
 
+if typing.TYPE_CHECKING:
+  conf: conf.Configuration  # Configuration is only available here in typing.
+
 
 def get_version():
   """Returns the version string of the 'openhtf' package.

openhtf/core/diagnoses_lib.py

@@ -123,13 +123,11 @@ def main():
 """
 
 import abc
-import collections
 import logging
-from typing import Any, Callable, DefaultDict, Dict, Iterable, Iterator, List, Optional, Sequence, Set, Text, Type, TYPE_CHECKING, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Text, Type, TYPE_CHECKING, Union
 
 import attr
 import enum  # pylint: disable=g-bad-import-order
-from openhtf.core import phase_descriptor
 from openhtf.core import test_record
 from openhtf.util import data
 import six
@@ -147,10 +145,6 @@ class InvalidDiagnosisError(Exception):
   """A Diagnosis was constructed incorrectly."""
 
 
-class DuplicateResultError(Exception):
-  """Different DiagResultEnum instances define the same value."""
-
-
 @attr.s(slots=True)
 class DiagnosesStore(object):
   """Storage and lookup of diagnoses."""
@@ -265,42 +259,6 @@ def execute_test_diagnoser(self, diagnoser: 'BaseTestDiagnoser',
       self._add_diagnosis(diag)
 
 
-def check_for_duplicate_results(
-    phase_iterator: Iterator[phase_descriptor.PhaseDescriptor],
-    test_diagnosers: Sequence['BaseTestDiagnoser']) -> None:
-  """Check for any results with the same enum value in different ResultTypes.
-
-  Args:
-    phase_iterator: iterator over the phases to check.
-    test_diagnosers: list of test level diagnosers.
-
-  Raises:
-    DuplicateResultError: when duplicate enum values are found.
-  """
-  all_result_enums = set()  # type: Set[Type['DiagResultEnum']]
-  for phase in phase_iterator:
-    for phase_diag in phase.diagnosers:
-      all_result_enums.add(phase_diag.result_type)
-  for test_diag in test_diagnosers:
-    all_result_enums.add(test_diag.result_type)
-
-  values_to_enums = collections.defaultdict(
-      list)  # type: DefaultDict[str, Type['DiagResultEnum']]
-  for enum_cls in all_result_enums:
-    for entry in enum_cls:
-      values_to_enums[entry.value].append(enum_cls)
-
-  duplicates = []  # type: List[str]
-  for result_value, enum_classes in sorted(values_to_enums.items()):
-    if len(enum_classes) > 1:
-      duplicates.append('Value "{}" defined by {}'.format(
-          result_value, enum_classes))
-  if not duplicates:
-    return
-  raise DuplicateResultError('Duplicate DiagResultEnum values: {}'.format(
-      '\n'.join(duplicates)))
-
-
 def _check_diagnoser(diagnoser: '_BaseDiagnoser',
                      diagnoser_cls: Type['_BaseDiagnoser']) -> None:
   """Check that a diagnoser is properly created."""
@@ -377,8 +335,7 @@ class BasePhaseDiagnoser(six.with_metaclass(abc.ABCMeta, _BaseDiagnoser)):
   __slots__ = ()
 
   @abc.abstractmethod
-  def run(self,
-          phase_record: phase_descriptor.PhaseDescriptor) -> DiagnoserReturnT:
+  def run(self, phase_record: test_record.PhaseRecord) -> DiagnoserReturnT:
     """Must be implemented to return list of Diagnoses instances.
 
     Args:
@@ -547,21 +504,3 @@ def __attrs_post_init__(self) -> None:
   def as_base_types(self) -> Dict[Text, Any]:
     return data.convert_to_base_types(
         attr.asdict(self, filter=_diagnosis_serialize_filter))
-
-
-def diagnose(
-    *diagnosers: BasePhaseDiagnoser
-) -> Callable[[phase_descriptor.PhaseT], phase_descriptor.PhaseDescriptor]:
-  """Decorator to add diagnosers to a PhaseDescriptor."""
-  check_diagnosers(diagnosers, BasePhaseDiagnoser)
-  diags = list(diagnosers)
-
-  def decorate(
-      wrapped_phase: phase_descriptor.PhaseT
-  ) -> phase_descriptor.PhaseDescriptor:
-    """Phase decorator to be returned."""
-    phase = phase_descriptor.PhaseDescriptor.wrap_or_copy(wrapped_phase)
-    phase.diagnosers.extend(diags)
-    return phase
-
-  return decorate

openhtf/core/measurements.py

@@ -22,7 +22,7 @@
 Measurements are described by the measurements.Measurement class.  Essentially,
 the Measurement class is used by test authors to declare measurements by name,
 and to optionally provide unit, type, and validation information.  Measurements
-are attached to Test Phases using the @measurements.measures() decorator.
+are attached to Test Phases using the @openhtf.measures() decorator.
 
 When measurements are output by the OpenHTF framework, the Measurement objects
 are serialized into the 'measurements' field on the PhaseRecord, which contain
@@ -45,11 +45,11 @@
 
 Examples:
 
-  @measurements.measures(
+  @openhtf.measures(
       measurements.Measurement(
           'number_widgets').in_range(5, 10).doc(
           '''This phase parameter tracks the number of widgets.'''))
-  @measurements.measures(
+  @openhtf.measures(
       *(measurements.Measurement('level_%s' % lvl)
         for lvl in ('none', 'some', 'all')))
   def WidgetTestPhase(test):
@@ -62,17 +62,18 @@ def WidgetTestPhase(test):
 import enum
 import functools
 import logging
+import typing
 from typing import Any, Callable, Dict, Iterator, List, Optional, Text, Tuple, Union
 
 import attr
 
 from openhtf import util
-from openhtf.core import diagnoses_lib
-from openhtf.core import phase_descriptor
 from openhtf.util import data
 from openhtf.util import units as util_units
 from openhtf.util import validators
 import six
+if typing.TYPE_CHECKING:
+  from openhtf.core import diagnoses_lib
 
 try:
   # pylint: disable=g-import-not-at-top
@@ -119,7 +120,7 @@ class _ConditionalValidator(object):
   """Conditional validator declaration."""
 
   # The diagnosis result required for the validator to be used.
-  result = attr.ib(type=diagnoses_lib.DiagResultEnum)
+  result = attr.ib(type='diagnoses_lib.DiagResultEnum')
 
   # The validator to use when the result is present.
   validator = attr.ib(type=Callable[[Any], bool])
@@ -187,6 +188,8 @@ class Measurement(object):
     notification_cb: An optional function to be called when the measurement is
       set.
     outcome: One of the Outcome() enumeration values, starting at UNSET.
+    marginal: A bool flag indicating if this measurement is marginal if the
+      outcome is PASS.
     _cached: A cached dict representation of this measurement created initially
       during as_base_types and updated in place to save allocation time.
   """
@@ -211,6 +214,7 @@ class Measurement(object):
       type=Union['MeasuredValue', 'DimensionedMeasuredValue'], default=None)
   _notification_cb = attr.ib(type=Optional[Callable[[], None]], default=None)
   outcome = attr.ib(type=Outcome, default=Outcome.UNSET)
+  marginal = attr.ib(type=bool, default=False)
 
   # Runtime cache to speed up conversions.
   _cached = attr.ib(type=Optional[Dict[Text, Any]], default=None)
@@ -341,7 +345,7 @@ def with_validator(self, validator: Callable[[Any], bool]) -> 'Measurement':
     return self
 
   def validate_on(
-      self, result_to_validator_mapping: Dict[diagnoses_lib.DiagResultEnum,
+      self, result_to_validator_mapping: Dict['diagnoses_lib.DiagResultEnum',
                                               Callable[[Any], bool]]
   ) -> 'Measurement':
     """Adds conditional validators.
@@ -414,11 +418,17 @@ def _with_validator(*args, **kwargs):
     return _with_validator
 
   def validate(self) -> 'Measurement':
-    """Validate this measurement and update its 'outcome' field."""
+    """Validate this measurement and update 'outcome' and 'marginal' fields."""
     # PASS if all our validators return True, otherwise FAIL.
     try:
       if all(v(self._measured_value.value) for v in self.validators):
         self.outcome = Outcome.PASS
+
+        # Only check marginality for passing measurements.
+        if any(
+            hasattr(v, 'is_marginal') and
+            v.is_marginal(self._measured_value.value) for v in self.validators):
+          self.marginal = True
       else:
         self.outcome = Outcome.FAIL
       return self
@@ -811,69 +821,9 @@ def __getitem__(self, name: Text) -> Any:
     # Return the MeasuredValue's value, MeasuredValue will raise if not set.
     return m.measured_value.value
 
+
 # Work around for attrs bug in 20.1.0; after the next release, this can be
 # removed and `Collection._custom_setattr` can be renamed to `__setattr__`.
 # https://github.com/python-attrs/attrs/issues/680
 Collection.__setattr__ = Collection._custom_setattr  # pylint: disable=protected-access
 del Collection._custom_setattr
-
-
-def measures(
-    *measurements: Union[Text, Measurement], **kwargs: Any
-) -> Callable[[phase_descriptor.PhaseT], phase_descriptor.PhaseDescriptor]:
-  """Decorator-maker used to declare measurements for phases.
-
-  See the measurements module docstring for examples of usage.
-
-  Args:
-    *measurements: Measurement objects to declare, or a string name from which
-      to create a Measurement.
-    **kwargs: Keyword arguments to pass to Measurement constructor if we're
-      constructing one.  Note that if kwargs are provided, the length of
-      measurements must be 1, and that value must be a string containing the
-      measurement name.  For valid kwargs, see the definition of the Measurement
-      class.
-
-  Raises:
-    InvalidMeasurementTypeError: When the measurement is not defined correctly.
-
-  Returns:
-    A decorator that declares the measurement(s) for the decorated phase.
-  """
-
-  def _maybe_make(meas: Union[Text, Measurement]) -> Measurement:
-    """Turn strings into Measurement objects if necessary."""
-    if isinstance(meas, Measurement):
-      return meas
-    elif isinstance(meas, six.string_types):
-      return Measurement(meas, **kwargs)
-    raise InvalidMeasurementTypeError('Expected Measurement or string', meas)
-
-  # In case we're declaring a measurement inline, we can only declare one.
-  if kwargs and len(measurements) != 1:
-    raise InvalidMeasurementTypeError(
-        'If @measures kwargs are provided, a single measurement name must be '
-        'provided as a positional arg first.')
-
-  # Unlikely, but let's make sure we don't allow overriding initial outcome.
-  if 'outcome' in kwargs:
-    raise ValueError('Cannot specify outcome in measurement declaration!')
-
-  measurements = [_maybe_make(meas) for meas in measurements]
-
-  # 'measurements' is guaranteed to be a list of Measurement objects here.
-  def decorate(
-      wrapped_phase: phase_descriptor.PhaseT
-  ) -> phase_descriptor.PhaseDescriptor:
-    """Phase decorator to be returned."""
-    phase = phase_descriptor.PhaseDescriptor.wrap_or_copy(wrapped_phase)
-    duplicate_names = (
-        set(m.name for m in measurements)
-        & set(m.name for m in phase.measurements))
-    if duplicate_names:
-      raise DuplicateNameError('Measurement names duplicated', duplicate_names)
-
-    phase.measurements.extend(measurements)
-    return phase
-
-  return decorate

openhtf/core/monitors.py

@@ -155,7 +155,7 @@ def wrapper(
 
     @openhtf.PhaseOptions(requires_state=True)
     @plugs.plug(update_kwargs=False, **monitor_plugs)
-    @measurements.measures(
+    @openhtf.measures(
         measurements.Measurement(measurement_name).with_units(
             units).with_dimensions(uom.MILLISECOND))
     @functools.wraps(phase_desc.func)