Skip to content

Commit 6627a7f

Browse files
bmd3kbmd3k
authored
Hparams: Allow sessions without name to pull all run,tag combinations as metrics. (#6546)
Motivation: Sometimes a user is viewing a single experiment and the DataProvider determines that experiment contains exactly one session. In this case the DataProvider will return a session with both empty experiment_id and empty run. We want this "empty-name session" to represent all runs in the experiment. But we have a bug where the /experiment and /session_group responses do not contain any metrics for the session. We want to instead designate all (run,tag) combinations as separate metrics to include in the /experiment and /session_groups responses. The first change to get this to work: * When generating session names, a session with experiment_id="" and run="" should generate an empty session_name "" (instead of using the somewhat-meaningless input experiment id as the session_name). The side-effect of the first change is that calls to _find_longest_parent_path() will return "" for all runs passed into it - effectively saying that all runs belong to the "empty-name session". So the second change to get this to work: * When _find_longest_parent_path() returns "" instead of None, treat this as a match with "empty-name session" instead of ignoring it.
1 parent 4bbe464 commit 6627a7f

File tree

4 files changed

+141
-6
lines changed

4 files changed

+141
-6
lines changed

‎tensorboard/plugins/hparams/backend_context.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ def compute_metric_infos_from_data_provider_session_groups(
403403
self, ctx, experiment_id, session_groups
404404
):
405405
session_runs = set(
406-
generate_data_provider_session_name(experiment_id, s)
406+
generate_data_provider_session_name(s)
407407
for sg in session_groups
408408
for s in sg.sessions
409409
)
@@ -446,7 +446,7 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
446446
)
447447
for run, tags in scalars_run_to_tag_to_content.items():
448448
session = _find_longest_parent_path(session_runs, run)
449-
if not session:
449+
if session is None:
450450
continue
451451
group = os.path.relpath(run, session)
452452
# relpath() returns "." for the 'session' directory, we use an empty
@@ -460,14 +460,14 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
460460
return metric_names_list
461461

462462

463-
def generate_data_provider_session_name(experiment_id, session):
463+
def generate_data_provider_session_name(session):
464464
"""Generates a name from a HyperparameterSesssionRun.
465465
466466
If the HyperparameterSessionRun contains no experiment or run information
467467
then the name is set to the original experiment_id.
468468
"""
469469
if not session.experiment_id and not session.run:
470-
return experiment_id
470+
return ""
471471
elif not session.experiment_id:
472472
return session.run
473473
elif not session.run:

‎tensorboard/plugins/hparams/backend_context_test.py

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -689,6 +689,106 @@ def test_experiment_from_data_provider_session_group_without_experiment_name(
689689
"""
690690
self.assertProtoEquals(expected_exp, actual_exp)
691691

692+
def test_experiment_from_data_provider_session_group_without_session_names(
693+
self,
694+
):
695+
self._mock_tb_context.data_provider.list_tensors.side_effect = None
696+
self._hyperparameters = provider.ListHyperparametersResult(
697+
hyperparameters=[],
698+
session_groups=[
699+
provider.HyperparameterSessionGroup(
700+
root=provider.HyperparameterSessionRun(
701+
experiment_id="", run=""
702+
),
703+
sessions=[
704+
provider.HyperparameterSessionRun(
705+
experiment_id="", run=""
706+
),
707+
],
708+
hyperparameter_values=[],
709+
),
710+
],
711+
)
712+
actual_exp = self._experiment_from_metadata()
713+
# The result specifies a single session without explicit identifier. It
714+
# therefore represents a session that includes all run/tag combinations
715+
# as separate metric values.
716+
expected_exp = """
717+
metric_infos {
718+
name {
719+
group: "exp/session_1"
720+
tag: "accuracy"
721+
}
722+
}
723+
metric_infos {
724+
name {
725+
group: "exp/session_2"
726+
tag: "accuracy"
727+
}
728+
}
729+
metric_infos {
730+
name {
731+
group: "exp/session_3"
732+
tag: "accuracy"
733+
}
734+
}
735+
metric_infos {
736+
name {
737+
group: "exp/session_1"
738+
tag: "loss"
739+
}
740+
}
741+
metric_infos {
742+
name {
743+
group: "exp/session_1/eval"
744+
tag: "loss"
745+
}
746+
}
747+
metric_infos {
748+
name {
749+
group: "exp/session_1/train"
750+
tag: "loss"
751+
}
752+
}
753+
metric_infos {
754+
name {
755+
group: "exp/session_2"
756+
tag: "loss"
757+
}
758+
}
759+
metric_infos {
760+
name {
761+
group: "exp/session_2/eval"
762+
tag: "loss"
763+
}
764+
}
765+
metric_infos {
766+
name {
767+
group: "exp/session_2/train"
768+
tag: "loss"
769+
}
770+
}
771+
metric_infos {
772+
name {
773+
group: "exp/session_3"
774+
tag: "loss"
775+
}
776+
}
777+
metric_infos {
778+
name {
779+
group: "exp/session_3/eval"
780+
tag: "loss"
781+
}
782+
}
783+
metric_infos {
784+
name {
785+
group: "exp/session_3xyz"
786+
tag: "loss2"
787+
}
788+
}
789+
"""
790+
self.assertProtoEquals(expected_exp, actual_exp)
791+
692792
def test_experiment_from_data_provider_old_response_type(self):
693793
self._hyperparameters = [
694794
provider.Hyperparameter(

‎tensorboard/plugins/hparams/list_session_groups.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ def _session_groups_from_data_provider(self):
136136
for session in provider_group.sessions:
137137
session_name = (
138138
backend_context_lib.generate_data_provider_session_name(
139-
self._experiment_id, session
139+
session
140140
)
141141
)
142142
sessions.append(
@@ -150,8 +150,10 @@ def _session_groups_from_data_provider(self):
150150
)
151151

152152
name = backend_context_lib.generate_data_provider_session_name(
153-
self._experiment_id, provider_group.root
153+
provider_group.root
154154
)
155+
if not name:
156+
name = self._experiment_id
155157
session_group = api_pb2.SessionGroup(
156158
name=name,
157159
sessions=sessions,

‎tensorboard/plugins/hparams/list_session_groups_test.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2066,6 +2066,39 @@ def test_experiment_from_data_provider_with_metric_values_from_run_name(
20662066
response.session_groups[0].sessions[0],
20672067
)
20682068

2069+
def test_experiment_from_data_provider_with_metric_values_empty_session_names(
2070+
self,
2071+
):
2072+
self._mock_tb_context.data_provider.list_tensors.side_effect = None
2073+
self._hyperparameters = [
2074+
provider.HyperparameterSessionGroup(
2075+
root=provider.HyperparameterSessionRun(
2076+
experiment_id="", run=""
2077+
),
2078+
sessions=[
2079+
provider.HyperparameterSessionRun(experiment_id="", run="")
2080+
],
2081+
hyperparameter_values=[],
2082+
),
2083+
]
2084+
request = """
2085+
start_index: 0
2086+
slice_size: 10
2087+
"""
2088+
response = self._run_handler(request)
2089+
self.assertLen(response.session_groups, 1)
2090+
# The name comes from the experiment id.
2091+
self.assertEquals(response.session_groups[0].name, "123")
2092+
self.assertLen(response.session_groups[0].sessions, 1)
2093+
self.assertEquals(response.session_groups[0].sessions[0].name, "")
2094+
# The result specifies a single session without explicit identifier. It
2095+
# therefore represents a session that includes all run/tag combinations
2096+
# as separate metric values.
2097+
# There are 11 total run/tag combinations across session_1, _2, _3, _4,
2098+
# and _5.
2099+
self.assertLen(response.session_groups[0].metric_values, 11)
2100+
self.assertLen(response.session_groups[0].sessions[0].metric_values, 11)
2101+
20692102
def test_experiment_from_data_provider_with_metric_values_aggregates(
20702103
self,
20712104
):

0 commit comments

Comments
 (0)