Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions concordia/components/game_master/open_ended_questionnaire.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,21 +237,25 @@ def _process_answer(
questionnaire.questionnaire_type == 'free'
or questionnaire.questionnaire_type == 'open-ended'
):
# Embedd answer and choices, computer their cosine similarity
answer_embedding = self._embedder(answer_text) # pyrefly: ignore[not-callable]
choice_similarities = []
for choice in current_question.choices: # pyrefly: ignore[not-iterable]
choice_embedding = self._embedder(choice) # pyrefly: ignore[not-callable]
similarity = np.dot(answer_embedding, choice_embedding)
choice_similarities.append({'choice': choice, 'similarity': similarity})
answer_embedding = None
value = answer_text
if self._embedder:
# Embed answer and choices, compute their cosine similarity.
answer_embedding = self._embedder(answer_text)
choice_similarities = []
for choice in current_question.choices: # pyrefly: ignore[not-iterable]
choice_embedding = self._embedder(choice)
similarity = np.dot(answer_embedding, choice_embedding)
choice_similarities.append({'choice': choice, 'similarity': similarity})
value = choice_similarities

self._answers[self._event_counter][player_name][questionnaire_name][
question_id
] = {
'statement': current_question.statement,
'text': answer_text,
'dimension': dimension,
'value': choice_similarities,
'value': value,
'embedding': answer_embedding,
}
elif (
Expand Down
96 changes: 96 additions & 0 deletions concordia/components/game_master/open_ended_questionnaire_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Copyright 2026 DeepMind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the OpenEndedQuestionnaire component."""

from absl.testing import absltest
from concordia.components.game_master import open_ended_questionnaire
from concordia.contrib.data.questionnaires import base_questionnaire
import numpy as np


class _FreeQuestionnaire(base_questionnaire.QuestionnaireBase):
"""Minimal free-text questionnaire for tests."""

def aggregate_results(self, player_answers):
return {}

def plot_results(self, results_df, label_column=None, kwargs=None):
pass

def get_dimension_ranges(self):
return {}


class OpenEndedQuestionnaireTest(absltest.TestCase):
"""Tests for the OpenEndedQuestionnaire component."""

def _make_component(self, questionnaire_type, embedder):
questionnaire = _FreeQuestionnaire(
name='mood',
description='Mood survey',
questionnaire_type=questionnaire_type,
observation_preprompt='Answer the question.',
questions=[
base_questionnaire.Question(
statement='How do you feel?',
dimension='mood',
choices=['good', 'bad'],
)
],
)
return open_ended_questionnaire.OpenEndedQuestionnaire(
questionnaires=[questionnaire],
player_names=['Alice'],
sequence_of_events=['event'],
embedder=embedder,
)

def test_free_answer_without_embedder_does_not_crash(self):
# Answering a free-text question without an embedder used to raise
# `TypeError: 'NoneType' object is not callable`.
component = self._make_component('free', embedder=None)
component.pre_observe('[putative_event] Alice: mood_0: happy')
answer = component.get_answers()[0]['Alice']['mood']['mood_0']
self.assertEqual(answer['text'], 'happy')
self.assertEqual(answer['value'], 'happy')
self.assertIsNone(answer['embedding'])

def test_open_ended_answer_without_embedder_does_not_crash(self):
component = self._make_component('open-ended', embedder=None)
component.pre_observe('[putative_event] Alice: mood_0: happy')
answer = component.get_answers()[0]['Alice']['mood']['mood_0']
self.assertEqual(answer['value'], 'happy')

def test_free_answer_with_embedder_computes_similarities(self):
def embedder(text):
# Deterministic one-hot embedding keyed on the first letter.
index = ord(text[0].lower()) - ord('a')
vector = np.zeros(26)
vector[index] = 1.0
return vector

component = self._make_component('open-ended', embedder=embedder)
component.pre_observe('[putative_event] Alice: mood_0: great')
answer = component.get_answers()[0]['Alice']['mood']['mood_0']
self.assertEqual(answer['text'], 'great')
# 'great' and 'good' both start with 'g' so their similarity is 1.0 while
# 'bad' starts with 'b' so its similarity is 0.0.
self.assertEqual(answer['value'][0], {'choice': 'good', 'similarity': 1.0})
self.assertEqual(answer['value'][1], {'choice': 'bad', 'similarity': 0.0})
self.assertIsNotNone(answer['embedding'])


if __name__ == '__main__':
absltest.main()
3 changes: 2 additions & 1 deletion concordia/contrib/language_models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
_REGISTRY = types.MappingProxyType({
'amazon_bedrock': 'amazon.amazon_bedrock_model.AmazonBedrockLanguageModel',
'gemini': 'google.gemini_model.GeminiModel',
'gemini_vision': 'google.gemini_model_vision.GeminiModelVision',
'gemini_vision': 'google.gemini_model_multimodal.GeminiModelVision',
'google_aistudio': 'google.gemini_model.GeminiModel',
'google_cloud_custom_model': 'google.google_cloud_custom_model.VertexAI',
'groq': 'groq.groq_model.GroqModel',
'huggingface': 'huggingface.huggingface_model.HuggingFaceLanguageModel',
Expand Down
45 changes: 45 additions & 0 deletions concordia/contrib/language_models/language_models_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Copyright 2026 DeepMind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the language model setup registry."""

import importlib.util

from absl.testing import absltest
from concordia.contrib import language_models


class LanguageModelsRegistryTest(absltest.TestCase):
"""Tests that every registry entry resolves to an importable module."""

def test_all_registry_entries_resolve_to_modules(self):
for api_type, model_path in language_models._REGISTRY.items():
module_path, _ = model_path.rsplit('.', 1)
full_module = f'concordia.contrib.language_models.{module_path}'
self.assertIsNotNone(
importlib.util.find_spec(full_module),
msg=(
f'api_type {api_type!r} references module {full_module!r} which'
' does not exist.'
),
)

def test_google_aistudio_is_a_registered_api_type(self):
# `google_aistudio` is the default api_type used by the example run
# scripts and the persona generator CLI.
self.assertIn('google_aistudio', language_models._REGISTRY)


if __name__ == '__main__':
absltest.main()
Loading