diff --git a/concordia/components/game_master/open_ended_questionnaire.py b/concordia/components/game_master/open_ended_questionnaire.py index b10103bad..a10007447 100644 --- a/concordia/components/game_master/open_ended_questionnaire.py +++ b/concordia/components/game_master/open_ended_questionnaire.py @@ -237,13 +237,17 @@ def _process_answer( questionnaire.questionnaire_type == 'free' or questionnaire.questionnaire_type == 'open-ended' ): - # Embedd answer and choices, computer their cosine similarity - answer_embedding = self._embedder(answer_text) # pyrefly: ignore[not-callable] - choice_similarities = [] - for choice in current_question.choices: # pyrefly: ignore[not-iterable] - choice_embedding = self._embedder(choice) # pyrefly: ignore[not-callable] - similarity = np.dot(answer_embedding, choice_embedding) - choice_similarities.append({'choice': choice, 'similarity': similarity}) + answer_embedding = None + value = answer_text + if self._embedder: + # Embed answer and choices, compute their cosine similarity. + answer_embedding = self._embedder(answer_text) + choice_similarities = [] + for choice in current_question.choices: # pyrefly: ignore[not-iterable] + choice_embedding = self._embedder(choice) + similarity = np.dot(answer_embedding, choice_embedding) + choice_similarities.append({'choice': choice, 'similarity': similarity}) + value = choice_similarities self._answers[self._event_counter][player_name][questionnaire_name][ question_id @@ -251,7 +255,7 @@ def _process_answer( 'statement': current_question.statement, 'text': answer_text, 'dimension': dimension, - 'value': choice_similarities, + 'value': value, 'embedding': answer_embedding, } elif ( diff --git a/concordia/components/game_master/open_ended_questionnaire_test.py b/concordia/components/game_master/open_ended_questionnaire_test.py new file mode 100644 index 000000000..7cd945b0d --- /dev/null +++ b/concordia/components/game_master/open_ended_questionnaire_test.py @@ -0,0 +1,96 @@ +# Copyright 2026 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the OpenEndedQuestionnaire component.""" + +from absl.testing import absltest +from concordia.components.game_master import open_ended_questionnaire +from concordia.contrib.data.questionnaires import base_questionnaire +import numpy as np + + +class _FreeQuestionnaire(base_questionnaire.QuestionnaireBase): + """Minimal free-text questionnaire for tests.""" + + def aggregate_results(self, player_answers): + return {} + + def plot_results(self, results_df, label_column=None, kwargs=None): + pass + + def get_dimension_ranges(self): + return {} + + +class OpenEndedQuestionnaireTest(absltest.TestCase): + """Tests for the OpenEndedQuestionnaire component.""" + + def _make_component(self, questionnaire_type, embedder): + questionnaire = _FreeQuestionnaire( + name='mood', + description='Mood survey', + questionnaire_type=questionnaire_type, + observation_preprompt='Answer the question.', + questions=[ + base_questionnaire.Question( + statement='How do you feel?', + dimension='mood', + choices=['good', 'bad'], + ) + ], + ) + return open_ended_questionnaire.OpenEndedQuestionnaire( + questionnaires=[questionnaire], + player_names=['Alice'], + sequence_of_events=['event'], + embedder=embedder, + ) + + def test_free_answer_without_embedder_does_not_crash(self): + # Answering a free-text question without an embedder used to raise + # `TypeError: 'NoneType' object is not callable`. + component = self._make_component('free', embedder=None) + component.pre_observe('[putative_event] Alice: mood_0: happy') + answer = component.get_answers()[0]['Alice']['mood']['mood_0'] + self.assertEqual(answer['text'], 'happy') + self.assertEqual(answer['value'], 'happy') + self.assertIsNone(answer['embedding']) + + def test_open_ended_answer_without_embedder_does_not_crash(self): + component = self._make_component('open-ended', embedder=None) + component.pre_observe('[putative_event] Alice: mood_0: happy') + answer = component.get_answers()[0]['Alice']['mood']['mood_0'] + self.assertEqual(answer['value'], 'happy') + + def test_free_answer_with_embedder_computes_similarities(self): + def embedder(text): + # Deterministic one-hot embedding keyed on the first letter. + index = ord(text[0].lower()) - ord('a') + vector = np.zeros(26) + vector[index] = 1.0 + return vector + + component = self._make_component('open-ended', embedder=embedder) + component.pre_observe('[putative_event] Alice: mood_0: great') + answer = component.get_answers()[0]['Alice']['mood']['mood_0'] + self.assertEqual(answer['text'], 'great') + # 'great' and 'good' both start with 'g' so their similarity is 1.0 while + # 'bad' starts with 'b' so its similarity is 0.0. + self.assertEqual(answer['value'][0], {'choice': 'good', 'similarity': 1.0}) + self.assertEqual(answer['value'][1], {'choice': 'bad', 'similarity': 0.0}) + self.assertIsNotNone(answer['embedding']) + + +if __name__ == '__main__': + absltest.main() diff --git a/concordia/contrib/language_models/__init__.py b/concordia/contrib/language_models/__init__.py index 520676be7..ce2d1297a 100644 --- a/concordia/contrib/language_models/__init__.py +++ b/concordia/contrib/language_models/__init__.py @@ -24,7 +24,8 @@ _REGISTRY = types.MappingProxyType({ 'amazon_bedrock': 'amazon.amazon_bedrock_model.AmazonBedrockLanguageModel', 'gemini': 'google.gemini_model.GeminiModel', - 'gemini_vision': 'google.gemini_model_vision.GeminiModelVision', + 'gemini_vision': 'google.gemini_model_multimodal.GeminiModelVision', + 'google_aistudio': 'google.gemini_model.GeminiModel', 'google_cloud_custom_model': 'google.google_cloud_custom_model.VertexAI', 'groq': 'groq.groq_model.GroqModel', 'huggingface': 'huggingface.huggingface_model.HuggingFaceLanguageModel', diff --git a/concordia/contrib/language_models/language_models_test.py b/concordia/contrib/language_models/language_models_test.py new file mode 100644 index 000000000..566ba1b58 --- /dev/null +++ b/concordia/contrib/language_models/language_models_test.py @@ -0,0 +1,45 @@ +# Copyright 2026 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the language model setup registry.""" + +import importlib.util + +from absl.testing import absltest +from concordia.contrib import language_models + + +class LanguageModelsRegistryTest(absltest.TestCase): + """Tests that every registry entry resolves to an importable module.""" + + def test_all_registry_entries_resolve_to_modules(self): + for api_type, model_path in language_models._REGISTRY.items(): + module_path, _ = model_path.rsplit('.', 1) + full_module = f'concordia.contrib.language_models.{module_path}' + self.assertIsNotNone( + importlib.util.find_spec(full_module), + msg=( + f'api_type {api_type!r} references module {full_module!r} which' + ' does not exist.' + ), + ) + + def test_google_aistudio_is_a_registered_api_type(self): + # `google_aistudio` is the default api_type used by the example run + # scripts and the persona generator CLI. + self.assertIn('google_aistudio', language_models._REGISTRY) + + +if __name__ == '__main__': + absltest.main()