Python: Add ability to specify encoding when adding a plugin. Add tests. (#12797)

### Motivation and Context

Right now when adding a plugin, we don't allow one to specify the type
of encoding used.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

Allow configuration for the type of encoding used.
- Adds unit tests
- Closes #12440

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

Author: moonbox3

python/semantic_kernel/functions/kernel_function_extension.py

@@ -68,6 +68,7 @@ def add_plugin(
         parent_directory: str | None = None,
         description: str | None = None,
         class_init_arguments: dict[str, dict[str, Any]] | None = None,
+        encoding: str = "utf-8",
     ) -> "KernelPlugin":
         """Adds a plugin to the kernel's collection of plugins.
 
@@ -88,6 +89,7 @@ def add_plugin(
             parent_directory: The parent directory path where the plugin directory resides
             description: The description of the plugin, used if the plugin is not a KernelPlugin.
             class_init_arguments: The class initialization arguments
+            encoding: The encoding to use when reading text files. Defaults to "utf-8".
 
         Returns:
             KernelPlugin: The plugin that was added.
@@ -116,6 +118,7 @@ def add_plugin(
                 parent_directory=parent_directory,
                 description=description,
                 class_init_arguments=class_init_arguments,
+                encoding=encoding,
             )
             return self.plugins[plugin_name]
         raise ValueError("plugin or parent_directory must be provided.")

python/semantic_kernel/functions/kernel_function_from_prompt.py

@@ -355,13 +355,20 @@ def from_yaml(cls, yaml_str: str, plugin_name: str | None = None) -> "KernelFunc
         )
 
     @classmethod
-    def from_directory(cls, path: str, plugin_name: str | None = None) -> "KernelFunctionFromPrompt":
+    def from_directory(
+        cls, path: str, plugin_name: str | None = None, encoding: str = "utf-8"
+    ) -> "KernelFunctionFromPrompt":
         """Creates a new instance of the KernelFunctionFromPrompt class from a directory.
 
         The directory needs to contain:
         - A prompt file named `skprompt.txt`
         - A config file named `config.json`
 
+        Args:
+            path: The path to the directory containing the prompt and config files.
+            plugin_name: The name of the plugin.
+            encoding: The encoding to use when reading the files. Defaults to "utf-8".
+
         Returns:
             KernelFunctionFromPrompt: The kernel function from prompt
         """
@@ -387,11 +394,11 @@ def from_directory(cls, path: str, plugin_name: str | None = None) -> "KernelFun
 
         function_name = os.path.basename(path)
 
-        with open(config_path) as config_file:
+        with open(config_path, encoding=encoding) as config_file:
             prompt_template_config = PromptTemplateConfig.from_json(config_file.read())
         prompt_template_config.name = function_name
 
-        with open(prompt_path) as prompt_file:
+        with open(prompt_path, encoding=encoding) as prompt_file:
             prompt_template_config.template = prompt_file.read()
 
         prompt_template = TEMPLATE_FORMAT_MAP[prompt_template_config.template_format](  # type: ignore

python/semantic_kernel/functions/kernel_plugin.py

@@ -256,6 +256,7 @@ def from_directory(
         parent_directory: str,
         description: str | None = None,
         class_init_arguments: dict[str, dict[str, Any]] | None = None,
+        encoding: str = "utf-8",
     ) -> _T:
         """Create a plugin from a specified directory.
 
@@ -294,6 +295,7 @@ def from_directory(
             parent_directory (str): The parent directory path where the plugin directory resides
             description (str | None): The description of the plugin
             class_init_arguments (dict[str, dict[str, Any]] | None): The class initialization arguments
+            encoding (str): The encoding to use when reading text files. Defaults to "utf-8".
 
         Returns:
             KernelPlugin: The created plugin of type KernelPlugin.
@@ -313,11 +315,11 @@ def from_directory(
                 if os.path.basename(object).startswith("__"):
                     continue
                 try:
-                    functions.append(KernelFunctionFromPrompt.from_directory(path=object))
+                    functions.append(KernelFunctionFromPrompt.from_directory(path=object, encoding=encoding))
                 except FunctionInitializationError:
                     logger.warning(f"Failed to create function from directory: {object}")
             elif object.endswith(".yaml") or object.endswith(".yml"):
-                with open(object) as file:
+                with open(object, encoding=encoding) as file:
                     try:
                         functions.append(KernelFunctionFromPrompt.from_yaml(file.read()))
                     except FunctionInitializationError:

python/tests/unit/functions/test_kernel_function_from_prompt.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import os
+import tempfile
 from unittest.mock import patch
 
 import pytest
@@ -418,3 +419,210 @@ def test_function_model_dump_json():
     model_dump_json = function.model_dump_json()
     assert isinstance(model_dump_json, str)
     assert "test" in model_dump_json
+
+
+def test_from_directory_utf8_encoding_default():
+    """Test loading plugin with default UTF-8 encoding."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        # UTF-8 content with international characters
+        prompt_content = """Hello! I can help with questions in multiple languages:
+        English: Hello world!
+        Spanish: ¡Hola mundo!
+        Chinese: 你好世界!
+        Japanese: こんにちは世界!
+        
+        Question: {{$input}}
+        """
+
+        config_content = """{
+    "schema": 1,
+    "description": "A multilingual assistant function",
+    "input_variables": [
+        {
+            "name": "input",
+            "description": "User's question",
+            "required": true
+        }
+    ]
+}"""
+
+        # Write files with UTF-8 encoding
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Test default behavior (should use UTF-8)
+        function = KernelFunctionFromPrompt.from_directory(temp_dir)
+        assert function.name == os.path.basename(temp_dir)
+        assert function.description == "A multilingual assistant function"
+        assert "你好世界" in function.prompt_template.prompt_template_config.template
+        assert "こんにちは世界" in function.prompt_template.prompt_template_config.template
+
+
+def test_from_directory_explicit_utf8_encoding():
+    """Test loading plugin with explicit UTF-8 encoding."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        prompt_content = "Hello with UTF-8 characters: ñáéíóú {{$input}}"
+        config_content = '{"schema": 1, "description": "Test with UTF-8 characters"}'
+
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Test explicit UTF-8 encoding
+        function = KernelFunctionFromPrompt.from_directory(temp_dir, encoding="utf-8")
+        assert function.description == "Test with UTF-8 characters"
+        assert "ñáéíóú" in function.prompt_template.prompt_template_config.template
+
+
+def test_from_directory_latin1_encoding():
+    """Test loading plugin with Latin-1 encoding."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        # Content with Latin-1 characters (Western European)
+        prompt_content = """Assistant for Western European languages:
+        French: café, naïve, résumé
+        German: Müller, Größe, weiß
+        Spanish: niño, señora, años
+        
+        Question: {{$input}}
+        """
+
+        config_content = """{
+    "schema": 1,
+    "description": "Western European language assistant",
+    "input_variables": [
+        {
+            "name": "input",
+            "description": "User's question",
+            "required": true
+        }
+    ]
+}"""
+
+        # Write files with Latin-1 encoding
+        with open(prompt_path, "w", encoding="latin-1") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="latin-1") as f:
+            f.write(config_content)
+
+        # Load with Latin-1 encoding
+        function = KernelFunctionFromPrompt.from_directory(temp_dir, encoding="latin-1")
+        assert function.description == "Western European language assistant"
+        assert "café" in function.prompt_template.prompt_template_config.template
+        assert "Müller" in function.prompt_template.prompt_template_config.template
+        assert "niño" in function.prompt_template.prompt_template_config.template
+
+
+def test_from_directory_cp1252_encoding():
+    """Test loading plugin with Windows-1252 encoding."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        # Content with Windows-1252 specific characters
+        prompt_content = """Windows text processing assistant:
+        Smart quotes: "Hello" and 'world'
+        Em dash: Yes—absolutely!
+        Ellipsis: Wait…
+        
+        Question: {{$input}}
+        """
+
+        config_content = """{
+    "schema": 1,
+    "description": "Windows text processing assistant",
+    "input_variables": [
+        {
+            "name": "input",
+            "description": "User's question about text processing",
+            "required": true
+        }
+    ]
+}"""
+
+        # Write files with Windows-1252 encoding
+        with open(prompt_path, "w", encoding="cp1252") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="cp1252") as f:
+            f.write(config_content)
+
+        # Load with Windows-1252 encoding
+        function = KernelFunctionFromPrompt.from_directory(temp_dir, encoding="cp1252")
+        assert function.description == "Windows text processing assistant"
+        assert '"Hello"' in function.prompt_template.prompt_template_config.template
+        assert "Yes—absolutely" in function.prompt_template.prompt_template_config.template
+        assert "Wait…" in function.prompt_template.prompt_template_config.template
+
+
+def test_from_directory_with_plugin_name_and_encoding():
+    """Test loading plugin with both plugin name and encoding specified."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        prompt_content = "Simple assistant: {{$input}}"
+        config_content = '{"schema": 1, "description": "Simple assistant"}'
+
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Load with both plugin name and encoding specified
+        function = KernelFunctionFromPrompt.from_directory(
+            path=temp_dir, plugin_name="MyCustomPlugin", encoding="utf-8"
+        )
+        assert function.metadata.plugin_name == "MyCustomPlugin"
+        assert function.description == "Simple assistant"
+        assert function.prompt_template.prompt_template_config.template == "Simple assistant: {{$input}}"
+
+
+def test_from_directory_encoding_error_handling():
+    """Test that incorrect encoding raises appropriate error."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        # Write UTF-8 content
+        prompt_content = "Hello with UTF-8: 你好世界 {{$input}}"
+        config_content = '{"schema": 1, "description": "UTF-8 content"}'
+
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Try to read with ASCII encoding - should fail
+        with pytest.raises(UnicodeDecodeError):
+            KernelFunctionFromPrompt.from_directory(temp_dir, encoding="ascii")
+
+
+def test_from_directory_backward_compatibility():
+    """Test that existing code without encoding parameter still works."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        prompt_path = os.path.join(temp_dir, "skprompt.txt")
+        config_path = os.path.join(temp_dir, "config.json")
+
+        prompt_content = "Basic ASCII content: {{$input}}"
+        config_content = '{"schema": 1, "description": "Basic function"}'
+
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Test that old calling style still works
+        function = KernelFunctionFromPrompt.from_directory(temp_dir)
+        assert function.description == "Basic function"
+        assert function.prompt_template.prompt_template_config.template == "Basic ASCII content: {{$input}}"

python/tests/unit/kernel/test_kernel.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import os
+import tempfile
 from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
@@ -597,6 +598,60 @@ def test_add_plugin_from_directory(kernel: Kernel):
     assert func_handlebars is not None
 
 
+def test_add_plugin_from_directory_with_encoding(kernel: Kernel):
+    """Test kernel.add_plugin with custom encoding parameter."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Create a plugin directory with UTF-8 content
+        plugin_dir = os.path.join(temp_dir, "test_encoding_plugin")
+        os.makedirs(plugin_dir)
+
+        function_dir = os.path.join(plugin_dir, "test_function")
+        os.makedirs(function_dir)
+
+        prompt_path = os.path.join(function_dir, "skprompt.txt")
+        config_path = os.path.join(function_dir, "config.json")
+
+        # UTF-8 content with international characters
+        # Hello World Test
+        prompt_content = """Multi-language assistant:
+        Chinese: 你好世界!
+        Japanese: こんにちは世界!
+        Question: {{$input}}
+        """
+
+        config_content = """{
+    "schema": 1,
+    "description": "Test encoding function",
+    "input_variables": [
+        {
+            "name": "input",
+            "description": "User's question",
+            "required": true
+        }
+    ]
+}"""
+
+        # Write files with UTF-8 encoding
+        with open(prompt_path, "w", encoding="utf-8") as f:
+            f.write(prompt_content)
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(config_content)
+
+        # Test with explicit encoding
+        plugin = kernel.add_plugin(parent_directory=temp_dir, plugin_name="test_encoding_plugin", encoding="utf-8")
+
+        assert plugin is not None
+        assert plugin.name == "test_encoding_plugin"
+        assert "test_function" in plugin.functions
+
+        function = plugin.functions["test_function"]
+        template = function.prompt_template.prompt_template_config.template
+        # Assert "Hello World"
+        assert "你好世界" in template
+        assert "こんにちは世界" in template
+        assert function.description == "Test encoding function"
+
+
 def test_plugin_no_plugin(kernel: Kernel):
     with pytest.raises(ValueError):
         kernel.add_plugin(plugin_name="test")