Skip to content

Commit 941c51f

Browse files
HelgeSverreHelgeSverre
committed
Added: StructuredOutputParser which takes "plain php class" converts it into a json schema, and marshals the JSON response back into an instance of that class.
Moved OutputParser tests into its own folder, added new method on LLM interace that takes a prompttemplate instead of a string.
1 parent c0043f0 commit 941c51f

File tree

12 files changed

+369
-65
lines changed

12 files changed

+369
-65
lines changed

‎src/Contracts/LLM.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,13 @@
22

33
namespace Mindwave\Mindwave\Contracts;
44

5+
use Mindwave\Mindwave\Prompts\PromptTemplate;
6+
57
interface LLM
68
{
7-
// TODO(11 May 2023) ~ Helge: make an interface that makes sense
9+
// TODO(29 May 2023) ~ Helge: These methods names are vague, rename them to something better.
810

911
public function predict(string $prompt): ?string;
12+
13+
public function run(PromptTemplate $promptTemplate): mixed;
1014
}

‎src/LLM/Drivers/Fake.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,17 @@
33
namespace Mindwave\Mindwave\LLM\Drivers;
44

55
use Mindwave\Mindwave\Contracts\LLM;
6+
use Mindwave\Mindwave\Prompts\PromptTemplate;
67

78
class Fake implements LLM
89
{
910
public function predict(string $prompt): ?string
1011
{
1112
return $prompt;
1213
}
14+
15+
public function run(PromptTemplate $promptTemplate): mixed
16+
{
17+
return 'implement this';
18+
}
1319
}

‎src/LLM/Drivers/OpenAIChat.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace Mindwave\Mindwave\LLM\Drivers;
44

55
use Mindwave\Mindwave\Contracts\LLM;
6+
use Mindwave\Mindwave\Prompts\PromptTemplate;
67
use OpenAI\Client;
78
use OpenAI\Responses\Chat\CreateResponseMessage;
89

@@ -48,4 +49,13 @@ public function predict(string $prompt): ?string
4849

4950
return $message->content;
5051
}
52+
53+
public function run(PromptTemplate $promptTemplate, array $inputs = []): mixed
54+
{
55+
$formatted = $promptTemplate->format($inputs);
56+
57+
$response = $this->predict($formatted);
58+
59+
return $promptTemplate->parse($response);
60+
}
5161
}

‎src/LLM/Drivers/OpenAICompletion.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace Mindwave\Mindwave\LLM\Drivers;
44

55
use Mindwave\Mindwave\Contracts\LLM;
6+
use Mindwave\Mindwave\Prompts\PromptTemplate;
67
use OpenAI\Client;
78

89
class OpenAICompletion implements LLM
@@ -38,4 +39,13 @@ public function predict(string $prompt): ?string
3839

3940
return $response->choices[0]?->text;
4041
}
42+
43+
public function run(PromptTemplate $promptTemplate, array $inputs = []): mixed
44+
{
45+
$formatted = $promptTemplate->format($inputs);
46+
47+
$response = $this->predict($formatted);
48+
49+
return $promptTemplate->parse($response);
50+
}
4151
}

‎src/Prompts/OutputParsers/StructuredOutputParser.php‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<?php
2+
3+
namespace Mindwave\Mindwave\Prompts\OutputParsers;
4+
5+
use Illuminate\Support\Collection;
6+
use Mindwave\Mindwave\Contracts\OutputParser;
7+
use ReflectionClass;
8+
9+
class StructuredOutputParser implements OutputParser
10+
{
11+
protected $schema;
12+
13+
public function __construct($schema = null)
14+
{
15+
$this->schema = $schema;
16+
}
17+
18+
public function fromClass($schema): self
19+
{
20+
$this->schema = $schema;
21+
22+
return $this;
23+
}
24+
25+
public function getSchemaStructure(): array
26+
{
27+
$reflectionClass = new ReflectionClass($this->schema);
28+
$properties = [];
29+
$required = [];
30+
31+
foreach ($reflectionClass->getProperties() as $property) {
32+
$propertyName = $property->getName();
33+
$propertyType = $property->getType()->getName();
34+
35+
if ($property->getType()->allowsNull() === false) {
36+
$required[] = $propertyName;
37+
}
38+
39+
$properties[$propertyName] = [
40+
'type' => match ($propertyType) {
41+
'string', 'int', 'float', 'bool' => $propertyType,
42+
'array', Collection::class => 'array',
43+
default => 'object',
44+
},
45+
];
46+
}
47+
48+
return [
49+
'properties' => $properties,
50+
'required' => $required,
51+
];
52+
}
53+
54+
public function getFormatInstructions(): string
55+
{
56+
$schema = json_encode($this->getSchemaStructure());
57+
58+
return trim('
59+
RESPONSE FORMAT INSTRUCTIONS
60+
----------------------------
61+
The output should be formatted as a JSON instance that conforms to the JSON schema below.
62+
63+
As an example, for the schema {{"properties": {{"foo": {{"title": "Foo", "description": "a list of strings", "type": "array", "items": {{"type": "string"}}}}}}, "required": ["foo"]}}}}
64+
the object {{"foo": ["bar", "baz"]}} is a well-formatted instance of the schema. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not well-formatted.
65+
66+
Here is the output schema:
67+
```json
68+
'.$schema.'
69+
```
70+
Remember to respond with a JSON blob, and NOTHING else.');
71+
}
72+
73+
public function parse(string $text): mixed
74+
{
75+
$reflectionClass = new ReflectionClass($this->schema);
76+
$data = json_decode($text, true);
77+
78+
if (! $data) {
79+
// TODO(29 May 2023) ~ Helge: Throw custom exception
80+
return null;
81+
}
82+
83+
$instance = new $this->schema();
84+
85+
foreach ($data as $key => $value) {
86+
87+
$type = $reflectionClass->getProperty($key)->getType();
88+
89+
// TODO(29 May 2023) ~ Helge: There are probably libraries that do this in a more clever way, but this is fine for now.
90+
$instance->{$key} = match ($type->getName()) {
91+
'bool' => boolval($value),
92+
'int' => intval($value),
93+
'float' => floatval($value),
94+
Collection::class => collect($value),
95+
default => $value,
96+
};
97+
}
98+
99+
return $instance;
100+
}
101+
}

‎tests/LLMTest.php‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?php
2+
3+
use Illuminate\Support\Collection;
4+
use Illuminate\Support\Facades\Config;
5+
use Mindwave\Mindwave\Facades\Mindwave;
6+
use Mindwave\Mindwave\Prompts\OutputParsers\StructuredOutputParser;
7+
use Mindwave\Mindwave\Prompts\PromptTemplate;
8+
9+
it('can use a structured output parser', function () {
10+
Config::set('mindwave-vectorstore.default', 'array');
11+
Config::set('mindwave-embeddings.embeddings.openai.api_key', env('MINDWAVE_OPENAI_API_KEY'));
12+
Config::set('mindwave-llm.llms.openai_chat.api_key', env('MINDWAVE_OPENAI_API_KEY'));
13+
14+
class Person
15+
{
16+
public string $name;
17+
18+
public ?int $age;
19+
20+
public ?bool $hasBusiness;
21+
22+
public ?array $interests;
23+
24+
public ?Collection $tags;
25+
}
26+
27+
$model = Mindwave::llm();
28+
$parser = new StructuredOutputParser(Person::class);
29+
30+
$result = $model->run(PromptTemplate::create(
31+
'Generate random details about a fictional person', $parser
32+
));
33+
34+
expect($result)->toBeInstanceOf(Person::class);
35+
36+
dump($result);
37+
});
38+
39+
it('We can parse a small recipe into an object', function () {
40+
Config::set('mindwave-vectorstore.default', 'array');
41+
Config::set('mindwave-embeddings.embeddings.openai.api_key', env('MINDWAVE_OPENAI_API_KEY'));
42+
Config::set('mindwave-llm.llms.openai_chat.api_key', env('MINDWAVE_OPENAI_API_KEY'));
43+
Config::set('mindwave-llm.llms.openai_chat.max_tokens', 2500);
44+
Config::set('mindwave-llm.llms.openai_chat.temperature', 0.2);
45+
46+
class Recipe
47+
{
48+
public string $dishName;
49+
50+
public ?string $description;
51+
52+
public ?int $portions;
53+
54+
public ?array $steps;
55+
}
56+
57+
// Source: https://sugarspunrun.com/the-best-pizza-dough-recipe/
58+
$rawRecipeText = file_get_contents(__DIR__.'/data/samples/pizza-recipe.txt');
59+
60+
$template = PromptTemplate::create(
61+
template: 'Extract details from this recipe: {recipe}',
62+
outputParser: new StructuredOutputParser(Recipe::class)
63+
);
64+
65+
$result = Mindwave::llm()->run($template, [
66+
'recipe' => $rawRecipeText,
67+
]);
68+
69+
expect($result)->toBeInstanceOf(Recipe::class);
70+
71+
dump($result);
72+
});

‎tests/Prompts/OutputParser/CommaSeparatedListOutputParserTest.php‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?php
2+
3+
use Mindwave\Mindwave\Prompts\OutputParsers\CommaSeparatedListOutputParser;
4+
5+
it('can parse comma separated output', function () {
6+
7+
$parser = new CommaSeparatedListOutputParser();
8+
9+
expect($parser->parse('monsters, bananas, flies, sausages'))->toEqual([
10+
'monsters',
11+
'bananas',
12+
'flies',
13+
'sausages',
14+
]);
15+
});

‎tests/Prompts/OutputParser/JsonListOutputParserTest.php‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
3+
use Mindwave\Mindwave\Prompts\OutputParsers\JsonListOutputParser;
4+
use Mindwave\Mindwave\Prompts\PromptTemplate;
5+
6+
it('json list output parser generates a list from constructor', function () {
7+
$outputParser = new JsonListOutputParser();
8+
$prompt = PromptTemplate::create(
9+
template: 'Generate 10 keywords for {topic}',
10+
outputParser: $outputParser
11+
)->format([
12+
'topic' => 'Mindwave',
13+
]);
14+
15+
expect($prompt)->toContain('Generate 10 keywords for Mindwave');
16+
expect($prompt)->toContain($outputParser->getFormatInstructions());
17+
});
18+
19+
it('json list output parser generates a list from method', function () {
20+
$outputParser = new JsonListOutputParser();
21+
22+
$prompt = PromptTemplate::create(
23+
template: 'Generate 10 keywords for {topic}',
24+
)->withOutputParser($outputParser)->format([
25+
'topic' => 'Laravel',
26+
]);
27+
28+
expect($prompt)->toContain('Generate 10 keywords for Laravel');
29+
expect($prompt)->toContain($outputParser->getFormatInstructions());
30+
});
31+
32+
it('can parse json array as array', function () {
33+
34+
$parser = new JsonListOutputParser();
35+
36+
expect($parser->parse('```json{"data": ["monsters", "bananas", "flies", "sausages"]}```'))->toEqual([
37+
'monsters',
38+
'bananas',
39+
'flies',
40+
'sausages',
41+
]);
42+
});

‎tests/Prompts/OutputParser/JsonOutputParserTest.php‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?php
2+
3+
use Mindwave\Mindwave\Prompts\OutputParsers\JsonOutputParser;
4+
use Mindwave\Mindwave\Prompts\PromptTemplate;
5+
6+
it('can parse a response', function () {
7+
$prompt = PromptTemplate::create('Test prompt', new JsonOutputParser())
8+
->parse('```json { "hello": "world", "nice":["mindwave", "package"] } ```');
9+
10+
expect($prompt)
11+
->toBeArray()
12+
->and($prompt)
13+
->toHaveKey('hello', 'world')
14+
->toHaveKey('nice', ['mindwave', 'package']);
15+
});

‎tests/Prompts/OutputParser/StructuredOutputParserTest.php‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<?php
2+
3+
use Illuminate\Support\Collection;
4+
use Mindwave\Mindwave\Prompts\OutputParsers\StructuredOutputParser;
5+
6+
class Person
7+
{
8+
public string $name;
9+
10+
public ?int $age;
11+
12+
public ?bool $hasBusiness;
13+
14+
public ?array $interests;
15+
16+
public ?Collection $tags;
17+
}
18+
19+
it('can convert a class into a schema for StructuredOutputParser', function () {
20+
$parser = new StructuredOutputParser(Person::class);
21+
22+
expect($parser->getSchemaStructure())
23+
->toBe([
24+
'properties' => [
25+
'name' => ['type' => 'string'],
26+
'age' => ['type' => 'int'],
27+
'hasBusiness' => ['type' => 'bool'],
28+
'interests' => ['type' => 'array'],
29+
'tags' => ['type' => 'array'],
30+
],
31+
'required' => ['name'],
32+
]);
33+
});
34+
35+
it('can parse response into class instance', function () {
36+
$parser = new StructuredOutputParser(Person::class);
37+
38+
/** @var Person $person */
39+
$person = $parser->parse('{"name": "Lila Jones", "age": 28, "hasBusiness": true, "interests": ["hiking", "reading", "painting"], "tags": ["adventurous", "creative", "entrepreneur"]}');
40+
41+
expect($person)->toBeInstanceOf(Person::class);
42+
expect($person->name)->toBe('Lila Jones');
43+
expect($person->age)->toBe(28);
44+
expect($person->hasBusiness)->toBe(true);
45+
expect($person->interests)->toBe(['hiking', 'reading', 'painting']);
46+
expect($person->tags)->toEqual(collect(['adventurous', 'creative', 'entrepreneur']));
47+
});
48+
49+
it('can returns null if parsing data fails.', function () {
50+
$parser = new StructuredOutputParser(Person::class);
51+
52+
$person = $parser->parse('broken and invalid data');
53+
54+
expect($person)->toBeNull(Person::class);
55+
});

0 commit comments

Comments
 (0)