Skip to content

Commit 0fde03e

Browse files
rhenniganrhennigan
authored
Merge pull request #1293 from WolframResearch/feature/multimodal-wl-tool-inputs
Support multimodal inputs in `WolframLanguageToolEvaluate`
2 parents 12da188 + 72f559f commit 0fde03e

File tree

2 files changed

+207
-9
lines changed

2 files changed

+207
-9
lines changed

‎Source/Chatbook/Sandbox.wl‎

Lines changed: 61 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ $nlpData = <| |>;
8888

8989
$$simpleTemplateBoxName = "Entity"|"EntityClass"|"EntityProperty"|"DateObject"|"Quantity";
9090

91+
(* ::**************************************************************************************************************:: *)
92+
(* ::Section::Closed:: *)
93+
(*Messages*)
94+
Chatbook::InvalidEvaluatorCode = "Invalid evaluator code: `1`.";
95+
Chatbook::InvalidEvaluatorProperty = "Invalid evaluator property: `1`.";
96+
9197
(* ::**************************************************************************************************************:: *)
9298
(* ::Section::Closed:: *)
9399
(*WolframLanguageToolEvaluate*)
@@ -103,8 +109,8 @@ WolframLanguageToolEvaluate // Options = {
103109
TimeConstraint -> 60
104110
};
105111

106-
WolframLanguageToolEvaluate[ code_? validCodeQ, opts: OptionsPattern[ ] ] :=
107-
catchMine @ WolframLanguageToolEvaluate[ code, "String", opts ];
112+
WolframLanguageToolEvaluate[ code_, opts: OptionsPattern[ ] ] :=
113+
catchMine @ WolframLanguageToolEvaluate[ Unevaluated @ code, "String", opts ];
108114

109115
WolframLanguageToolEvaluate[ code_? validCodeQ, property_? validPropertyQ, opts: OptionsPattern[ ] ] :=
110116
catchMine @ wolframLanguageToolEvaluate[
@@ -113,14 +119,21 @@ WolframLanguageToolEvaluate[ code_? validCodeQ, property_? validPropertyQ, opts:
113119
optionsAssociation[ WolframLanguageToolEvaluate, opts ]
114120
];
115121

122+
WolframLanguageToolEvaluate[ code_ /; ! validCodeQ @ code, property_, opts: OptionsPattern[ ] ] :=
123+
catchMine @ throwFailure[ "InvalidEvaluatorCode", HoldForm @ code ];
124+
125+
WolframLanguageToolEvaluate[ code_, property_ /; ! validPropertyQ @ property, opts: OptionsPattern[ ] ] :=
126+
catchMine @ throwFailure[ "InvalidEvaluatorProperty", HoldForm @ property ];
127+
116128
WolframLanguageToolEvaluate // endExportedDefinition;
117129

118130
(* ::**************************************************************************************************************:: *)
119131
(* ::Subsection::Closed:: *)
120132
(*validCodeQ*)
121133
validCodeQ // beginDefinition;
122-
validCodeQ[ KeyValuePattern[ "code" -> code_ ] ] := validCodeQ @ code;
123-
validCodeQ[ code_String ] := StringQ @ code;
134+
validCodeQ // Attributes = { HoldAllComplete };
135+
validCodeQ[ code_String ] := StringQ @ code && ! StringMatchQ[ code, WhitespaceCharacter... ];
136+
validCodeQ[ { __ } ] := True;
124137
validCodeQ[ HoldComplete[ __ ] ] := True;
125138
validCodeQ[ ___ ] := False;
126139
validCodeQ // endDefinition;
@@ -132,7 +145,7 @@ $$propertyName = "Packets"|"Result"|"SessionMX"|"String";
132145

133146
validPropertyQ // beginDefinition;
134147
validPropertyQ[ $$propertyName ] := True;
135-
validPropertyQ[ All ] := True;
148+
validPropertyQ[ All|Automatic ] := True;
136149
validPropertyQ[ { $$propertyName... } ] := True;
137150
validPropertyQ[ ___ ] := False;
138151
validPropertyQ // endDefinition;
@@ -141,6 +154,7 @@ validPropertyQ // endDefinition;
141154
(* ::Subsection::Closed:: *)
142155
(*wolframLanguageToolEvaluate*)
143156
wolframLanguageToolEvaluate // beginDefinition;
157+
wolframLanguageToolEvaluate // Attributes = { HoldFirst };
144158

145159
wolframLanguageToolEvaluate[ code_, property_, opts_Association ] := Enclose[
146160
Block[
@@ -154,7 +168,7 @@ wolframLanguageToolEvaluate[ code_, property_, opts_Association ] := Enclose[
154168
$sandboxEvaluationTimeout = getOption[ "TimeConstraint" , opts ],
155169
$Line = getOption[ "Line" , opts ]
156170
},
157-
getProperty[ sandboxEvaluate @ code, property ]
171+
getProperty[ sandboxEvaluate @ preprocessCodeForTool @ code, property ]
158172
],
159173
throwInternalFailure
160174
];
@@ -205,10 +219,46 @@ getOption // endDefinition;
205219
(*getProperty*)
206220
getProperty // beginDefinition;
207221
getProperty[ as_Association, property_String ] := Lookup[ as, property ];
208-
getProperty[ as_Association, All ] := as;
222+
getProperty[ as_Association, All|Automatic ] := as;
209223
getProperty[ as_Association, properties_List ] := AssociationMap[ getProperty[ as, # ] &, properties ];
210224
getProperty // endDefinition;
211225

226+
(* ::**************************************************************************************************************:: *)
227+
(* ::Subsubsection::Closed:: *)
228+
(*preprocessCodeForTool*)
229+
preprocessCodeForTool // beginDefinition;
230+
preprocessCodeForTool // Attributes = { HoldAllComplete };
231+
232+
preprocessCodeForTool[ code: _String | _HoldComplete ] :=
233+
code;
234+
235+
preprocessCodeForTool[ code_List ] := Enclose[
236+
StringJoin @ ConfirmMatch[ makeInlineExpressionString /@ Unevaluated @ code, { __String }, "Inlined" ],
237+
throwInternalFailure
238+
];
239+
240+
preprocessCodeForTool // endDefinition;
241+
242+
(* ::**************************************************************************************************************:: *)
243+
(* ::Subsubsection::Closed:: *)
244+
(*makeInlineExpressionString*)
245+
makeInlineExpressionString // beginDefinition;
246+
makeInlineExpressionString // Attributes = { HoldAllComplete };
247+
248+
makeInlineExpressionString[ fragment_String ] :=
249+
fragment;
250+
251+
makeInlineExpressionString[ expr_ ] := Enclose[
252+
Module[ { link, key },
253+
link = ConfirmBy[ MakeExpressionURI @ Unevaluated @ expr, StringQ, "Link" ];
254+
key = ConfirmBy[ expressionURIKey @ link, StringQ, "Key" ];
255+
"InlinedExpression[\"attachment://" <> key <> "\"]"
256+
],
257+
throwInternalFailure
258+
];
259+
260+
makeInlineExpressionString // endDefinition;
261+
212262
(* ::**************************************************************************************************************:: *)
213263
(* ::Section::Closed:: *)
214264
(*SandboxLinguisticAssistantData*)
@@ -941,7 +991,7 @@ toSandboxExpression[ s_String ] := $lastSandboxExpression =
941991
];
942992

943993
toSandboxExpression[ s_, expr_HoldComplete ] :=
944-
expandSandboxMacros @ expr;
994+
expr;
945995

946996
toSandboxExpression[ s_String, $Failed ] /; StringContainsQ[ s, "'" ] :=
947997
Module[ { new, held },
@@ -992,7 +1042,7 @@ toSandboxExpression // endDefinition;
9921042
toHeldExpression // beginDefinition;
9931043

9941044
toHeldExpression[ s_String ] :=
995-
Replace[
1045+
expandSandboxMacros @ Replace[
9961046
DeleteCases[ Quiet @ ToExpression[ s, InputForm, HoldComplete ], Null ],
9971047
{
9981048
HoldComplete[ xs__, CompoundExpression[ x_, Null ] ] :> Replace[
@@ -1069,6 +1119,8 @@ expandSandboxMacros[ expr_HoldComplete ] := Enclose[
10691119
throwInternalFailure
10701120
];
10711121

1122+
expandSandboxMacros[ $Failed ] := $Failed;
1123+
10721124
expandSandboxMacros // endDefinition;
10731125

10741126
(* ::**************************************************************************************************************:: *)

‎Tests/WolframLanguageToolEvaluate.wlt‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
(* ::**************************************************************************************************************:: *)
2+
(* ::Section::Closed:: *)
3+
(*Initialization*)
4+
VerificationTest[
5+
Needs[ "Wolfram`ChatbookTests`", FileNameJoin @ { DirectoryName[ $TestFileName ], "Common.wl" } ],
6+
Null,
7+
SameTest -> MatchQ,
8+
TestID -> "GetDefinitions@@Tests/WolframLanguageToolEvaluate.wlt:4,1-9,2"
9+
]
10+
11+
VerificationTest[
12+
Needs[ "Wolfram`Chatbook`" ],
13+
Null,
14+
SameTest -> MatchQ,
15+
TestID -> "LoadContext@@Tests/WolframLanguageToolEvaluate.wlt:11,1-16,2"
16+
]
17+
18+
VerificationTest[
19+
Context @ WolframLanguageToolEvaluate,
20+
"Wolfram`Chatbook`",
21+
SameTest -> MatchQ,
22+
TestID -> "WolframLanguageToolEvaluateContext@@Tests/WolframLanguageToolEvaluate.wlt:18,1-23,2"
23+
]
24+
25+
(* ::**************************************************************************************************************:: *)
26+
(* ::Section::Closed:: *)
27+
(*WolframLanguageToolEvaluate*)
28+
29+
(* ::**************************************************************************************************************:: *)
30+
(* ::Subsection::Closed:: *)
31+
(*Basic Evaluation*)
32+
VerificationTest[
33+
WolframLanguageToolEvaluate[ "1 + 1", Method -> "Session" ],
34+
_String? (StringMatchQ[ "Out["~~DigitCharacter..~~"]= 2"]),
35+
SameTest -> MatchQ,
36+
TestID -> "BasicEvaluation@@Tests/WolframLanguageToolEvaluate.wlt:32,1-37,2"
37+
]
38+
39+
VerificationTest[
40+
WolframLanguageToolEvaluate[ "1 + 1", "String", Method -> "Session" ],
41+
_String? (StringMatchQ[ "Out["~~DigitCharacter..~~"]= 2"]),
42+
SameTest -> MatchQ,
43+
TestID -> "StringProperty@@Tests/WolframLanguageToolEvaluate.wlt:39,1-44,2"
44+
]
45+
46+
VerificationTest[
47+
WolframLanguageToolEvaluate[ "1 + 1", "Result", Method -> "Session" ],
48+
HoldForm[ 2 ],
49+
SameTest -> MatchQ,
50+
TestID -> "ResultProperty@@Tests/WolframLanguageToolEvaluate.wlt:46,1-51,2"
51+
]
52+
53+
VerificationTest[
54+
WolframLanguageToolEvaluate[ "1 + 1", { "Result", "String" }, Method -> "Session" ],
55+
KeyValuePattern @ { "Result" -> HoldForm[ 2 ], "String" -> _String },
56+
SameTest -> MatchQ,
57+
TestID -> "MultipleProperties@@Tests/WolframLanguageToolEvaluate.wlt:53,1-58,2"
58+
]
59+
60+
VerificationTest[
61+
WolframLanguageToolEvaluate[ "1 + 1", All, Method -> "Session" ],
62+
KeyValuePattern @ { "Result" -> HoldForm[ 2 ], "String" -> _String },
63+
SameTest -> MatchQ,
64+
TestID -> "AllProperties@@Tests/WolframLanguageToolEvaluate.wlt:60,1-65,2"
65+
]
66+
67+
(* ::**************************************************************************************************************:: *)
68+
(* ::Subsection::Closed:: *)
69+
(*Natural Language Input*)
70+
VerificationTest[
71+
string = WolframLanguageToolEvaluate[ "\[FreeformPrompt][\"Boston, MA\"]", Method -> "Session" ],
72+
_String,
73+
SameTest -> MatchQ,
74+
TestID -> "NaturalLanguageInput-1@@Tests/WolframLanguageToolEvaluate.wlt:70,1-75,2"
75+
]
76+
77+
VerificationTest[
78+
StringContainsQ[
79+
string,
80+
"[INFO] Interpreted \"Boston, MA\" as: Entity[\"City\", {\"Boston\", \"Massachusetts\", \"UnitedStates\"}]"
81+
],
82+
True,
83+
SameTest -> MatchQ,
84+
TestID -> "NaturalLanguageInput-1-InfoMessage@@Tests/WolframLanguageToolEvaluate.wlt:77,1-85,2"
85+
]
86+
87+
VerificationTest[
88+
StringEndsQ[
89+
string,
90+
"Out[" ~~ DigitCharacter.. ~~ "]= Entity[\"City\", {\"Boston\", \"Massachusetts\", \"UnitedStates\"}]"
91+
],
92+
True,
93+
SameTest -> MatchQ,
94+
TestID -> "NaturalLanguageInput-1-Output@@Tests/WolframLanguageToolEvaluate.wlt:87,1-95,2"
95+
]
96+
97+
VerificationTest[
98+
string = WolframLanguageToolEvaluate[ "\[FreeformPrompt][\"Springfield\"]", Method -> "Session" ],
99+
_String,
100+
SameTest -> MatchQ,
101+
TestID -> "NaturalLanguageInput-2@@Tests/WolframLanguageToolEvaluate.wlt:97,1-102,2"
102+
]
103+
104+
VerificationTest[
105+
StringContainsQ[
106+
string,
107+
"[WARNING] Interpreted \"Springfield\" as " ~~ __ ~~ " with other possible interpretations:"
108+
],
109+
True,
110+
SameTest -> MatchQ,
111+
TestID -> "NaturalLanguageInput-2-WarningMessage@@Tests/WolframLanguageToolEvaluate.wlt:104,1-112,2"
112+
]
113+
114+
VerificationTest[
115+
StringContainsQ[ string, "Entity[\"City\", {\"Springfield\", \"Illinois\", \"UnitedStates\"}]" ],
116+
True,
117+
SameTest -> MatchQ,
118+
TestID -> "NaturalLanguageInput-2-Result@@Tests/WolframLanguageToolEvaluate.wlt:114,1-119,2"
119+
]
120+
121+
(* ::**************************************************************************************************************:: *)
122+
(* ::Subsection::Closed:: *)
123+
(*Multimodal Input*)
124+
VerificationTest[
125+
WolframLanguageToolEvaluate[ { "ImageDimensions[", RandomImage[ ], "]" }, "Result", Method -> "Session" ],
126+
HoldForm @ { _Integer, _Integer },
127+
SameTest -> MatchQ,
128+
TestID -> "MultimodalInput-1@@Tests/WolframLanguageToolEvaluate.wlt:124,1-129,2"
129+
]
130+
131+
(* ::**************************************************************************************************************:: *)
132+
(* ::Subsection::Closed:: *)
133+
(*Auto-Correcting Input*)
134+
VerificationTest[
135+
WolframLanguageToolEvaluate[ "Dimensions[{{1,2},{3,4},{5,6}}", "Result", Method -> "Session" ],
136+
HoldForm @ { 3, 2 },
137+
SameTest -> MatchQ,
138+
TestID -> "AutoCorrectingInput-1@@Tests/WolframLanguageToolEvaluate.wlt:134,1-139,2"
139+
]
140+
141+
VerificationTest[
142+
WolframLanguageToolEvaluate[ { "ImageDimensions[", RandomImage[ ] }, "Result", Method -> "Session" ],
143+
HoldForm @ { _Integer, _Integer },
144+
SameTest -> MatchQ,
145+
TestID -> "AutoCorrectingInput-2@@Tests/WolframLanguageToolEvaluate.wlt:141,1-146,2"
146+
]

0 commit comments

Comments
 (0)