28
28
# Detecting this ahead of time is difficult so for now we use a global variable which stores whether the API has
29
29
# responded with a HTTP 400 error and formats all subsequent request to avoid using a system role.
30
30
INCLUDE_SYSTEM_PROMPT = True
31
+
32
+
31
33
class PossibleSystemPromptException (Exception ):
32
34
pass
33
35
36
+
34
37
llm = ChatOpenAI (
35
38
base_url = urljoin (backend_url , "v1" ),
36
39
model = settings .hf_model_name ,
@@ -47,7 +50,6 @@ class PossibleSystemPromptException(Exception):
47
50
48
51
49
52
def inference (latest_message , history ):
50
-
51
53
# Allow mutating global variable
52
54
global BACKEND_INITIALISED
53
55
@@ -67,10 +69,9 @@ def inference(latest_message, history):
67
69
68
70
response = ""
69
71
for chunk in llm .stream (context ):
70
-
71
72
# If this is our first successful response from the backend
72
73
# then update the status variable to allow future error messages
73
- # to be more informative
74
+ # to be more informative
74
75
if not BACKEND_INITIALISED and len (response ) > 0 :
75
76
BACKEND_INITIALISED = True
76
77
@@ -87,7 +88,7 @@ def inference(latest_message, history):
87
88
88
89
except openai .BadRequestError as err :
89
90
logger .error ("Received BadRequestError from backend API: %s" , err )
90
- message = err .response .json ()[' message' ]
91
+ message = err .response .json ()[" message" ]
91
92
if INCLUDE_SYSTEM_PROMPT :
92
93
raise PossibleSystemPromptException ()
93
94
else :
@@ -98,13 +99,17 @@ def inference(latest_message, history):
98
99
except openai .APIConnectionError as err :
99
100
if not BACKEND_INITIALISED :
100
101
logger .info ("Backend API not yet ready" )
101
- gr .Info ("Backend not ready - model may still be initialising - please try again later" )
102
+ gr .Info (
103
+ "Backend not ready - model may still be initialising - please try again later"
104
+ )
102
105
else :
103
106
logger .error ("Failed to connect to backend API: %s" , err )
104
107
gr .Warning ("Failed to connect to backend API" )
105
108
106
109
except openai .InternalServerError as err :
107
- gr .Warning ("Internal server error encountered in backend API - see API logs for details." )
110
+ gr .Warning (
111
+ "Internal server error encountered in backend API - see API logs for details."
112
+ )
108
113
109
114
# Catch-all for unexpected exceptions
110
115
except err :
@@ -145,6 +150,7 @@ def inference_wrapper(*args):
145
150
for chunk in inference (* args ):
146
151
yield chunk
147
152
153
+
148
154
# Build main chat interface
149
155
with gr .ChatInterface (
150
156
inference_wrapper ,
0 commit comments