Improved prompts for LINDDUN Go

Author: AndreaBissoli

llms/linddun_go.py

@@ -130,8 +130,8 @@ def get_linddun_go(api_key, model_name, inputs, threats_to_analyze, temperature,
 
         if model_name in ["gpt-4o", "gpt-4o-mini"] or lmstudio:
             class Threat(BaseModel):
-                reply: bool
                 reason: str
+                reply: bool
             response = client.beta.chat.completions.parse(
                 model=model_name,
                 messages=messages,
@@ -296,8 +296,8 @@ def get_response_openai(client, model, temperature, system_prompt, user_prompt,
     ]
     if model in ["gpt-4o", "gpt-4o-mini"] or lmstudio:
         class Threat(BaseModel):
-            reply: bool
             reason: str
+            reply: bool
         response = client.beta.chat.completions.parse(
             model=model,
             response_format=Threat,
@@ -411,8 +411,8 @@ def judge(keys, models, previous_analysis, temperature, lmstudio=False):
     ]
     if models["openai_model"] in ["gpt-4o", "gpt-4o-mini"] or lmstudio:
         class Threat(BaseModel):
-            reply: bool
             reason: str
+            reply: bool
         response = client.beta.chat.completions.parse(
             model=models["openai_model"] if not lmstudio else models["lmstudio_model"],
             response_format=Threat,

llms/prompts.py

@@ -157,12 +157,21 @@ def LINDDUN_GO_USER_PROMPT(inputs, question, title, description):
 LINDDUN_GO_SYSTEM_PROMPT = """
 When providing the answer, you MUST reply with a JSON object with the following structure:
 {
-    "reply": <boolean>,
     "reason": <string>
+    "reply": <boolean>,
 }
 
-When the answer to the questions is positive or indicates the presence of the threat, set the "reply" field to true. If the answer is negative or indicates the absence of the threat, set the "reply" field to false. The "reason" field should contain a string explaining why the threat is present or not.
-Ensure that the reason is specific to the application description and the question asked, referring to both of them in your response.
+When the answer to the questions is positive or indicates the presence of the
+threat, set the "reply" field to true. If the answer is negative or indicates
+the absence of the threat, set the "reply" field to false. The "reason" field
+should contain a string explaining extensively why the threat is present or
+not, and some concrete examples of how it could be exploited.
+BE VERY CRITICAL AND THOROUGH IN YOUR ANALYSIS: do not assume the threat is
+always present. ONLY set the "reply" field to true if you are mostly sure the
+threat is applicable to the system.
+Ensure that the reason is VERY SPECIFIC to the application description and the
+question asked, referring to both of them in your response and tailoring it
+accordingly.
 
 
 The input is enclosed in triple quotes.
@@ -198,17 +207,11 @@ def LINDDUN_GO_USER_PROMPT(inputs, question, title, description):
 ]}
 DATA POLICY: the data policy of the application
 USER DATA CONTROL: the control the user has over their data
-QUESTIONS: the questions associated with the threat, which you need to answer
+QUESTIONS: the questions associated with the threat, which you need to answer to understand if the threat is present or not
 THREAT_TITLE: the threat title
 THREAT_DESCRIPTION: the threat description
 '''
 
-Example of expected JSON response format:
-
-{
-    "reply": true,
-    "reason": "The threat is present because the application description mentions that the application is internet facing and uses a weak authentication method."
-}
 """
 
 LINDDUN_GO_JUDGE_PROMPT="""