added code for tool curation

Author: Andy-jqa

tool_curtaion/README.md

@@ -0,0 +1,50 @@
+__author__ = "qiao"
+
+"""
+Classify whether the candidate article introduces a new risk score / calculator.
+"""
+
+import json
+import os
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+if __name__ == "__main__":
+	# load cached results
+	target_path = "file1_sample_classification_results.jsonl"
+	done_pmids = []
+	if os.path.exists(target_path):
+		with open(target_path, "r") as f:
+			for line in f.readlines():
+				done_pmids.append(json.loads(line.strip())["pmid"])
+	done_pmids = set(done_pmids)
+
+	# loop over all pmids
+	cands = json.load(open("file0_sample_candidate_articles.json", "r"))
+	for pmid, info in cands.items():
+		if pmid in done_pmids:
+			continue
+
+		prompt = "Here is a PubMed article:\n" 
+		prompt += info["t"] + "\n" 
+		prompt += info["a"] + "\n"
+		prompt += "Does this article describe a new risk score or risk calculator? In healthcare, a risk score quantitatively estimates the probability of a clinical event or outcome, such as disease development or progression, within a specified period. These scores are derived from algorithms using variables like patient demographics, clinical history, laboratory results, and other relevant health indicators. They aid clinicians in decision-making, allowing for personalized patient care and resource allocation. Simply answer with \"yes\" or \"no\":"
+
+		response = client.chat.completions.create(
+			model="gpt-35-turbo",
+			messages=[{"role": "user", "content": prompt}],
+			temperature=0,
+		)
+
+		result = response.choices[0].message.content
+
+		with open(target_path, "a") as f:
+			f.write(json.dumps({"pmid": pmid, "result": result}) + "\n")
+
+		done_pmids.add(pmid)

tool_curtaion/dir1_pubmed_risk_calcs/__init__.py

@@ -0,0 +1,82 @@
+__author__ = "qiao"
+
+"""
+Draft risk calculators
+"""
+
+import json
+import os
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+
+if __name__ == "__main__":
+	system = "You are a helpful assistant programmer for medical calculators. Your task is to read a PubMed article about a medical calculator, and if applicable, write a two-step calculator: (1) calculator a risk score based on multiple criteria; (2) interpret different ranges of the computed risk score into probabilities of risks."
+	
+	pmid2info = json.load(open("file0_sample_candidate_articles.json"))
+	
+	cand_pmids = []
+	with open("file1_full_classification_results.jsonl", "r") as f:
+		for line in f.readlines():
+			line = json.loads(line.strip())
+			
+			if line["result"].lower() == "yes":
+				cand_pmids.append(line["pmid"])
+	
+	for pmid in cand_pmids:
+		target_path = os.path.join("dir1_pubmed_risk_calcs", pmid)
+
+		if os.path.exists(target_path):
+			continue
+
+		if pmid not in pmid2info:
+			print(f"PMID {pmid} information not found. Please load the full file 0 candidate articles!")
+			continue
+
+		prompt = "Here is a PubMed article:\n"
+		prompt += pmid2info[pmid]["t"] + "\n"
+		prompt += pmid2info[pmid]["a"] + "\n"
+
+		prompt += "Does the article describes a simple two-step risk calculator, where the first step is to compute a risk score, and the second step is to interpret different risk scores? If no, please directly and only output \"NO\". Otherwise, please standardize the calculator into:\n"
+
+		prompt += "#Title\nThe name of the calculator(s).\n"
+
+		prompt += "##Purpose\nDescribe when this calculator should be used.\n"
+
+		prompt += "##Specialty\nshould be a list of calculator types, one or more of (Allergy and Immunology, Anesthesiology, Cardiology, Dermatology, Emergency Medicine, Endocrinology, Family Medicine, Gastroenterology, Geriatrics, Hematology, Infectious Disease, Internal Medicine, Nephrology, Neurology, Obstetrics and Gynecology, Oncology, Ophthalmology, Orthopedic Surgery, Otolaryngology, Pathology, Pediatrics, Physical Medicine and Rehabilitation, Plastic Surgery, Psychiatry, Pulmonology, Radiology, Rheumatology, Surgery, Urology), seperated by \",\".\n"
+
+		prompt += "##Eligibility\nDescribe what patients are eligible.\n"
+
+		prompt += "##Size\nThe exact number of patients used to derive this calculator. Only put a number here without any other texts.\n"
+
+		prompt += "##Computation\nDetailed instructions of how to use the calculator, including Python functions with clear docstring documentation. Please be self-contained and detailed. For example, if the computation involves multiple items, please clearly list each item. If one item has multiple possible values (e.g., 0-2), you also need to clearly define what each value means.\n"
+
+		prompt += "##Interpretation\nShould be a list, where each item describes the interpretation (actual risk) for a value or a range of the computed risk score.\n"
+
+		prompt += "##Utility\nEvaluation results of the  clinical utility of the risk score, such as AUC, F-score, PPV.\n"
+
+		prompt += "##Example\nGenerate a sample patient note and a detailed demonstration of using the calculator and interpret the results. Think step-by-step here.\n"
+
+		prompt += "Please be as detailed as possible.\n"
+
+		messages = [
+			{"role": "system", "content": system},
+			{"role": "user", "content": prompt},
+		]
+
+		response = client.chat.completions.create(
+			model="gpt-4",
+			messages=messages,
+			temperature=0,
+		)
+
+		result = response.choices[0].message.content
+		
+		with open(target_path, "w") as f:
+			f.write(result)

tool_curtaion/dir2_risk_calc_verifications/__init__.py

@@ -0,0 +1,86 @@
+__author__ = "qiao"
+
+"""
+Verify the drafted clinical calculators
+"""
+
+import glob
+import json
+import os
+import sys
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+if __name__ == "__main__":
+	pmid2info = json.load(open("file0_sample_candidate_articles.json")) 
+	
+	for path in glob.glob("dir1_pubmed_risk_calcs/*"):
+
+		if ".py" in path:
+			continue
+
+		pmid = os.path.basename(path)
+		target_path = os.path.join("dir2_risk_calc_verifications", pmid)
+
+		if os.path.exists(target_path):
+			continue
+		
+		with open(path, "r") as f:
+			text = f.read()
+		
+		if text.lower().strip() == "no":
+			continue
+		
+		questions = [
+			"Are the parameters clearly defined in the #Computation? If a parameter can have different scores, the definitions for each score must be provided.",
+			"Are the parameters defined exactly the same in the article and the calculator?",
+			"Is the #Computation logic in the calculator fully based on the original article without any assumptions? Answer no if the article does not provide clear computing logics or weights.",
+			"Is the #Interpretation of the calculator fully based on the original article without any assumptions? Score ranges and corresponding risks should be exactly the same between the calculator and the article.",
+			"Is the #Interpretation of the calculator useful? A useful calculator should contain quantitative risk rates or qualitative risk groups for different score ranges.",
+			"Is the calculator free from any bug or other issue?",
+		]
+		answer_list = []
+
+		system = "You are a critical evaluator for a calculator that's supposed to describe a PubMed article. The calculator might contain errors. Always response in a JSON dict formatted as Dict{\"reasoning\": Str(critical_reasoning), \"answer\": Str(yes/no)}."
+
+		prompt = ""
+		prompt += "Here is the original PubMed article:\n"
+		prompt += pmid2info[pmid]["t"] + "\n"
+		prompt += pmid2info[pmid]["a"] + "\n"
+		prompt += "Here is the calculator that's supposed to describe the article above:\n"
+		prompt += text + "\n\n"
+
+		early_stop = False
+
+		for question in questions:
+			q_prompt = prompt + question
+
+			messages = [
+				{"role": "system", "content": system},
+				{"role": "user", "content": q_prompt},
+			]
+
+			response = client.chat.completions.create(
+				model="gpt-4",
+				messages=messages,
+				temperature=0,
+			)
+
+			response = json.loads(response.choices[0].message.content)
+
+			answer_list.append(response)
+
+			if response["answer"].lower() == "no":
+				early_stop = True
+			
+			if early_stop:
+				break
+
+		with open(target_path, "w") as f:
+			json.dump(answer_list, f, indent=4)