Skip to content

Commit 4a6bbbb

Browse files
augustepoirouxaugustepoiroux
committed
Export proofs extracted from TPTP to JSON
1 parent 57a57f0 commit 4a6bbbb

File tree

4 files changed

+132
-81
lines changed

4 files changed

+132
-81
lines changed

‎build.sbt‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@ ThisBuild / semanticdbEnabled := true
1616
ThisBuild / semanticdbVersion := scalafixSemanticdb.revision
1717
ThisBuild / scalafixDependencies += "com.github.liancheng" %% "organize-imports" % "0.6.0"
1818

19-
20-
21-
2219
val commonSettings = Seq(
2320
crossScalaVersions := Seq("2.12.13", "2.13.4", "3.0.1", "3.2.0"),
2421
run / fork := true
@@ -41,6 +38,7 @@ val commonSettings3 = commonSettings ++ Seq(
4138
),
4239
libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.10" % "test",
4340
libraryDependencies += "com.lihaoyi" %% "sourcecode" % "0.3.0",
41+
libraryDependencies += "com.lihaoyi" %% "ujson" % "3.1.0",
4442
// libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1",
4543
libraryDependencies += ("io.github.uuverifiers" %% "princess" % "2023-06-19").cross(CrossVersion.for3Use2_13),
4644
Test / parallelExecution := false
@@ -102,7 +100,6 @@ ThisBuild / assemblyMergeStrategy := {
102100
oldStrategy(x)
103101
}
104102

105-
106103
lazy val examples = Project(
107104
id = "lisa-examples",
108105
base = file("lisa-examples")

‎lisa-examples/src/main/scala/TPTPSolver.scala‎

Lines changed: 104 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,88 +7,126 @@ import lisa.utils.ProofsConverter.*
77
import lisa.utils.RunSolver.*
88
import lisa.utils.tptp.*
99
import lisa.utils.tptp.KernelParser.*
10-
import lisa.utils.tptp.ProblemGatherer.*
1110
import lisa.kernel.proof.SCProof
1211
import lisa.kernel.proof.SequentCalculus.Sequent
13-
import lisa.kernel.proof.SCProofChecker.checkSCProof
12+
import lisa.utils.ProofsShrink.optimizeProofIteratively
1413

1514
object TPTPSolver extends lisa.Main {
16-
try {
17-
val spc = Seq("PRP", "FOF") // type of problems we want to extract and solve
18-
// val spc = Seq("PRP", "FOF", "CNF") // almost no CNF problems are solved by Tableau, TODO: investigate why
19-
20-
// val d = new File(TPTPProblemPath)
21-
// val libraries = d.listFiles.filter(_.isDirectory)
22-
// val probfiles = libraries.flatMap(_.listFiles).filter(_.isFile)
23-
24-
val d = new File(TPTPProblemPath + "SYN/")
25-
val probfiles = d.listFiles.filter(_.isFile)
26-
27-
// We limit the execution time to solve each problem
28-
val timeoutTableau = .1.second
29-
val timeoutTautology = .1.second
30-
31-
var nbProblemsExtracted = 0
32-
var nbProblemsSolvedByTableau = 0
33-
var nbProblemsSolvedByTautology = 0
34-
35-
for ((probfile, i) <- probfiles.zipWithIndex) {
36-
// Progress bar
37-
if ((i + 1) % 10 == 0 || i + 1 == probfiles.size) {
38-
val pbarLength = 30
39-
var pbarContent = "=" * (((i + 1) * pbarLength) / probfiles.size)
40-
pbarContent += " " * (pbarLength - pbarContent.length)
41-
print(s"[$pbarContent]")
42-
print(s" -- ${i + 1}/${probfiles.size} processed files")
43-
print(s" -- $nbProblemsExtracted extracted problems")
44-
print(s" -- Tableau: $nbProblemsSolvedByTableau solved")
45-
println(s" -- Tautology: $nbProblemsSolvedByTautology solved")
46-
}
15+
sealed trait ProofType
16+
case object BySolver extends ProofType
17+
case object Kernel extends ProofType
18+
case object KernelOptimized extends ProofType
19+
20+
class ProblemSolverResults(val problem: Problem, val solverName: String, val solverStatus: String, val proofCode: String, val proofType: ProofType)
21+
22+
val exportOnlySolvedProblems = true
23+
val exportOptimizedProofs = true
24+
val exportBySolverProofs = true
25+
26+
val jsonResultsPath: String = "/home/auguste/Documents/EPFL/PhD/Projects/lisa/lisa-examples/src/main/resources/TPTPResults.json"
27+
val TPTPProblemPath: String = "/home/auguste/Documents/EPFL/PhD/Projects/TPTP-v8.2.0/Problems/"
28+
29+
val spc = Seq("PRP", "FOF") // type of problems we want to extract and solve
30+
// val spc = Seq("CNF") // almost no CNF problems are solved by Tableau, TODO: investigate why
31+
32+
// val d = new File(TPTPProblemPath)
33+
// val libraries = d.listFiles.filter(_.isDirectory)
34+
// val probfiles = libraries.flatMap(_.listFiles).filter(_.isFile)
35+
36+
val d = new File(TPTPProblemPath + "SYN/")
37+
val probfiles = d.listFiles.filter(_.isFile)
38+
39+
// We limit the execution time to solve each problem
40+
val timeoutTableau = .1.second
41+
val timeoutTautology = .1.second
4742

48-
// Try to extract and solve the problem
49-
try {
50-
val md = getProblemInfos(probfile)
43+
var nbProblemsExtracted = 0
44+
var nbProblemsSolved = Map("Tableau" -> 0, "Tautology" -> 0)
45+
var nbInvalidProofs = Map("Tableau" -> 0, "Tautology" -> 0)
46+
var results = Seq.empty[ProblemSolverResults]
47+
48+
for ((probfile, i) <- probfiles.zipWithIndex) {
49+
// Progress bar
50+
if ((i + 1) % 10 == 0 || i + 1 == probfiles.size) {
51+
val pbarLength = 20
52+
var pbarContent = "=" * (((i + 1) * pbarLength) / probfiles.size)
53+
pbarContent += " " * (pbarLength - pbarContent.length)
54+
print(s"[$pbarContent]")
55+
print(s" -- ${i + 1}/${probfiles.size} processed files")
56+
print(s" -- $nbProblemsExtracted extracted problems")
57+
print(s" -- Tableau: ${nbProblemsSolved("Tableau")} solved + ${nbInvalidProofs("Tableau")} invalid")
58+
println(s" -- Tautology: ${nbProblemsSolved("Tautology")} solved + ${nbInvalidProofs("Tautology")} invalid")
59+
}
60+
61+
// Try to extract and solve the problem
62+
try {
63+
val md = getProblemInfos(probfile)
64+
if (!(md.spc.contains("SAT"))) {
5165
if (md.spc.exists(spc.contains)) {
52-
val p = problemToKernel(probfile, md)
53-
val seq = problemToSequent(p)
66+
val problem = problemToKernel(probfile, md)
67+
val seq = problemToSequent(problem)
5468
nbProblemsExtracted += 1
5569

70+
def exportResults(problem: Problem, solverName: String, solverResult: SolverResult): Unit =
71+
val solverStatus = solverResult.getClass.getSimpleName.stripSuffix("$")
72+
solverResult match
73+
case Solved(proof) =>
74+
nbProblemsSolved += (solverName -> (nbProblemsSolved(solverName) + 1))
75+
results :+= new ProblemSolverResults(problem, solverName, solverStatus, generateStandaloneTheoremFileContent(problem.name, proof), Kernel)
76+
if (exportOptimizedProofs)
77+
results :+= new ProblemSolverResults(problem, solverName, solverStatus, generateStandaloneTheoremFileContent(problem.name, optimizeProofIteratively(proof)), KernelOptimized)
78+
if (exportBySolverProofs)
79+
val statementString = any2code(seq)
80+
val proofCode = s"have(thesis) by $solverName"
81+
val symbolDeclarations = generateSymbolDeclarationCode(Set(K.sequentToFormula(seq)), "private")
82+
results :+= new ProblemSolverResults(problem, solverName, solverStatus, generateStandaloneTheoremFileContent(problem.name, statementString, proofCode, symbolDeclarations), BySolver)
83+
case InvalidProof(proof) =>
84+
nbInvalidProofs += (solverName -> (nbInvalidProofs(solverName) + 1))
85+
if (!exportOnlySolvedProblems)
86+
results :+= new ProblemSolverResults(problem, solverName, solverStatus, generateStandaloneTheoremFileContent(problem.name, proof), Kernel)
87+
case _ =>
88+
if (!exportOnlySolvedProblems)
89+
results :+= new ProblemSolverResults(problem, solverName, solverStatus, "", Kernel)
90+
5691
// Attempting proof by Tableau
57-
proveSequent(seq, timeoutTableau, Tableau.solve) match {
58-
case Solved(proof) =>
59-
if (checkSCProof(proof).isValid)
60-
nbProblemsSolvedByTableau += 1
61-
// writeProof(p, proof, "examples/proofs/tableau/")
62-
else throw new Exception("Tableau proof is not valid")
63-
case _ => ()
64-
}
92+
val tableauResult = proveSequent(seq, timeoutTableau, Tableau.solve)
93+
exportResults(problem, "Tableau", tableauResult)
6594

6695
// Attempting proof by Tautology
6796
def tautologySolver(s: lisa.utils.K.Sequent): Option[SCProof] = Tautology.solveSequent(s) match
6897
case Left(proof) => Some(proof)
6998
case _ => None
70-
proveSequent(seq, timeoutTautology, tautologySolver) match {
71-
case Solved(proof) =>
72-
if (checkSCProof(proof).isValid)
73-
nbProblemsSolvedByTautology += 1
74-
// writeProof(p, proof, "examples/proofs/tautology/")
75-
else throw new Exception("Tautology proof is not valid")
76-
case _ => ()
77-
}
99+
val tautologyResult = proveSequent(seq, timeoutTautology, tautologySolver)
100+
exportResults(problem, "Tautology", tautologyResult)
78101
}
79-
} catch {
80-
case e => println(s"Error while processing $probfile: $e")
102+
// } else println(s"Problem $probfile not extracted because of its type: ${md.spc.mkString(",")}")
81103
}
82-
}
83-
} catch {
84-
case error: NullPointerException => println("You can download the tptp library at http://www.tptp.org/ and put it in main/resources")
104+
} catch case e => println(s"Error while processing $probfile (index $i): $e")
85105
}
86-
}
87106

88-
def writeProof(problem: Problem, proof: SCProof, path: String): Unit = {
89-
val file = new File(path + problem.name + ".sc")
90-
val bw = new FileWriter(file)
91-
val fileContent = generateStandaloneTheoremFileContent(problem.name, proof)
92-
bw.write(fileContent)
93-
bw.close()
107+
// Write results to a JSON file
108+
val jsonResultsFile = new File(jsonResultsPath)
109+
if (!jsonResultsFile.getParentFile.exists())
110+
jsonResultsFile.getParentFile.mkdirs()
111+
val jsonWriter = new java.io.PrintWriter(jsonResultsPath)
112+
ujson.writeTo(
113+
results.map(r =>
114+
ujson.Obj(
115+
"problemName" -> r.problem.name,
116+
"problemDomain" -> r.problem.domain,
117+
"problemStatus" -> r.problem.status,
118+
"problemSPC" -> r.problem.spc.mkString(","),
119+
"problemSequent" -> any2code(problemToSequent(r.problem)),
120+
"problemFile" -> r.problem.file,
121+
"solver" -> r.solverName,
122+
"solverStatus" -> r.solverStatus,
123+
"solverProofCode" -> r.proofCode,
124+
"proofType" -> r.proofType.getClass.getSimpleName.stripSuffix("$")
125+
)
126+
),
127+
jsonWriter,
128+
indent = 2
129+
)
130+
jsonWriter.close()
131+
94132
}

‎lisa-utils/src/main/scala/lisa/utils/ProofsConverter.scala‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ object ProofsConverter {
166166
*
167167
* @return Scala code representing the variables in string format
168168
*/
169-
private def generateVariablesCode(formulas: Set[K.Formula], accessibility: String): String =
169+
def generateSymbolDeclarationCode(formulas: Set[K.Formula], accessibility: String): String =
170170
val (variableSet, functionSet, formulaVariableSet, predicateSet, connectorSet) = extractSymbols(formulas)
171171
val access = if accessibility != "" then accessibility.strip() + " " else ""
172172
(variableSet.map(v => access + s"val ${v.id} = variable").toList.sorted ++
@@ -184,8 +184,8 @@ object ProofsConverter {
184184
*
185185
* @return Scala code representing the variables in string format
186186
*/
187-
def generateVariablesCode(proof: K.SCProof, accessibility: String = "private"): String =
188-
generateVariablesCode(extractFormulasFromProof(proof), accessibility)
187+
def generateSymbolDeclarationCode(proof: K.SCProof, accessibility: String = "private"): String =
188+
generateSymbolDeclarationCode(extractFormulasFromProof(proof), accessibility)
189189

190190
/**
191191
* Generates a valid Scala/Lisa code of a theorem and its proof
@@ -195,13 +195,13 @@ object ProofsConverter {
195195
*
196196
* @return Scala code representing the theorem in string format
197197
*/
198-
private def generateTheoremCode(name: String, proof: K.SCProof): String = {
198+
def generateTheoremCode(name: String, statementString: String, proofCode: String): String = {
199199
// lowercase and underscore-separated version of the theorem name
200200
val filteredName = "[A-Za-z0-9]+".r.findAllIn(name).mkString("_").toLowerCase
201201
s"val $filteredName = Theorem(\n" +
202-
indent(any2code(proof.conclusion)) +
202+
indent(statementString) +
203203
s"\n) {\n" +
204-
indent(scproof2code(proof)) +
204+
indent(proofCode) +
205205
s"\n}"
206206
}
207207

@@ -210,20 +210,32 @@ object ProofsConverter {
210210
* The theorem and its proof must be self-contained, i.e. no dependencies to other theorems, axioms, definitions, etc.
211211
*
212212
* @param name name of the theorem
213-
* @param proof proof of the theorem
213+
* @param proofCode code of the proof of the theorem
214214
*
215215
* @return Scala code representing the theorem in string format
216216
*/
217-
def generateStandaloneTheoremFileContent(name: String, proof: K.SCProof): String =
217+
def generateStandaloneTheoremFileContent(name: String, statementString: String, proofCode: String, symbolDeclarations: String): String =
218218
val camelName = "[A-Za-z0-9]+".r.findAllIn(name).map(_.capitalize).mkString
219219
s"object $camelName extends lisa.Main {\n\n" +
220220
indent(
221-
generateVariablesCode(proof) +
221+
symbolDeclarations +
222222
"\n\n" +
223-
generateTheoremCode(name, proof)
223+
generateTheoremCode(name, statementString, proofCode)
224224
) +
225225
"\n}"
226226

227+
/**
228+
* Generates a valid Scala/Lisa code of a theorem and its proof in a standalone file, including the necessary variables declarations.
229+
* The theorem and its proof must be self-contained, i.e. no dependencies to other theorems, axioms, definitions, etc.
230+
*
231+
* @param name name of the theorem
232+
* @param proof proof of the theorem
233+
*
234+
* @return Scala code representing the theorem in string format
235+
*/
236+
def generateStandaloneTheoremFileContent(name: String, proof: K.SCProof): String =
237+
generateStandaloneTheoremFileContent(name, any2code(proof.conclusion), scproof2code(proof), generateSymbolDeclarationCode(proof))
238+
227239
/**
228240
* Parse and check that a generated theorem file is valid, i.e. that it compiles and the theorem is proven
229241
* @param fileContent content of the generated file

‎lisa-utils/src/main/scala/lisa/utils/RunSolver.scala‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ import java.util.concurrent.CancellationException
77

88
import lisa.kernel.proof.SCProof
99
import lisa.kernel.proof.SequentCalculus.Sequent
10+
import lisa.kernel.proof.SCProofChecker.checkSCProof
1011

1112
object RunSolver {
1213
sealed trait SolverResult
1314
case class Solved(proof: SCProof) extends SolverResult
15+
case class InvalidProof(proof: SCProof) extends SolverResult
1416
case object Unsolved extends SolverResult
1517
case object Timeout extends SolverResult
1618
case class SolverThrow(t: Throwable) extends SolverResult
@@ -19,7 +21,9 @@ object RunSolver {
1921
val (futureSolver, cancelSolver) = Future.interruptibly { solver(sequent) }
2022
try
2123
Await.result(futureSolver, timeout) match
22-
case Some(proof) => Solved(proof)
24+
case Some(proof) =>
25+
if (checkSCProof(proof).isValid) Solved(proof)
26+
else InvalidProof(proof)
2327
case None => Unsolved
2428
catch
2529
case _: TimeoutException =>

0 commit comments

Comments
 (0)