Skip to content

Commit 57fc81c

Browse files
progoramurprogoramur
committed
Add tests for OCaml indexer.
1 parent a2e360c commit 57fc81c

File tree

6 files changed

+377
-0
lines changed

6 files changed

+377
-0
lines changed

‎opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/ocaml/OCamlSymbolTokenizerTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Portions Copyright (c) 2025, Yelisey Romanov <[email protected]>.
22+
*/
23+
package org.opengrok.indexer.analysis.ocaml;
24+
25+
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
26+
27+
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
28+
import org.junit.jupiter.api.Test;
29+
import org.opengrok.indexer.analysis.AbstractAnalyzer;
30+
import org.opengrok.indexer.analysis.JFlexTokenizer;
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.io.InputStreamReader;
34+
import java.io.Reader;
35+
import java.nio.charset.StandardCharsets;
36+
import java.util.LinkedList;
37+
import java.util.List;
38+
39+
/**
40+
* Tests the {@link OCamlSymbolTokenizer} class.
41+
*/
42+
class OCamlSymbolTokenizerTest {
43+
44+
private final AbstractAnalyzer analyzer;
45+
46+
OCamlSymbolTokenizerTest() {
47+
this.analyzer = new OCamlAnalyzerFactory().getAnalyzer();
48+
}
49+
50+
private String[] getTermsFor(Reader r) {
51+
List<String> l = new LinkedList<>();
52+
JFlexTokenizer ts = (JFlexTokenizer) this.analyzer.tokenStream("refs", r);
53+
ts.setReader(r);
54+
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
55+
try {
56+
ts.reset();
57+
while (ts.incrementToken()) {
58+
l.add(term.toString());
59+
}
60+
} catch (IOException ex) {
61+
throw new RuntimeException(ex);
62+
}
63+
64+
return l.toArray(new String[0]);
65+
}
66+
67+
@Test
68+
void sampleTest() throws IOException {
69+
try (InputStream res = getClass().getClassLoader().getResourceAsStream("analysis/ocaml/sample.ml");
70+
InputStreamReader r = new InputStreamReader(res, StandardCharsets.UTF_8)) {
71+
String[] termsFor = getTermsFor(r);
72+
assertArrayEquals(
73+
new String[] {
74+
"print_string", "again", "print_string", "again",
75+
"'a", "tau", "Tau", "'a", "Phi", "'a", "Omicron",
76+
"weLovePolymorphicVariants", "`Right", "`OrNot", "`OrUnsure",
77+
"weLoveVariablesWithQuotes'", "None",
78+
"failwith",
79+
"Some", "reason", "_is_needed_for", "result",
80+
"reason", "failwith", "result",
81+
"_sum_some_numbers", "Int64", "to_int",
82+
"Nativeint", "to_int",
83+
"Int32", "to_int",
84+
"Int32", "to_int",
85+
"_float_around"
86+
},
87+
termsFor);
88+
}
89+
}
90+
}

‎opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/ocaml/OCamlXrefTest.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Portions Copyright (c) 2025, Yelisey Romanov <[email protected]>.
22+
*/
23+
package org.opengrok.indexer.analysis.ocaml;
24+
25+
import java.io.BufferedReader;
26+
import java.io.ByteArrayOutputStream;
27+
import java.io.IOException;
28+
import java.io.InputStream;
29+
import java.io.InputStreamReader;
30+
import java.io.PrintStream;
31+
import java.io.StringReader;
32+
import java.io.StringWriter;
33+
import java.io.Writer;
34+
import java.nio.charset.StandardCharsets;
35+
36+
import org.junit.jupiter.api.Test;
37+
import org.opengrok.indexer.analysis.AbstractAnalyzer;
38+
import org.opengrok.indexer.analysis.CtagsReader;
39+
import org.opengrok.indexer.analysis.Definitions;
40+
import org.opengrok.indexer.analysis.WriteXrefArgs;
41+
import org.opengrok.indexer.analysis.Xrefer;
42+
43+
import static org.junit.jupiter.api.Assertions.assertEquals;
44+
import static org.junit.jupiter.api.Assertions.assertNotNull;
45+
import static org.opengrok.indexer.util.CustomAssertions.assertLinesEqual;
46+
import static org.opengrok.indexer.util.StreamUtils.copyStream;
47+
48+
/**
49+
* Tests the {@link OCamlXref} class.
50+
*/
51+
class OCamlXrefTest {
52+
53+
@Test
54+
void basicTest() throws IOException {
55+
String s = "print_string \"Hello, world!\"";
56+
Writer w = new StringWriter();
57+
OCamlAnalyzerFactory fac = new OCamlAnalyzerFactory();
58+
AbstractAnalyzer analyzer = fac.getAnalyzer();
59+
WriteXrefArgs xargs = new WriteXrefArgs(new StringReader(s), w);
60+
Xrefer xref = analyzer.writeXref(xargs);
61+
assertLinesEqual("OCaml basicTest",
62+
"<a class=\"l\" name=\"1\" href=\"#1\">1</a>" +
63+
"<a href=\"/source/s?defs=print_string\" class=\"intelliWindow-symbol\"" +
64+
" data-definition-place=\"undefined-in-file\">print_string</a>" +
65+
" <span class=\"s\">&quot;Hello, world!&quot;</span>\n",
66+
w.toString());
67+
assertEquals(1, xref.getLOC(), "OCaml LOC");
68+
}
69+
70+
private static int writeOCamlXref(InputStream is, PrintStream os,
71+
Definitions defs) throws IOException {
72+
os.println("<!DOCTYPE html><html lang=\"en\"><head><meta http-equiv=\"content-type\" content=\"text/html;charset=UTF-8\" />"
73+
+ "<link rel=\"stylesheet\" type=\"text/css\" "
74+
+ "href=\"http://localhost:8080/source/default/style.css\" /><title>OCaml Xref Test</title></head>");
75+
os.println("<body><div id=\"src\"><pre>");
76+
Writer w = new StringWriter();
77+
OCamlAnalyzerFactory fac = new OCamlAnalyzerFactory();
78+
AbstractAnalyzer analyzer = fac.getAnalyzer();
79+
WriteXrefArgs args = new WriteXrefArgs(new InputStreamReader(is, StandardCharsets.UTF_8), w);
80+
args.setDefs(defs);
81+
Xrefer xref = analyzer.writeXref(args);
82+
os.print(w.toString());
83+
os.println("</pre></div></body></html>");
84+
return xref.getLOC();
85+
}
86+
87+
@Test
88+
void sampleTest() throws IOException {
89+
// load sample source
90+
InputStream sampleInputStream = getClass().getClassLoader().getResourceAsStream(
91+
"analysis/ocaml/sample.ml");
92+
ByteArrayOutputStream sampleOutputStream = new ByteArrayOutputStream();
93+
94+
Definitions defs = new Definitions();
95+
defs.addTag(6, "x'y'", "functions",
96+
"let x'y' = let f' = 1; g'h = 2 in f' + g'h", 0, 0);
97+
98+
int actLOC;
99+
try {
100+
actLOC = writeOCamlXref(sampleInputStream, new PrintStream(sampleOutputStream), defs);
101+
} finally {
102+
sampleInputStream.close();
103+
sampleOutputStream.close();
104+
}
105+
106+
// load expected xref
107+
InputStream expectedInputStream = getClass().getClassLoader().getResourceAsStream(
108+
"analysis/ocaml/sample_xref.html");
109+
ByteArrayOutputStream expectedOutputSteam = new ByteArrayOutputStream();
110+
try {
111+
byte[] buffer = new byte[8192];
112+
int numBytesRead;
113+
do {
114+
numBytesRead = expectedInputStream.read(buffer, 0, buffer.length);
115+
if (numBytesRead > 0) {
116+
expectedOutputSteam.write(buffer, 0, numBytesRead);
117+
}
118+
} while (numBytesRead >= 0);
119+
} finally {
120+
expectedInputStream.close();
121+
expectedOutputSteam.close();
122+
}
123+
124+
String[] actual = new String(sampleOutputStream.toByteArray(), StandardCharsets.UTF_8).split("\\r?\\n");
125+
String[] expected = new String(expectedOutputSteam.toByteArray(), StandardCharsets.UTF_8).split("\\r?\\n");
126+
assertLinesEqual("OCaml sampleTest()", expected, actual);
127+
assertEquals(17, actLOC, "OCaml LOC");
128+
}
129+
130+
@Test
131+
void sampleTest2() throws IOException {
132+
writeAndCompare("analysis/ocaml/sample2.ml",
133+
"analysis/ocaml/sample2_xref.html",
134+
getTagsDefinitions(), 11);
135+
}
136+
137+
private void writeAndCompare(String sourceResource, String resultResource,
138+
Definitions defs, int expLOC) throws IOException {
139+
140+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
141+
142+
InputStream res = getClass().getClassLoader().getResourceAsStream(
143+
sourceResource);
144+
assertNotNull(res, sourceResource + " should get-as-stream");
145+
int actLOC = writeOCamlXref(res, new PrintStream(baos), defs);
146+
res.close();
147+
148+
InputStream exp = getClass().getClassLoader().getResourceAsStream(
149+
resultResource);
150+
assertNotNull(exp, resultResource + " should get-as-stream");
151+
byte[] expbytes = copyStream(exp);
152+
exp.close();
153+
baos.close();
154+
155+
String ostr = new String(baos.toByteArray(), StandardCharsets.UTF_8);
156+
String[] gotten = ostr.split("\\r?\\n");
157+
158+
String estr = new String(expbytes, StandardCharsets.UTF_8);
159+
String[] expected = estr.split("\n");
160+
161+
assertLinesEqual("OCaml xref", expected, gotten);
162+
assertEquals(expLOC, actLOC, "OCaml LOC");
163+
}
164+
165+
private Definitions getTagsDefinitions() throws IOException {
166+
InputStream res = getClass().getClassLoader().getResourceAsStream(
167+
"analysis/ocaml/sampletags");
168+
assertNotNull(res, "though sampletags should stream,");
169+
170+
BufferedReader in = new BufferedReader(new InputStreamReader(res, StandardCharsets.UTF_8));
171+
172+
CtagsReader rdr = new CtagsReader();
173+
String line;
174+
while ((line = in.readLine()) != null) {
175+
rdr.readLine(line);
176+
}
177+
return rdr.getDefinitions();
178+
}
179+
}

‎opengrok-indexer/src/test/resources/analysis/ocaml/sample.ml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
(* The sample file. *)
2+
print_string "Hello World!\n";;
3+
4+
let again = print_string and
5+
string = {bar|Another string|bar} in
6+
(again [@tailcall]) string;;
7+
(* Note, identifier 'string' is deliberately ignored
8+
by tokenizer. *)
9+
10+
type 'a tau = Tau of 'a | Phi of 'a list | Omicron;;
11+
(* Btw, do you know that
12+
'a is read as α
13+
'b is read as ß
14+
'c is γ - γάμμα ! *)
15+
16+
let weLovePolymorphicVariants = [`Right ; `OrNot ; `OrUnsure];;
17+
18+
let weLoveVariablesWithQuotes' = function None -> failwith "???"
19+
| Some reason ->
20+
let _is_needed_for = 8n and
21+
result = reason in
22+
failwith result;;
23+
(* Note: 'result' is not ignored, like 'string' *)
24+
let _sum_some_numbers = Int64.to_int 10_8_8L +
25+
Nativeint.to_int 0xDEADF00Dn + Int32.to_int 0o76l + 0b101 +
26+
Int32.to_int 0b111001l in
27+
();;
28+
29+
let _float_around = 1.8E+23 +. 1_2_3_4.8_8E-2 in
30+
();;

‎opengrok-indexer/src/test/resources/analysis/ocaml/sample2.ml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
(* Test comments and extension nodes *)
2+
3+
(* "*)" *)
4+
5+
let _c = 'c' and
6+
_d = '\78' and
7+
_e = '\o003' and
8+
_f = '\xAf'
9+
10+
(* {|*)|} *)
11+
12+
let str = {| (* *) |}
13+
14+
(* '"' *)
15+
16+
let _ = [%string {| (* *) |}]
17+
18+
(* f' '"' *)
19+
20+
let f = {%string | (* *) |}]

‎opengrok-indexer/src/test/resources/analysis/ocaml/sample2_xref.html‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!DOCTYPE html><html lang="en"><head><meta http-equiv="content-type" content="text/html;charset=UTF-8" /><link rel="stylesheet" type="text/css" href="http://localhost:8080/source/default/style.css" /><title>OCaml Xref Test</title></head>
2+
<body><div id="src"><pre>
3+
<script type="text/javascript">/* <![CDATA[ */
4+
function get_sym_list(){return [["Function","xf",[["add_suffixes",158],["by_code",75],["by_quoted",73],["by_x",83],["by_y",84],["check_duplicate_vars",133],["check_join_vars",103],["dup",116],["dup",134],["i",164],["join_cols",26],["missing",124],["na",108],["nm",165],["out",163],["standardise_join_by",64],["standardise_join_suffix",143],["x_by",33],["x_is_aux",42],["x_loc",36],["y_aux",41],["y_by",34],["y_loc",50]]]];} /* ]]> */</script><a class="l" name="1" href="#1">1</a><span class="c">(* Test comments and extension nodes *)</span>
5+
<a class="l" name="2" href="#2">2</a>
6+
<a class="l" name="3" href="#3">3</a><span class="c">(* &quot;*)&quot;</span><span class="c"> *)</span>
7+
<a class="l" name="4" href="#4">4</a>
8+
<a class="l" name="5" href="#5">5</a><b>let</b> <a href="/source/s?defs=_c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_c</a> = <span class="s">&apos;c&apos;</span> <b>and</b>
9+
<a class="l" name="6" href="#6">6</a> <a href="/source/s?defs=_d" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_d</a> = &apos;\<span class="n">78</span>&apos; <b>and</b>
10+
<a class="l" name="7" href="#7">7</a> <a href="/source/s?defs=_e" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_e</a> = <span class="s">&apos;\o003&apos;</span> <b>and</b>
11+
<a class="l" name="8" href="#8">8</a> <a href="/source/s?defs=_f" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_f</a> = &apos;\<a href="/source/s?defs=xAf%27" class="intelliWindow-symbol" data-definition-place="undefined-in-file">xAf&apos;</a>
12+
<a class="l" name="9" href="#9">9</a>
13+
<a class="hl" name="10" href="#10">10</a><span class="c">(* {|*)|}</span><span class="c"> *)</span>
14+
<a class="l" name="11" href="#11">11</a>
15+
<a class="l" name="12" href="#12">12</a><b>let</b> <a href="/source/s?defs=str" class="intelliWindow-symbol" data-definition-place="undefined-in-file">str</a> = <span class="s">{| (* *) |}</span>
16+
<a class="l" name="13" href="#13">13</a>
17+
<a class="l" name="14" href="#14">14</a><span class="c">(* &apos;&quot;&apos; *)
18+
<a class="l" name="15" href="#15">15</a>
19+
<a class="l" name="16" href="#16">16</a>let _ = [%string {| (* *) |}]
20+
<a class="l" name="17" href="#17">17</a>
21+
<a class="l" name="18" href="#18">18</a>(* f&apos; &apos;&quot;</span><span class="c">&apos; *)</span>
22+
<a class="l" name="19" href="#19">19</a>
23+
<a class="hl" name="20" href="#20">20</a><b>let</b> <a href="/source/s?defs=f" class="intelliWindow-symbol" data-definition-place="undefined-in-file">f</a> = <span class="xm">{%string </span><span class="s">| (* *) |}</span>]
24+
<a class="l" name="21" href="#21">21</a></pre></div></body></html>

0 commit comments

Comments
 (0)