Skip to content

Commit 4cf8802

Browse files
zspitzerzspitzer
committed
LDEV-171 add lucee docs basic export for AI / LLM RAG
https://luceeserver.atlassian.net/browse/LD-171
1 parent 776417f commit 4cf8802

File tree

8 files changed

+317
-119
lines changed

8 files changed

+317
-119
lines changed

‎api/build/BuildRunner.cfc‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ component accessors=true {
4848
var builders = [];
4949

5050
for( var dir in dirs ){
51-
if ( dir.type == "dir"
52-
&& FileExists( dir.directory & "/#dir.name#/Builder.cfc" )
51+
if ( dir.type == "dir"
52+
&& FileExists( dir.directory & "/#dir.name#/Builder.cfc" )
5353
&& dir.name neq "dash" ) {
5454
builders.append( dir.name );
5555
}
@@ -88,14 +88,18 @@ component accessors=true {
8888

8989
arguments.builder.injectMethod = this.injectMethod;
9090

91-
arguments.builder.injectMethod( "renderLinks", function( required string text ){
92-
return new api.rendering.WikiLinksRenderer( docTree=variables.docTree ).renderLinks( text=arguments.text, builder=variables._builder );
91+
arguments.builder.injectMethod( "renderLinks", function( required string text, required struct args ){
92+
return new api.rendering.WikiLinksRenderer( docTree=variables.docTree ).renderLinks(
93+
text = arguments.text,
94+
builder = variables._builder,
95+
args = arguments.args
96+
);
9397
} );
9498
arguments.builder.injectMethod( "renderTemplate", function( required string template, struct args={} ){
9599
var renderer = new api.rendering.TemplateRenderer();
96100
var rendered = renderer.render( argumentCollection=arguments, template=_rootPathForRenderer & arguments.template );
97101

98-
return builder.renderLinks( rendered );
102+
return builder.renderLinks( rendered, args );
99103
} );
100104

101105
StructDelete( arguments.builder, "injectMethod" );

‎api/rendering/WikiLinksRenderer.cfc‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,16 @@ component accessors=true {
22

33
property name="docTree";
44

5-
public string function renderLinks( required string text, required any builder ) {
5+
public string function renderLinks( required string text, required any builder, required struct args ) {
66
var rendered = arguments.text;
77
var link = "";
88
var startPos = 1;
99
do {
1010
link = _getNextLink( rendered, startPos );
1111
if ( !IsNull( link ) ) {
12-
rendered = Replace( rendered, link.rawMatch, arguments.builder.renderLink( link.page ?: NullValue(), link.title ), "all" );
12+
rendered = Replace( rendered, link.rawMatch,
13+
arguments.builder.renderLink( link.page ?: NullValue(), link.title, args ),
14+
"all" );
1315
startPos = link.nextStartPos;
1416
}
1517
} while( !IsNull( link ) );

‎builders/html/Builder.cfc‎

Lines changed: 186 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,42 @@
11
component {
2-
public string function renderLink( any page, required string title ) {
3-
if ( IsNull( arguments.page ) ) {
4-
if (arguments.title.left(4) eq "http"){
5-
return '<a href="#arguments.title#">#HtmlEditFormat( arguments.title )#</a>';
6-
} else {
7-
request.logger (text="Missing docs link: [[#HtmlEditFormat( arguments.title )#]]", type="WARN");
8-
return '<a class="missing-link" title="missing link">#HtmlEditFormat( arguments.title )#</a>';
9-
}
10-
}
11-
var link = arguments.page.getPath() & ".html";
12-
return '<a href="#link#">#HtmlEditFormat( arguments.title )#</a>';
13-
}
14-
15-
public string function _getIssueTrackerLink(required string name) {
16-
var link = Replace( new api.build.BuildProperties().getIssueTrackerLink(), "{search}", urlEncodedFormat(arguments.name) )
17-
return '<a href="#link#" class="no-oembed" target="_blank">Search Issue Tracker <i class="fa fa-external-link"></i></a>';
18-
}
19-
20-
public string function _getTestCasesLink(required string name) {
21-
var link = Replace( new api.build.BuildProperties().getTestCasesLink(), "{search}", urlEncodedFormat(arguments.name) )
22-
return '<a href="#link#" class="no-oembed" target="_blank">Search Lucee Test Cases <i class="fa fa-external-link"></i></a> (good for further, detailed examples)';
23-
}
242

253
public void function build( required any docTree, required string buildDirectory, required numeric threads) {
264
var pagePaths = arguments.docTree.getPageCache().getPages();
275

286
request.filesWritten = 0;
297
request.filesToWrite = StructCount(pagePaths);
308

9+
var simpleBuildDirectory = arguments.buildDirectory & "Basic";
10+
11+
// purge previous build directory contents
12+
loop array="#[buildDirectory, simpleBuildDirectory]#" item="local.dir" {
13+
if ( directoryExists( dir ) )
14+
DirectoryDelete(dir, true);
15+
directoryCreate( dir );
16+
}
3117
request.logger (text="Builder HTML directory: #arguments.buildDirectory#");
18+
request.logger (text="Builder Simple HTML directory: #simplebuildDirectory#");
3219

3320
new api.parsers.ParserFactory().getMarkdownParser(); // so the parser in use shows up in logs
3421

3522
//for ( var path in pagePaths ) {
3623
each(pagePaths, function(path){
3724
var tick = getTickCount();
38-
_writePage( pagePaths[arguments.path].page, buildDirectory, docTree );
25+
var page = pagePaths[ arguments.path ].page;
26+
// write out full html page
27+
var pageContent = renderPageContent( page, docTree, false, {} );
28+
_writePage( page, buildDirectory, docTree, pageContent, {} );
29+
30+
// write out reduced stripped down basic html page with no navigation etc
31+
var basicArgs = {
32+
//"no_css": true,
33+
"no_google_analytics": true,
34+
"no_navigation": true,
35+
"mainTemplate": "main_basic.cfm",
36+
"base_href": _calcRelativeBaseHref( page, simpleBuildDirectory )
37+
};
38+
var basicPageContent = renderPageContent( page, docTree, false, basicArgs );
39+
_writePage( page, simpleBuildDirectory, docTree, basicPageContent, basicArgs );
3940

4041
request.filesWritten++;
4142
if ((request.filesWritten mod 100) eq 0){
@@ -51,13 +52,19 @@ component {
5152
_copyStaticAssets( arguments.buildDirectory );
5253
_copySiteImages( arguments.buildDirectory, arguments.docTree );
5354
_writeSearchIndex( arguments.docTree, arguments.buildDirectory );
55+
56+
_copyStaticAssets( simpleBuildDirectory);
57+
_zipBasicPages( arguments.buildDirectory, simpleBuildDirectory, "lucee-docs-basic" );
5458
}
5559

56-
public string function renderPage( required any page, required any docTree, required boolean edit ){
60+
public string function renderPageContent( required any page, required any docTree,
61+
required boolean edit, required struct htmlOpts ){
5762
try {
58-
var renderedPage = renderTemplate(
59-
template = "templates/#_getPageLayoutFile( arguments.page )#.cfm"
60-
, args = { page = arguments.page, docTree=arguments.docTree, edit=arguments.edit }
63+
var contentArgs = { page = arguments.page, docTree=arguments.docTree, edit=arguments.edit, htmlOpts=arguments.htmlOpts };
64+
65+
var pageContent = renderTemplate(
66+
template = "templates/#_getPageLayoutFile( arguments.page )#.cfm"
67+
, args = contentArgs
6168
, helpers = "/builders/html/helpers"
6269
);
6370
} catch( any e ) {
@@ -66,7 +73,13 @@ component {
6673
e.additional.luceeDocsPageId = arguments.page.getid();
6774
rethrow;
6875
}
69-
var crumbs = arguments.docTree.getPageBreadCrumbs(arguments.page);
76+
return pageContent;
77+
}
78+
79+
public string function renderPage( required any page, required any docTree,
80+
required string pageContent, required boolean edit, required struct htmlOptions ){
81+
82+
var crumbs = arguments.docTree.getPageBreadCrumbs( arguments.page );
7083
var excludeLinkMap = {}; // tracks links to exclude from See also
7184
var links = [];
7285
var categories = [];
@@ -112,21 +125,50 @@ component {
112125
break;
113126
}
114127
}
128+
129+
var template = arguments.htmlOptions.mainTemplate ?: "main.cfm";
130+
var crumbsArgs = {
131+
crumbs:crumbs,
132+
page: arguments.page,
133+
docTree: arguments.docTree,
134+
categories: categories.sort("textNoCase"),
135+
edit: arguments.edit,
136+
htmlOpts: arguments.htmlOptions
137+
};
138+
var seeAlsoArgs = {
139+
links= links,
140+
htmlOpts=arguments.htmlOptions
141+
}
142+
115143
try {
144+
145+
var args = {
146+
body = Trim( arguments.pageContent )
147+
, htmlOpts = arguments.htmlOptions
148+
, page = arguments.page
149+
, edit = arguments.edit
150+
, crumbs = renderTemplate( template="layouts/breadcrumbs.cfm", helpers = "/builders/html/helpers",
151+
args = crumbsArgs
152+
)
153+
, seeAlso = renderTemplate( template="layouts/seeAlso.cfm" , helpers = "/builders/html/helpers",
154+
args = seeAlsoArgs )
155+
};
156+
157+
if ( !structKeyExists(arguments.htmlOptions, "no_navigation" ) ){
158+
args.navTree = renderTemplate( template="layouts/sideNavTree.cfm", helpers = "/builders/html/helpers", args={
159+
crumbs=crumbs,
160+
docTree=arguments.docTree,
161+
pageLineage=arguments.page.getLineage(),
162+
pageLineageMap=arguments.page.getPageLineageMap()
163+
} );
164+
} else {
165+
args.navTree = "";
166+
}
167+
116168
var pageContent = renderTemplate(
117-
template = "layouts/main.cfm"
169+
template = "layouts/#template#"
118170
, helpers = "/builders/html/helpers"
119-
, args = {
120-
body = Trim( renderedPage )
121-
, page = arguments.page
122-
, edit = arguments.edit
123-
, crumbs = renderTemplate( template="layouts/breadcrumbs.cfm", helpers = "/builders/html/helpers", args={ crumbs=crumbs, page=arguments.page, docTree=arguments.docTree, categories=categories.sort("textNoCase"), edit= arguments.edit } )
124-
, navTree = renderTemplate( template="layouts/sideNavTree.cfm", helpers = "/builders/html/helpers", args={
125-
crumbs=crumbs, docTree=arguments.docTree, pageLineage=arguments.page.getLineage(), pageLineageMap=arguments.page.getPageLineageMap()
126-
} )
127-
, seeAlso = renderTemplate( template="layouts/seeAlso.cfm" , helpers = "/builders/html/helpers",
128-
args={ links=links } )
129-
}
171+
, args = args
130172
);
131173
} catch( any e ) {
132174
//e.additional.luceeDocsPage = arguments.page;
@@ -173,18 +215,24 @@ component {
173215
}
174216

175217
// PRIVATE HELPERS
176-
private void function _writePage( required any page, required string buildDirectory, required any docTree ) {
218+
private void function _writePage( required any page, required string buildDirectory,
219+
required any docTree, required string pageContent, required struct htmlOptions ) {
177220
var filePath = variables._getHtmlFilePath( arguments.page, arguments.buildDirectory );
178221
var fileDirectory = GetDirectoryFromPath( filePath );
179222

180-
//var starttime = getTickCount();
181223
lock name="CreateDirectory" timeout=10 {
182224
if ( !DirectoryExists( fileDirectory ) ) {
183225
DirectoryCreate( fileDirectory );
184226
}
185227
}
186-
var pageContent = variables.cleanHtml(variables.renderPage( arguments.page, arguments.docTree, false ));
187-
FileWrite( filePath, pageContent );
228+
229+
var html = variables.cleanHtml(
230+
variables.renderPage( arguments.page,
231+
arguments.docTree, arguments.pageContent, false ,
232+
arguments.htmlOptions, arguments.buildDirectory
233+
)
234+
);
235+
FileWrite( filePath, html );
188236
}
189237

190238
// regex strips left over whitespace multiple new lines
@@ -201,6 +249,18 @@ component {
201249
return arguments.buildDirectory & arguments.page.getPath() & ".html";
202250
}
203251

252+
private string function _calcRelativeBaseHref( required any page, required string buildDirectory ) {
253+
var path = arguments.page.getPath();
254+
var depth = listLen( path, "/" );
255+
if ( depth eq 1)
256+
return "";
257+
var baseHref = [];
258+
loop times="#depth-1#" {
259+
arrayAppend(baseHref, "..");
260+
}
261+
return ArrayToList(baseHref, "/");
262+
}
263+
204264
private void function _copyStaticAssets( required string buildDirectory ) {
205265
updateHighlightsCss( arguments.buildDirectory );
206266
var subdirs = directoryList(path=GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets", type="dir", recurse="false");
@@ -214,8 +274,10 @@ component {
214274

215275
private function updateHighlightsCss( required string buildDirectory ){
216276
var highlighter = new api.rendering.Pygments();
217-
var cssFile = path=GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets/css/highlight.css";
218-
fileWrite( cssFile, highlighter.getCss() );
277+
var cssFile = GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets/css/highlight.css";
278+
var css = highlighter.getCss();
279+
if ( trim( css ) neq trim( fileRead( cssFile ) ) )
280+
fileWrite( cssFile, highlighter.getCss() ); // only update if changed
219281
}
220282

221283
private void function _copySiteImages( required string buildDirectory, required any docTree ) {
@@ -305,4 +367,82 @@ component {
305367
return '<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#chr(10)#' &
306368
ArrayToList(siteMap, chr(10) ) & '#chr(10)#</urlset>';
307369
}
370+
371+
public string function renderLink( any page, required string title, required struct args ) {
372+
if ( IsNull( arguments.page ) ) {
373+
if (arguments.title.left(4) eq "http"){
374+
return '<a href="#arguments.title#">#HtmlEditFormat( arguments.title )#</a>';
375+
} else {
376+
request.logger (text="Missing docs link: [[#HtmlEditFormat( arguments.title )#]]", type="WARN");
377+
return '<a class="missing-link" title="missing link">#HtmlEditFormat( arguments.title )#</a>';
378+
}
379+
}
380+
var link = arguments.page.getPath() & ".html";
381+
if (!structKeyExists( args, "htmlOpts" ) ){
382+
SystemOutput(structKeyList(args), true);
383+
throw "zac";
384+
}
385+
//if ( arguments.page.getPath() contains "ormFlush" ) SystemOutput(args, true);
386+
if ( structKeyExists( args, "htmlOpts" )
387+
&& structKeyExists( args.htmlOpts, "base_href" ) ){
388+
link = args.htmlOpts.base_href & link;
389+
}
390+
391+
return '<a href="#link#">#HtmlEditFormat( arguments.title )#</a>';
392+
}
393+
394+
public string function _getIssueTrackerLink(required string name) {
395+
var link = Replace( new api.build.BuildProperties().getIssueTrackerLink(), "{search}", urlEncodedFormat(arguments.name) )
396+
return '<a href="#link#" class="no-oembed" target="_blank">Search Issue Tracker <i class="fa fa-external-link"></i></a>';
397+
}
398+
399+
public string function _getTestCasesLink(required string name) {
400+
var link = Replace( new api.build.BuildProperties().getTestCasesLink(), "{search}", urlEncodedFormat(arguments.name) )
401+
return '<a href="#link#" class="no-oembed" target="_blank">Search Lucee Test Cases <i class="fa fa-external-link"></i></a> (good for further, detailed examples)';
402+
}
403+
404+
public function _zipBasicPages( buildDirectory, simpleBuildDirectory, zipName ){
405+
406+
var zipFilename = arguments.zipName & ".zip";
407+
var doubleZipFilename = arguments.zipName & "-zipped.zip";
408+
409+
// neat trick, storing then zipping the stored zip reduces the file size from 496 Kb to 216 Kb
410+
var tempStoredZip = getTempFile( "", "#zipfileName#-store", "zip" );
411+
var tempDoubleZip = getTempFile( "", "#zipfileName#-normal", "zip" );
412+
var tempNormalZip = getTempFile( "", "#zipfileName#-normal", "zip" );
413+
414+
zip action="zip"
415+
source="#arguments.simpleBuildDirectory#"
416+
file="#tempStoredZip#"
417+
compressionmethod="store"
418+
recurse="true";
419+
420+
zip action="zip"
421+
source="#arguments.simpleBuildDirectory#"
422+
file="#tempDoubleZip#"
423+
compressionmethod="deflateUtra" // typo in cfzip!
424+
recurse="false" {
425+
zipparam entrypath="#zipFilename#" source="#tempStoredZip#";
426+
};
427+
fileDelete( tempStoredZip );
428+
429+
zip action="zip"
430+
source="#arguments.simpleBuildDirectory#"
431+
file="#tempNormalZip#"
432+
recurse="true";
433+
434+
publishWithChecksum( tempNormalZip, "#buildDirectory#/#zipFilename#" );
435+
publishWithChecksum( tempDoubleZip, "#buildDirectory#/#doubleZipFilename#" );
436+
};
437+
438+
function publishWithChecksum( src, dest ){
439+
request.logger (text="Builder copying zip to #dest#");
440+
fileCopy( src, dest );
441+
loop list="md5,sha1" item="local.hashType" {
442+
var checksumPath = left( dest, len( dest ) - 3 ) & hashType;
443+
filewrite( checksumPath, lcase( hash( fileReadBinary( arguments.src ), hashType ) ) );
444+
request.logger (text="Builder added #checksumPath# checksum");
445+
}
446+
}
447+
308448
}

‎builders/html/layouts/breadcrumbs.cfm‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@
33
<cfparam name="args.categories" type="array" />
44
<cfparam name="args.docTree" type="any" />
55
<cfparam name="args.page" type="any" />
6-
76
<cfif args.edit>
87
<cfset local.docs_base_url = "http://#cgi.http_host#">
98
<cfelse>
10-
<cfset local.docs_base_url = "https://docs.lucee.org">
9+
<cfif structKeyExists( args, "htmlOpts" )
10+
&& structKeyExists( args.htmlOpts, "base_href" )>
11+
<cfset local.docs_base_url = args.htmlOpts.base_href>
12+
<cfelse>
13+
<cfset local.docs_base_url = "https://docs.lucee.org">
14+
</cfif>
1115
</cfif>
1216

13-
1417
<cfif args.page.getId() neq "/home" and ArrayLen(args.crumbs)>
1518
<!--- pages may have multiple crumbs LD-112 --->
1619
<cfloop array="#args.crumbs#" item="local._crumbs">

0 commit comments

Comments
 (0)