Skip to content

Commit fc538bf

Browse files
jeclrsgjeclrsg
committed
HPCC-34914 add WsWorkunits/WUHelperFileArchive
adds the WUHelperFileArchive endpoint to WsWorkunits, enabling ESP to generate a .gz or .zip archive containing the specified helper files Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com>
1 parent 6f00503 commit fc538bf

File tree

6 files changed

+119
-14
lines changed

6 files changed

+119
-14
lines changed

‎esp/scm/ws_workunits.ecm‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp);
2525

2626
ESPservice [
2727
auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
28-
version("2.05"), default_client_version("2.05"), cache_group("ESPWsWUs"),
28+
version("2.06"), default_client_version("2.06"), cache_group("ESPWsWUs"),
2929
noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
3030
{
3131
ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")] WUQuery(WUQueryRequest, WUQueryResponse);
@@ -116,6 +116,7 @@ ESPservice [
116116
ESPmethod [cache_seconds(60), min_ver("1.74")] WUGetThorJobList(WUGetThorJobListRequest, WUGetThorJobListResponse);
117117
ESPmethod [cache_seconds(60), min_ver("1.79")] WUGetPlugins(WUGetPluginsRequest, WUGetPluginsResponse);
118118
ESPmethod [min_ver("1.94")] WUAnalyseHotspot(WUAnalyseHotspotRequest, WUAnalyseHotspotResponse);
119+
ESPmethod [min_ver("2.06")] WUHelperFileArchive(WUHelperFileArchiveRequest, WUHelperFileArchiveResponse);
119120
};
120121

121122

‎esp/scm/ws_workunits_req_resp.ecm‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,3 +1145,18 @@ ESPresponse [exceptions_inline] WUAnalyseHotspotResponse
11451145
};
11461146

11471147
// ----------------------------------------------------------------------------------
1148+
1149+
ESPrequest [nil_remove] WUHelperFileArchiveRequest
1150+
{
1151+
string Wuid;
1152+
ESPenum WUFileDownloadOption DownloadOption(2); // defaults to ZIP
1153+
ESParray<ESPstruct WUFileOption> WUFileOptions;
1154+
};
1155+
1156+
ESPresponse [exceptions_inline] WUHelperFileArchiveResponse
1157+
{
1158+
[http_content("application/octet-stream")] binary thefile;
1159+
string FileName;
1160+
};
1161+
1162+
// ----------------------------------------------------------------------------------

‎esp/services/ws_workunits/ws_workunitsHelpers.cpp‎

Lines changed: 62 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetc
242242
StringBuffer logcontent;
243243
unsigned totalRecsRead = 0;
244244
unsigned recsRead = 0;
245-
245+
246246
if (logFormat == LOGACCESS_LOGFORMAT_json)
247247
writeStringToStream(*outIOS, "{\"lines\": [");
248248
else if (logFormat == LOGACCESS_LOGFORMAT_xml)
@@ -273,19 +273,19 @@ void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetc
273273
if (logFormat == LOGACCESS_LOGFORMAT_json)
274274
{
275275
VStringBuffer jsonMessage("], \"Message\": \"%s\"}", truncationWarnmessage.str()); //close lines array, append Message object
276-
writeStringToStream(*outIOS, jsonMessage.str());
276+
writeStringToStream(*outIOS, jsonMessage.str());
277277
}
278278
else if (logFormat == LOGACCESS_LOGFORMAT_xml)
279279
{
280280
VStringBuffer xmlMessage("</lines>\n<!-- %s -->", truncationWarnmessage.str()); //close lines element, append comment message
281-
writeStringToStream(*outIOS, xmlMessage.str());
281+
writeStringToStream(*outIOS, xmlMessage.str());
282282
}
283283
else if (logFormat == LOGACCESS_LOGFORMAT_csv)
284284
{
285-
writeStringToStream(*outIOS, truncationWarnmessage);
285+
writeStringToStream(*outIOS, truncationWarnmessage);
286286
}
287287
}
288-
else //close the lines container
288+
else //close the lines container
289289
{
290290
if (logFormat == LOGACCESS_LOGFORMAT_json)
291291
writeStringToStream(*outIOS, "]}");
@@ -2546,7 +2546,7 @@ void WsWuInfo::readWorkunitThorLog(const char* processName, const char* log, con
25462546
{
25472547
StringBuffer thorMasterLog, ext;
25482548
splitFilename(logSpec, nullptr, nullptr, &thorMasterLog, &ext);
2549-
2549+
25502550
StringBuffer logSpecStr(log);
25512551
addPathSepChar(logSpecStr);
25522552
//Append the file name of the slave log to logSpecStr.
@@ -2569,7 +2569,7 @@ void WsWuInfo::readWorkunitThorLog(const char* processName, const char* log, con
25692569
if (!rFile)
25702570
throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE, "Cannot open file %s.", logSpec);
25712571
}
2572-
2572+
25732573
readWorkunitThorLogOneDay(rFile, processID, buf, outIOS);
25742574
}
25752575
}
@@ -2603,7 +2603,7 @@ void WsWuInfo::readWorkunitThorLogOneDay(IFile* sourceFile, unsigned& processID,
26032603
bool outputThisLine = false;
26042604
if (processID > 0) //after the 1st page of the log
26052605
outputThisLine = true;
2606-
bool foundEndWUID = false;
2606+
bool foundEndWUID = false;
26072607
while (!eof)
26082608
{
26092609
if (outputThisLine)
@@ -2666,7 +2666,7 @@ void WsWuInfo::getWUProcessLogSpecs(const char* processName, const char* logSpec
26662666
//Parse the process name from the logSpec or logDir.
26672667
if (isEmptyString(logSpec) && isEmptyString(logDir))
26682668
throw makeStringException(ECLWATCH_ECLAGENT_LOG_NOT_FOUND, "Process name and log file not specified");
2669-
2669+
26702670
StringBuffer path, process;
26712671
if (!isEmptyString(logDir))
26722672
path.set(logDir);
@@ -5143,6 +5143,59 @@ void CWsWuFileHelper::readLocalFileToBuffer(const char* file, offset_t sizeLimit
51435143
throw MakeStringException(ECLWATCH_CANNOT_READ_FILE, "Cannot read %s.", file);
51445144
}
51455145

5146+
void CWsWuFileHelper::createHelperFileArchive(IEspContext& context, const char* wuid,
5147+
IArrayOf<IConstWUFileOption>& wuFileOptions, bool useGZip, MemoryBuffer& result, StringBuffer& archiveFileName)
5148+
{
5149+
StringBuffer namePrefixStr, workingFolder;
5150+
Owned<IFile> workingDir = createWorkingFolder(context, wuid, "WUHelpers_", namePrefixStr, workingFolder);
5151+
5152+
WsWuInfo winfo(context, wuid);
5153+
unsigned filesAdded = 0;
5154+
ForEachItemIn(i, wuFileOptions)
5155+
{
5156+
StringBuffer fileName, fileMimeType;
5157+
try
5158+
{
5159+
readWUFile(wuid, workingFolder.str(), winfo, wuFileOptions.item(i), fileName, fileMimeType);
5160+
filesAdded++;
5161+
}
5162+
catch (IException* e)
5163+
{
5164+
StringBuffer msg;
5165+
WARNLOG("Failed to read helper file for workunit %s: %s", wuid, e->errorMessage(msg).str());
5166+
e->Release();
5167+
}
5168+
}
5169+
5170+
if (filesAdded == 0)
5171+
{
5172+
cleanFolder(workingDir, true);
5173+
throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "No helper files could be retrieved for workunit %s.", wuid);
5174+
}
5175+
5176+
archiveFileName.set(namePrefixStr).append(useGZip ? ".gz" : ".zip");
5177+
StringBuffer zipFileNameWithPath(zipFolder);
5178+
zipFileNameWithPath.append(archiveFileName.str());
5179+
{
5180+
OwnedIFile oldFile = createIFile(zipFileNameWithPath.str());
5181+
if (oldFile->isFile() == fileBool::foundYes)
5182+
oldFile->remove();
5183+
}
5184+
int zipRet = zipAFolder(workingFolder.str(), useGZip, zipFileNameWithPath.str());
5185+
cleanFolder(workingDir, true);
5186+
5187+
if (zipRet != 0)
5188+
throw makeStringExceptionV(ECLWATCH_CANNOT_COMPRESS_DATA, "Failed to create helper file archive for workunit %s.", wuid);
5189+
5190+
Owned<IFile> zipFile = createIFile(zipFileNameWithPath.str());
5191+
Owned<IFileIO> zipIO = zipFile->open(IFOread);
5192+
size32_t zipSize = (size32_t) zipIO->size();
5193+
void* data = result.reserve(zipSize);
5194+
result.setLength(zipIO->read(0, zipSize, data));
5195+
zipIO->close();
5196+
zipFile->remove();
5197+
}
5198+
51465199
void CWsWuEmailHelper::send(const char* body, const void* attachment, size32_t lenAttachment, StringArray& warnings)
51475200
{
51485201
if (lenAttachment == 0)

‎esp/services/ws_workunits/ws_workunitsHelpers.hpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ struct WUComponentLogOptions
225225
{
226226
StringBuffer customFieldsList;
227227
zapHttpRequest->getParameter("LogFilter_CustomColumns", customFieldsList);
228-
228+
229229
if(!customFieldsList.isEmpty())
230230
customFields.appendList(customFieldsList.str(), ",");
231231
}
@@ -354,7 +354,7 @@ struct WUComponentLogOptions
354354

355355
logFetchOptions.setFilter(logFetchFilter.getClear());
356356
CSortDirection espSortDirection = logFilterReq.getSortByTimeDirection();
357-
logFetchOptions.addSortByCondition(LOGACCESS_MAPPEDFIELD_timestamp, "", espSortDirection == CSortDirection_ASC
357+
logFetchOptions.addSortByCondition(LOGACCESS_MAPPEDFIELD_timestamp, "", espSortDirection == CSortDirection_ASC
358358
? SORTBY_DIRECTION_ascending : SORTBY_DIRECTION_descending);
359359
}
360360
};
@@ -917,6 +917,9 @@ class CWsWuFileHelper
917917
IFileIOStream* createWUFileIOStream(IEspContext &context, const char *wuid, IArrayOf<IConstWUFileOption> &wuFileOptions,
918918
CWUFileDownloadOption &downloadOptions, StringBuffer &contentType);
919919

920+
void createHelperFileArchive(IEspContext &context, const char *wuid, IArrayOf<IConstWUFileOption> &wuFileOptions,
921+
bool useGZip, MemoryBuffer &result, StringBuffer &archiveFileName);
922+
920923
void validateFilePath(const char *file, WsWuInfo &winfo, CWUFileType wuFileType, bool UNCFileName, const char *fileType, const char *compType, const char *compName);
921924
bool validateWUFile(const char *file, WsWuInfo &winfo, CWUFileType wuFileType);
922925
void readLocalFileToBuffer(const char *file, offset_t sizeLimit, MemoryBuffer &mb);

‎esp/services/ws_workunits/ws_workunitsService.cpp‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4273,6 +4273,38 @@ bool CWsWorkunitsEx::onWUAnalyseHotspot(IEspContext &context, IEspWUAnalyseHotsp
42734273
return true;
42744274
}
42754275

4276+
bool CWsWorkunitsEx::onWUHelperFileArchive(IEspContext &context, IEspWUHelperFileArchiveRequest &req, IEspWUHelperFileArchiveResponse &resp)
4277+
{
4278+
try
4279+
{
4280+
StringBuffer wuid(req.getWuid());
4281+
WsWuHelpers::checkAndTrimWorkunit("WUHelperFileArchive", wuid);
4282+
ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
4283+
PROGLOG("WUHelperFileArchive: %s", wuid.str());
4284+
4285+
IArrayOf<IConstWUFileOption>& wuFileOptions = req.getWUFileOptions();
4286+
if (!wuFileOptions.ordinality())
4287+
throw MakeStringException(ECLWATCH_INVALID_INPUT, "No file options specified.");
4288+
4289+
bool useGZip = (req.getDownloadOption() == CWUFileDownloadOption_GZIP);
4290+
MemoryBuffer mb;
4291+
StringBuffer archiveFileName;
4292+
CWsWuFileHelper helper(directories);
4293+
helper.createHelperFileArchive(context, wuid.str(), wuFileOptions, useGZip, mb, archiveFileName);
4294+
4295+
resp.setThefile(mb);
4296+
resp.setThefile_mimetype(HTTP_TYPE_OCTET_STREAM);
4297+
resp.setFileName(archiveFileName.str());
4298+
VStringBuffer headerStr("attachment;filename=%s", archiveFileName.str());
4299+
context.addCustomerHeader("Content-disposition", headerStr.str());
4300+
}
4301+
catch(IException* e)
4302+
{
4303+
FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
4304+
}
4305+
return true;
4306+
}
4307+
42764308

42774309
static void getWUDetailsMetaProperties(double version, IArrayOf<IEspWUDetailsMetaProperty> & properties)
42784310
{
@@ -4630,7 +4662,7 @@ int CWsWorkunitsSoapBindingEx::onStartUpload(IEspContext &ctx, CHttpRequest* req
46304662
getUserWuAccessFlags(ctx, accessOwn, accessOthers, false);
46314663
if ((accessOwn != SecAccess_Full) || (accessOthers != SecAccess_Full))
46324664
throw makeStringExceptionV(-1, "Resources %s and/or %s : Permission denied. Full Access Required.", OWN_WU_ACCESS, OTHERS_WU_ACCESS);
4633-
4665+
46344666
StringBuffer password;
46354667
request->getParameter("Password", password);
46364668

@@ -4982,7 +5014,7 @@ bool CWsWorkunitsEx::onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfo
49825014
Owned<IFile> tempDir = createUniqueTempDirectory();
49835015

49845016
StringBuffer zipFileName, zipFileNameWithPath;
4985-
//CWsWuFileHelper may need ESP's <Directories> settings to locate log files.
5017+
//CWsWuFileHelper may need ESP's <Directories> settings to locate log files.
49865018
CWsWuFileHelper helper(directories);
49875019
helper.createWUZAPFile(context, cwu, zapInfoReq, tempDir->queryFilename(), zipFileName, zipFileNameWithPath, thorSlaveLogThreadPoolSize);
49885020

‎esp/services/ws_workunits/ws_workunitsService.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,7 @@ class CWsWorkunitsEx : public CWsWorkunits
341341
bool onWUEclDefinitionAction(IEspContext &context, IEspWUEclDefinitionActionRequest &req, IEspWUEclDefinitionActionResponse &resp);
342342
bool onWUGetPlugins(IEspContext &context, IEspWUGetPluginsRequest &req, IEspWUGetPluginsResponse &resp);
343343
virtual bool onWUAnalyseHotspot(IEspContext &context, IEspWUAnalyseHotspotRequest &req, IEspWUAnalyseHotspotResponse &resp) override;
344+
bool onWUHelperFileArchive(IEspContext &context, IEspWUHelperFileArchiveRequest &req, IEspWUHelperFileArchiveResponse &resp);
344345

345346
bool unsubscribeServiceFromDali() override
346347
{

0 commit comments

Comments
 (0)