Skip to content

#3171 Solved the problem of systematic data archiving #3173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Patrykb0802 wants to merge 7 commits into
base: release/2.8.0
Choose a base branch
from
Open

#3171 Solved the problem of systematic data archiving #3173

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6b40ab5
#3171 Solved the problem of systematic data archiving
Patrykb0802 Jul 18, 2025
ed6caea
#3171 Solved the problem of systematic data archiving
Patrykb0802 Jul 30, 2025
dc37796
#3171 Solved the problem of systematic data archiving
Patrykb0802 Jul 31, 2025
28dbf80
#3171 Solved the problem of systematic data archiving
Patrykb0802 Aug 4, 2025
520524d
#3171 Solved the problem of systematic data archiving
Patrykb0802 Aug 25, 2025
9d35f3d
#3171 Solved the problem of systematic data archiving
Patrykb0802 Sep 1, 2025
997c788
#3171 Solved the problem of systematic data archiving
Patrykb0802 Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions WebContent/WEB-INF/applicationContext.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
<bean id="watchListDAO" class="org.scada_lts.dao.watchlist.WatchListDAO"/>
<bean id="dataSourceDAO" class="org.scada_lts.dao.DataSourceDAO"/>
<bean id="userDAO" class="org.scada_lts.dao.UserDAO"/>
<bean id="systemSettingsDAO" class="org.scada_lts.dao.SystemSettingsDAO"/>

<bean id="watchListGetShareUsers" class="org.scada_lts.permissions.service.WatchListGetShareUsers">
<constructor-arg ref="watchListDAO"/>
Expand Down Expand Up @@ -358,6 +359,25 @@
<constructor-arg ref="updatePendingEventsJobDetail"/>
</bean>

<!-- Quartz Archiver -->

<bean id="archivingTrigger" class="org.quartz.CronTrigger">
<constructor-arg value="Quartz - trigger-Archiving"/>
<constructor-arg value="DEFAULT"/>
</bean>

<bean id="archivingJobDetail" class="org.quartz.JobDetail">
<constructor-arg value="Quartz - job-Archiving"/>
<constructor-arg value="DEFAULT"/>
<constructor-arg value="org.scada_lts.archiving.ArchiverJob" type="java.lang.Class"/>
</bean>

<bean id="archivingScheduler" class="org.scada_lts.quartz.CronTriggerScheduler" destroy-method="stop">
<constructor-arg ref="scadaScheduler"/>
<constructor-arg ref="archivingTrigger"/>
<constructor-arg ref="archivingJobDetail"/>
</bean>

<bean id="opcUaTrustListManager" class="org.scada_lts.ds.polling.protocol.opcua.client.impl.OpcUaTrustListManager"
destroy-method="close" lazy-init="true"/>
</beans>
6 changes: 6 additions & 0 deletions WebContent/WEB-INF/dwr.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,12 @@
<convert converter="exception" match="java.lang.Exception">
<param name="include" value="message"/>
</convert>

<!-- Archiving config -->
<convert converter="bean" match="org.scada_lts.archiving.ArchivalConfig"/>
<convert converter="bean" match="org.scada_lts.archiving.ArchivalTask"/>
<convert converter="enum" match="org.scada_lts.archiving.ArchivalFunction"/>
<convert converter="enum" match="java.time.temporal.ChronoUnit"/>
</allow>
<signatures>
<![CDATA[
Expand Down
177 changes: 176 additions & 1 deletion WebContent/WEB-INF/jsp/systemSettings.jsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,22 @@
$set("<c:out value="<%= SystemSettingsDAO.AGGREGATION_VALUES_LIMIT %>"/>", settings.<c:out value="<%= SystemSettingsDAO.AGGREGATION_VALUES_LIMIT %>"/>);
$set("<c:out value="<%= SystemSettingsDAO.AGGREGATION_LIMIT_FACTOR %>"/>", settings.<c:out value="<%= SystemSettingsDAO.AGGREGATION_LIMIT_FACTOR %>"/>);
$set("<c:out value="<%= SystemSettingsDAO.VALUES_LIMIT_FOR_PURGE %>"/>", settings.<c:out value="<%= SystemSettingsDAO.VALUES_LIMIT_FOR_PURGE %>"/>);

$set("<c:out value='<%= SystemSettingsDAO.ARCHIVE_ENABLED %>'/>", settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_ENABLED %>"/>);

$set("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL %>'/>", settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL %>"/>);
$set("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_USERNAME %>'/>", settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL_USERNAME %>"/>);
$set("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_PASSWORD %>'/>", settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL_PASSWORD %>"/>);
$set("<c:out value='<%= SystemSettingsDAO.BATCH_SIZE %>'/>", settings.<c:out value="<%= SystemSettingsDAO.BATCH_SIZE %>"/>);
$set("<c:out value='<%= SystemSettingsDAO.ARCHIVE_CRON %>'/>", settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_CRON %>"/>);
loadArchivingRules(settings.<c:out value="<%= SystemSettingsDAO.ARCHIVING_CONFIG %>"/>)

var archivingEnabled = !!settings.<c:out value="<%= SystemSettingsDAO.ARCHIVE_ENABLED %>"/>;
toggleArchiveFields(archivingEnabled);

document.getElementById("<c:out value='<%= SystemSettingsDAO.ARCHIVE_ENABLED %>'/>").onchange = function() {
toggleArchiveFields(this.checked);
};
});

<%--
Expand Down Expand Up @@ -663,6 +679,74 @@
initSizeField(webGraphicsUploadsPath);
initSizeField(smsDomain);
});

function saveDataArchivingSettings() {
const rules = getArchivingRulesFromTable();
const configJson = JSON.stringify({ tasks: rules });

var archiveEnabled = $get("<c:out value='<%= SystemSettingsDAO.ARCHIVE_ENABLED %>'/>");
var archiveDbUrl = $get("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL %>'/>");
var archiveDbUrlUsername = $get("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_USERNAME %>'/>");
var archiveDbUrlPassword = $get("<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_PASSWORD %>'/>");
var batchSize = $get("<c:out value='<%= SystemSettingsDAO.BATCH_SIZE %>'/>");
var archiveCron = $get("<c:out value='<%= SystemSettingsDAO.ARCHIVE_CRON %>'/>");

SystemSettingsDwr.saveArchivingConfig(
archiveEnabled,
archiveDbUrl,
archiveDbUrlUsername,
archiveDbUrlPassword,
batchSize,
archiveCron,
configJson,
function(response) {
stopImageFader("saveDataArchivingSettingsImg");
if (response.hasMessages) {
setUserMessage("dataArchivingMessage", response.messages);
} else {
setUserMessage("dataArchivingMessage", "<spring:message code='systemSettings.archiving.settingsSuccess'/>");
}
}
);
}

function toggleArchiveFields(enabled) {
var fields = [
"<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL %>'/>",
"<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_USERNAME %>'/>",
"<c:out value='<%= SystemSettingsDAO.ARCHIVE_DB_URL_PASSWORD %>'/>",
"<c:out value='<%= SystemSettingsDAO.BATCH_SIZE %>'/>",
"<c:out value='<%= SystemSettingsDAO.ARCHIVE_CRON %>'/>",
];
for (var i = 0; i < fields.length; i++) {
var el = document.getElementById(fields[i]);
if (el) el.disabled = !enabled;
}

const buttons = [
{ id: "addArchivingRuleImg", handler: addArchivingRuleRow },
{ id: "removeArchivingRuleRowImg", handler: removeArchivingRuleRow }
];

buttons.forEach(({ id, handler }) => {
const el = document.getElementById(id);
if (!el) return;

el.style.pointerEvents = enabled ? "" : "none";
el.style.opacity = enabled ? "" : "0.5";
el.onclick = enabled ? handler : null;
});
}

function loadArchivingRules(json) {
let rules = [];
try {
if (json) rules = JSON.parse(json).tasks || [];
} catch (e) {
console.warn("[ARCHIVER]Invalid archiving config JSON:", e);
}
renderArchivingRules(rules);
}
</script>

<div class="borderDivPadded marB marR" style="float:left">
Expand Down Expand Up @@ -1139,6 +1223,97 @@
</table>
</div>

<div class="borderDivPadded marB marR" style="float:left">
<table width="100%">
<tr>
<td>
<span class="smallTitle"><spring:message code="systemSettings.dataArchivingSettings"/></span>
<tag:help id="dataArchivingSettings"/>
</td>
<td align="right">
<tag:img id="saveDataArchivingSettingsImg" png="save" onclick="saveDataArchivingSettings();" title="common.save"/>
</td>
</tr>
</table>
<table>
<tr>
<td class="formLabelRequired">
<spring:message code="systemSettings.archiveEnabled"/>
</td>
<td class="formField">
<input
type="checkbox"
id="<c:out value='<%= SystemSettingsDAO.ARCHIVE_ENABLED %>'/>"
onchange="toggleArchiveFields(this.checked)"
/>
</td>
</tr>
<tr>
<td class="formLabelRequired"><spring:message code="systemSettings.archiveDbUrl"/></td>
<td class="formField">
<input id="<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL %>"/>" type="text" class="formWide"/>
</td>
</tr>
<tr>
<td class="formLabelRequired"><spring:message code="systemSettings.archiveDbUrlUsername"/></td>
<td class="formField">
<input id="<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL_USERNAME %>"/>" type="text" class="formWide"/>
</td>
</tr>
<tr>
<td class="formLabelRequired"><spring:message code="systemSettings.archiveDbUrlPassword"/></td>
<td class="formField">
<input id="<c:out value="<%= SystemSettingsDAO.ARCHIVE_DB_URL_PASSWORD %>"/>" type="text" class="formWide"/>
</td>
</tr>
<tr>
<td class="formLabelRequired"><spring:message code="systemSettings.batchSize"/></td>
<td class="formField">
<input id="<c:out value="<%= SystemSettingsDAO.BATCH_SIZE %>"/>" type="number" min="1" class="formMedium"/>
</td>
</tr>
<tr>
<td class="formLabelRequired"><spring:message code="systemSettings.archiveFrequency"/></td>
<td class="formField">
<input id="<c:out value="<%= SystemSettingsDAO.ARCHIVE_CRON %>"/>" type="text" class="formWide"/>
</td>
</tr>
<tr>
<td class="formLabelRequired">
<spring:message code="systemSettings.archiving.rules"/>
</td>
<td class="formField">
<table id="archivingRulesTable" style="border-collapse: collapse;">
<thead>
<tr>
<th><spring:message code="systemSettings.archiving.ageValue"/></th>
<th><spring:message code="systemSettings.archiving.ageUnit"/></th>
<th><spring:message code="systemSettings.archiving.action"/></th>
<th><spring:message code="systemSettings.archiving.table"/></th>
<th></th>
</tr>
</thead>
<tbody id="archivingRulesTbody">
<!-- Rendered by JS -->
</tbody>
<tfoot>
<tr>
<td colspan="5" align="right">
<img id="addArchivingRuleImg" src="images/add.png" title="Add" onclick="addArchivingRuleRow()"border="0"/>
</td>
</tr>
</tfoot>
</table>
<input type="hidden" id="archivingRulesJson" value="" />
</td>
</tr>
<tr>
<td colspan="2" id="dataArchivingMessage" class="formError"></td>
</tr>
</table>
</div>


<!-- amCharts Settings -->
<div class="borderDivPadded marB marR" style="float:left">
<table width="100%">
Expand Down Expand Up @@ -1384,4 +1559,4 @@


</tag:page>
<tag:newPageNotification href="./app.shtm#/system-settings" ref="systemSettingsNotification"/>
<tag:newPageNotification href="./app.shtm#/system-settings" ref="systemSettingsNotification"/>
90 changes: 90 additions & 0 deletions WebContent/resources/common.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1736,4 +1736,94 @@ function isSupported(browser, minVersion) {
return major >= minVersion;
}
return false;
}

const archivingUnits = [
//{ value: "MINUTES", labelKey: "MINUTES"}, //TO REMOVE!! ONLY FOR TESTING
{ value: "HOURS", labelKey: "HOURS" },
{ value: "DAYS", labelKey: "DAYS" },
{ value: "WEEKS", labelKey: "WEEKS" },
{ value: "MONTHS", labelKey: "MONTHS"},
{ value: "YEARS", labelKey: "YEARS" }
];
const archivingActions = [
{ value: "COPY_TO_ARCHIVE", labelKey: "Archive" },
{ value: "DELETE_IF_IN_ARCHIVE", labelKey: "Delete" }
];
const archivingTables = [
{ value: "pointValues", labelKey: "Point values" },
{ value: "events", labelKey: "events" }
];

function renderArchivingRules(rules) {
const tbody = document.getElementById('archivingRulesTbody');
tbody.innerHTML = '';
(rules || []).forEach((rule, i) => {
tbody.appendChild(archivingRuleRow(i, rule));
});
}

function archivingRuleRow(idx, rule) {
const tr = document.createElement('tr');
tr.appendChild(tdInput('number', 'ageValue', rule.ageValue || 30, 'min="1" class="formShort"'));
tr.appendChild(tdSelect('ageUnit', archivingUnits, rule.ageUnit));
tr.appendChild(tdSelect('action', archivingActions, rule.action));
tr.appendChild(tdSelect('table', archivingTables, rule.table));
const tdDel = document.createElement('td');
tdDel.innerHTML = `<img id="removeArchivingRuleRowImg" src="images/delete.png" alt="Remove" title="Remove" onclick="removeArchivingRuleRow(${idx})"border="0"/>`;
tr.appendChild(tdDel);
return tr;
}

function tdInput(type, name, value, extra = "") {
const td = document.createElement('td');
td.innerHTML = `<input type="${type}" name="${name}" value="${value}" ${extra}/>`;
return td;
}

function tdSelect(name, options, selected) {
const td = document.createElement('td');
let html = `<select name="${name}">`;
options.forEach(opt =>
html += `<option value="${opt.value}" ${opt.value === selected ? 'selected' : ''}>${opt.labelKey}</option>`
);
html += `</select>`;
td.innerHTML = html;
return td;
}

// Add rule row
function addArchivingRuleRow() {
const rules = getArchivingRulesFromTable();
rules.push({ageValue: 30, ageUnit: "DAYS", action: "ARCHIVE", table: "pointValues"});
renderArchivingRules(rules);
}

// Remove rule row
function removeArchivingRuleRow(idx) {
const rules = getArchivingRulesFromTable();
rules.splice(idx, 1);
renderArchivingRules(rules);
}

// Get all rules from table
function getArchivingRulesFromTable() {
const rules = [];
const rows = document.querySelectorAll("#archivingRulesTbody tr");

rows.forEach(row => {
const ageValue = parseInt(row.querySelector('input[name="ageValue"]').value);
const ageUnit = row.querySelector('select[name="ageUnit"]').value;
const func = row.querySelector('select[name="action"]').value;
const table = row.querySelector('select[name="table"]').value;

rules.push({
ageValue,
ageUnit,
function: func,
table
});
});

return rules;
}
8 changes: 8 additions & 0 deletions src/com/serotonin/mango/MangoContextListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.mozilla.javascript.ContextFactory;
import org.scada_lts.archiving.ArchiveUtils;
import org.scada_lts.cache.DataSourcePointsCache;
import org.scada_lts.cache.PointHierarchyCache;
import org.scada_lts.cache.ViewHierarchyCache;
Expand Down Expand Up @@ -708,6 +709,13 @@ private void initSchedule() {
} catch (Exception e) {
log.error(e.getMessage(), e);
}

try {
ArchiveUtils.init();
log.info("Quartz ArchiverJob initialized");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}

private void sessionsInitialize(ServletContextEvent evt) {
Expand Down
Loading
Loading