Skip to content

Commit 9f5d901

Browse files
arar
committed
feat: structured JSONL logging with tags, PCI protection, and flat metadata
Add structured logging support for multi-node Elasticsearch/JSONL ingestion: - Unseal AuditLogEvent to allow jPOS-EE custom event types - Add tags (Map<String,String>) to LogEvent with fluent API (withTag/withTags/getTags) - Store traceId as a regular tag ("trace-id") instead of a dedicated field - Fold realm, trace-id, and host into the flat tags map for uniform ES indexing - Rename LogEvt output fields: tag→kind, evt→payload - New JsonlLogWriter (extends BaseLogEventWriter, implements Configurable): one JSON line per event with built-in PCI protection for ISOMsg fields (PAN masking via ISOUtil.protect, track/PIN wipe, configurable protect/wipe properties) - Populate pcode, tid, mid, mti tags in BaseChannel send/receive - Deprecate JsonLogWriter, XmlLogWriter, TxtLogWriter, MarkdownLogWriter
1 parent 6391b4c commit 9f5d901

File tree

14 files changed

+533
-30
lines changed

14 files changed

+533
-30
lines changed

‎jpos/src/dist/deploy/00_logger.xml‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,10 @@
33
<logger name="Q2" class="org.jpos.q2.qbean.LoggerAdaptor" realm="system">
44
<property name="redirect" value="stdout, stderr" />
55

6-
<log-listener class="org.jpos.util.SimpleLogListener" enabled="${log.simple:true}" />
7-
8-
<log-listener class="org.jpos.util.SimpleLogListener" enabled="${log.xml:false}">
9-
<writer class="org.jpos.util.XmlLogWriter" enabled="true" />
10-
</log-listener>
11-
12-
<log-listener class="org.jpos.util.SimpleLogListener" enabled="${log.json:false}">
13-
<writer class="org.jpos.util.JsonLogWriter" enabled="true" />
6+
<log-listener class="org.jpos.util.SimpleLogListener">
7+
<writer class="org.jpos.util.JsonlLogWriter" enabled="true" />
148
</log-listener>
159

16-
<log-listener class="org.jpos.util.SimpleLogListener" enabled="${log.markdown:false}">
17-
<writer class="org.jpos.util.MarkdownLogWriter" enabled="true" />
18-
</log-listener>
19-
20-
<!-- <log-listener class="org.jpos.util.SimpleLogListener" enabled="true">-->
21-
<!-- <writer class="org.jpos.util.TxtLogWriter" enabled="true" />-->
22-
<!-- </log-listener>-->
23-
2410
<log-listener class="org.jpos.util.DailyLogListener" enabled="true">
2511
<property name="window" value="86400" /> <!-- optional, default one day -->
2612
<!--needed-->

‎jpos/src/main/java/org/jpos/iso/BaseChannel.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,7 @@ public void send (ISOMsg m)
667667
m.setPackager (p);
668668
m = applyOutgoingFilters (m, evt);
669669
evt.addMessage (m);
670+
applyTags (evt, m);
670671
m.setDirection(ISOMsg.OUTGOING); // filter may have dropped this info
671672
m.setPackager (p); // and could have dropped packager as well
672673
byte[] b = pack(m);
@@ -848,6 +849,7 @@ else if (len > 0 && len <= getMaxPacketLength()) {
848849
unpack (m, b);
849850
m.setDirection(ISOMsg.INCOMING);
850851
evt.addMessage (m);
852+
applyTags (evt, m);
851853
m = applyIncomingFilters (m, header, b, evt);
852854
m.setDirection(ISOMsg.INCOMING);
853855
cnt[RX]++;
@@ -1262,4 +1264,20 @@ protected void incrementMsgOutCounter(ISOMsg m) throws ISOException {
12621264
isoMsgMetrics.recordMessage(m, MeterInfo.ISOMSG_OUT);
12631265
}
12641266
}
1267+
private void applyTags (LogEvent evt, ISOMsg m) {
1268+
if (m.hasField(3)) {
1269+
String f3 = m.getString(3);
1270+
if (f3 != null && f3.length() >= 2)
1271+
evt.withTag("pcode", f3.substring(0, 2));
1272+
}
1273+
if (m.hasField(41))
1274+
evt.withTag("tid", m.getString(41));
1275+
if (m.hasField(42))
1276+
evt.withTag("mid", m.getString(42));
1277+
try {
1278+
String mti = m.getMTI();
1279+
if (mti != null)
1280+
evt.withTag("mti", mti);
1281+
} catch (ISOException ignored) { }
1282+
}
12651283
}

‎jpos/src/main/java/org/jpos/log/AuditLogEvent.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@
4747
@JsonSubTypes.Type(value = Txn.class, name = "txn")
4848
})
4949

50-
public sealed interface AuditLogEvent permits Connect, Deploy, DeployActivity, Disconnect, License, Listen, LogMessage, SessionEnd, SessionStart, Shutdown, Start, Stop, SysInfo, ThrowableAuditLogEvent, Txn, UnDeploy, Warning { }
50+
public interface AuditLogEvent { }

‎jpos/src/main/java/org/jpos/log/evt/LogEvt.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,20 @@
1818

1919
package org.jpos.log.evt;
2020

21+
import com.fasterxml.jackson.annotation.JsonInclude;
2122
import com.fasterxml.jackson.annotation.JsonProperty;
2223
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
2324
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
2425
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
2526
import org.jpos.log.AuditLogEvent;
2627
import java.time.Instant;
2728
import java.util.List;
29+
import java.util.Map;
2830

2931
@JacksonXmlRootElement(localName = "log")
3032
public record LogEvt(
3133
@JacksonXmlProperty(isAttribute = true) Instant ts,
32-
@JacksonXmlProperty(isAttribute = true) @JsonProperty("trace-id") String traceId,
33-
@JacksonXmlProperty(isAttribute = true) String realm,
34-
@JacksonXmlProperty(isAttribute = true) String tag,
34+
@JacksonXmlProperty(isAttribute = true) String kind,
3535
@JacksonXmlProperty(isAttribute = true) Long lifespan,
36-
@JsonProperty("evt") @JacksonXmlElementWrapper(useWrapping = false) List<AuditLogEvent> events) { }
36+
@JsonInclude(JsonInclude.Include.NON_EMPTY) Map<String,String> tags,
37+
@JsonProperty("payload") @JacksonXmlElementWrapper(useWrapping = false) List<AuditLogEvent> events) { }

‎jpos/src/main/java/org/jpos/log/render/json/LogEventJsonLogRenderer.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@
3636
import java.io.ByteArrayOutputStream;
3737
import java.io.PrintStream;
3838
import java.time.Duration;
39+
import java.util.LinkedHashMap;
3940
import java.util.List;
41+
import java.util.Map;
4042

4143
public final class LogEventJsonLogRenderer implements LogRenderer<LogEvent> {
4244
private final ObjectMapper mapper = new ObjectMapper();
@@ -64,10 +66,9 @@ public void render(LogEvent evt, PrintStream ps, String indent) {
6466
long elapsed = Duration.between(evt.getCreatedAt(), evt.getDumpedAt()).toMillis();
6567
LogEvt ev = new LogEvt (
6668
evt.getDumpedAt(),
67-
evt.getTraceId(),
68-
evt.getRealm(),
6969
evt.getTag(),
7070
elapsed == 0L ? null : elapsed,
71+
buildTags(evt),
7172
events
7273
);
7374
try {
@@ -83,6 +84,16 @@ public Type type() {
8384
return Type.JSON;
8485
}
8586

87+
private Map<String,String> buildTags(LogEvent evt) {
88+
evt.getTraceId(); // ensure trace-id is generated
89+
Map<String,String> tags = new LinkedHashMap<>();
90+
String realm = evt.getRealm();
91+
if (realm != null && !realm.isEmpty())
92+
tags.put("realm", realm);
93+
tags.putAll(evt.getTags());
94+
return tags;
95+
}
96+
8697
private String dump (Object obj) {
8798
if (obj instanceof Loggeable loggeable) {
8899
ByteArrayOutputStream baos = new ByteArrayOutputStream();

‎jpos/src/main/java/org/jpos/log/render/xml/LogEventXmlLogRenderer.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@
3636
import java.io.ByteArrayOutputStream;
3737
import java.io.PrintStream;
3838
import java.time.Duration;
39+
import java.util.LinkedHashMap;
3940
import java.util.List;
41+
import java.util.Map;
4042

4143
public final class LogEventXmlLogRenderer implements LogRenderer<LogEvent> {
4244
private final XmlMapper mapper = new XmlMapper();
@@ -65,10 +67,9 @@ public void render(LogEvent evt, PrintStream ps, String indent) {
6567
long elapsed = Duration.between(evt.getCreatedAt(), evt.getDumpedAt()).toMillis();
6668
LogEvt ev = new LogEvt (
6769
evt.getDumpedAt(),
68-
evt.getTraceId(),
69-
evt.getRealm(),
7070
evt.getTag(),
7171
elapsed == 0L ? null : elapsed,
72+
buildTags(evt),
7273
events
7374
);
7475
try {
@@ -84,6 +85,16 @@ public Type type() {
8485
return Type.XML;
8586
}
8687

88+
private Map<String,String> buildTags(LogEvent evt) {
89+
evt.getTraceId(); // ensure trace-id is generated
90+
Map<String,String> tags = new LinkedHashMap<>();
91+
String realm = evt.getRealm();
92+
if (realm != null && !realm.isEmpty())
93+
tags.put("realm", realm);
94+
tags.putAll(evt.getTags());
95+
return tags;
96+
}
97+
8798
private String kv (String k, String v) {
8899
return "{\"%s\":\"%s\"}".formatted(k,v);
89100
}

‎jpos/src/main/java/org/jpos/util/JsonLogWriter.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.io.PrintStream;
2525

26+
@Deprecated(forRemoval = false)
2627
public class JsonLogWriter implements LogEventWriter {
2728
private PrintStream ps;
2829
private final LogRenderer<LogEvent> renderer = LogRendererRegistry.getRenderer(LogEvent.class, LogRenderer.Type.JSON);

‎jpos/src/main/java/org/jpos/util/JsonlLogWriter.java‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/*
2+
* jPOS Project [http://jpos.org]
3+
* Copyright (C) 2000-2026 jPOS Software SRL
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as
7+
* published by the Free Software Foundation, either version 3 of the
8+
* License, or (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
*/
18+
19+
package org.jpos.util;
20+
21+
import com.fasterxml.jackson.annotation.JsonInclude;
22+
import com.fasterxml.jackson.core.JsonProcessingException;
23+
import com.fasterxml.jackson.databind.ObjectMapper;
24+
import com.fasterxml.jackson.databind.module.SimpleModule;
25+
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
26+
import org.jpos.core.Configurable;
27+
import org.jpos.core.Configuration;
28+
import org.jpos.core.ConfigurationException;
29+
import org.jpos.iso.ISOMsg;
30+
import org.jpos.iso.ISOUtil;
31+
import org.jpos.log.AuditLogEvent;
32+
import org.jpos.log.evt.LogEvt;
33+
import org.jpos.log.evt.LogMessage;
34+
import org.jpos.log.evt.ThrowableAuditLogEvent;
35+
import org.jpos.log.render.ThrowableSerializer;
36+
37+
import java.io.ByteArrayOutputStream;
38+
import java.io.PrintStream;
39+
import java.net.InetAddress;
40+
import java.time.Duration;
41+
import java.util.*;
42+
43+
/**
44+
* JSONL (one JSON object per line) LogEventWriter with built-in PCI protection.
45+
*
46+
* <p>When serializing ISOMsg objects in LogEvent payloads, sensitive fields are
47+
* masked or wiped inline — no upstream {@link ProtectedLogListener} required.</p>
48+
*
49+
* <p>Configuration properties (same convention as {@link ProtectedLogListener}):</p>
50+
* <ul>
51+
* <li>{@code protect} — space-separated field numbers to mask via {@link ISOUtil#protect(String)} (default: {@code "2"})</li>
52+
* <li>{@code wipe} — space-separated field numbers to replace with [WIPED] (default: {@code "35 45 48 52 55"})</li>
53+
* </ul>
54+
*
55+
* <p>Output is suitable for {@code jq}, Filebeat, and Elasticsearch ingestion.</p>
56+
*
57+
* @since 3.0.0
58+
*/
59+
public class JsonlLogWriter extends BaseLogEventWriter implements Configurable {
60+
private static final String WIPED = "[WIPED]";
61+
private static final Set<Integer> DEFAULT_SAFE_FIELDS = Set.of(
62+
3, 4, 7, 11, 12, 13, 18, 22, 24, 25, 32, 37, 38, 39, 41, 42, 49, 90
63+
);
64+
65+
private ObjectMapper mapper;
66+
private String host;
67+
private Set<Integer> protectFields = Set.of(2);
68+
private Set<Integer> wipeFields = Set.of(35, 45, 48, 52, 55);
69+
70+
public JsonlLogWriter() {
71+
initMapper();
72+
try {
73+
host = InetAddress.getLocalHost().getHostName();
74+
} catch (Exception e) {
75+
host = "unknown";
76+
}
77+
}
78+
79+
@Override
80+
public void setConfiguration(Configuration cfg) throws ConfigurationException {
81+
String protect = cfg.get("protect", null);
82+
if (protect != null) {
83+
protectFields = toIntSet(protect);
84+
}
85+
String wipe = cfg.get("wipe", null);
86+
if (wipe != null) {
87+
wipeFields = toIntSet(wipe);
88+
}
89+
initMapper();
90+
}
91+
92+
@Override
93+
public void write(LogEvent ev) {
94+
if (p == null || ev == null)
95+
return;
96+
try {
97+
List<AuditLogEvent> events;
98+
synchronized (ev.getPayLoad()) {
99+
events = ev.getPayLoad()
100+
.stream()
101+
.map(this::toAuditLogEvent)
102+
.toList();
103+
}
104+
long elapsed = Duration.between(ev.getCreatedAt(), ev.getDumpedAt()).toMillis();
105+
LogEvt logEvt = new LogEvt(
106+
ev.getDumpedAt(),
107+
ev.getTag(),
108+
elapsed == 0L ? null : elapsed,
109+
buildTags(ev),
110+
events
111+
);
112+
p.println(mapper.writeValueAsString(logEvt));
113+
p.flush();
114+
} catch (JsonProcessingException e) {
115+
p.println("{\"error\":\"" + e.getMessage().replace("\"", "'") + "\"}");
116+
p.flush();
117+
}
118+
}
119+
120+
private AuditLogEvent toAuditLogEvent(Object obj) {
121+
return switch (obj) {
122+
case AuditLogEvent ale -> ale;
123+
case ISOMsg m -> new LogMessage(protectAndDump(m));
124+
case Throwable t -> new ThrowableAuditLogEvent(t);
125+
default -> new LogMessage(dump(obj));
126+
};
127+
}
128+
129+
private Map<String,String> buildTags(LogEvent ev) {
130+
ev.getTraceId(); // ensure trace-id is generated
131+
Map<String,String> tags = new LinkedHashMap<>();
132+
String realm = ev.getRealm();
133+
if (realm != null && !realm.isEmpty())
134+
tags.put("realm", realm);
135+
if (host != null)
136+
tags.put("host", host);
137+
tags.putAll(ev.getTags());
138+
return tags;
139+
}
140+
141+
private String protectAndDump(ISOMsg original) {
142+
ISOMsg m = (ISOMsg) original.clone();
143+
for (int field : protectFields) {
144+
String v = m.getString(field);
145+
if (v != null) {
146+
m.set(field, ISOUtil.protect(v));
147+
}
148+
}
149+
for (int field : wipeFields) {
150+
if (m.hasField(field)) {
151+
m.set(field, WIPED);
152+
}
153+
}
154+
return dump(m);
155+
}
156+
157+
private String dump(Object obj) {
158+
if (obj instanceof Loggeable loggeable) {
159+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
160+
PrintStream ps = new PrintStream(baos);
161+
loggeable.dump(ps, "");
162+
return baos.toString().trim();
163+
}
164+
return obj.toString();
165+
}
166+
167+
private void initMapper() {
168+
mapper = new ObjectMapper();
169+
mapper.registerModule(new JavaTimeModule());
170+
mapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
171+
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
172+
SimpleModule module = new SimpleModule();
173+
module.addSerializer(Throwable.class, new ThrowableSerializer());
174+
mapper.registerModule(module);
175+
}
176+
177+
private static Set<Integer> toIntSet(String spaceSeparated) {
178+
if (spaceSeparated == null || spaceSeparated.isBlank())
179+
return Set.of();
180+
Set<Integer> result = new HashSet<>();
181+
for (String token : spaceSeparated.trim().split("\\s+")) {
182+
result.add(Integer.parseInt(token));
183+
}
184+
return Collections.unmodifiableSet(result);
185+
}
186+
}

0 commit comments

Comments
 (0)