Skip to content

Commit 6e94185

Browse files
Merge pull request #48 from superstreamlabs/master
Release
2 parents c294521 + 48c8a67 commit 6e94185

File tree

10 files changed

+302
-32
lines changed

10 files changed

+302
-32
lines changed
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
name=local-file-source-superstream
2+
connector.class=org.apache.kafka.connect.file.FileStreamSourceConnector
3+
tasks.max=1
4+
5+
file=examples/kafka-connect-example/quickstart-file.txt
6+
topic=example-topic
7+
8+
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
bootstrap.servers=localhost:9092
2+
3+
key.converter=org.apache.kafka.connect.storage.StringConverter
4+
value.converter=org.apache.kafka.connect.storage.StringConverter
5+
key.converter.schemas.enable=false
6+
value.converter.schemas.enable=false
7+
8+
offset.storage.file.filename=/tmp/connect-offsets-superstream-example.dat
9+
offset.flush.interval.ms=10000
10+
11+
# Use the built-in file connector from the classpath (connect-file dependency)
12+
plugin.path=.
13+
14+
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<parent>
8+
<groupId>ai.superstream</groupId>
9+
<artifactId>superstream-clients-java</artifactId>
10+
<version>1.0.0</version>
11+
<relativePath>../../pom.xml</relativePath>
12+
</parent>
13+
14+
<groupId>ai.superstream.examples</groupId>
15+
<artifactId>kafka-connect-example</artifactId>
16+
<version>1.0.0</version>
17+
18+
<dependencies>
19+
<!-- Kafka Connect runtime -->
20+
<dependency>
21+
<groupId>org.apache.kafka</groupId>
22+
<artifactId>connect-runtime</artifactId>
23+
<version>${kafka.version}</version>
24+
</dependency>
25+
26+
<!-- Built-in File connector (FileStreamSourceConnector) -->
27+
<dependency>
28+
<groupId>org.apache.kafka</groupId>
29+
<artifactId>connect-file</artifactId>
30+
<version>${kafka.version}</version>
31+
</dependency>
32+
33+
<dependency>
34+
<groupId>org.slf4j</groupId>
35+
<artifactId>slf4j-api</artifactId>
36+
</dependency>
37+
</dependencies>
38+
39+
<build>
40+
<plugins>
41+
<plugin>
42+
<groupId>org.apache.maven.plugins</groupId>
43+
<artifactId>maven-compiler-plugin</artifactId>
44+
<version>3.10.1</version>
45+
<configuration>
46+
<source>11</source>
47+
<target>11</target>
48+
</configuration>
49+
</plugin>
50+
<plugin>
51+
<groupId>org.apache.maven.plugins</groupId>
52+
<artifactId>maven-assembly-plugin</artifactId>
53+
<version>3.4.2</version>
54+
<executions>
55+
<execution>
56+
<phase>package</phase>
57+
<goals>
58+
<goal>single</goal>
59+
</goals>
60+
<configuration>
61+
<archive>
62+
<manifest>
63+
<mainClass>ai.superstream.examples.connect.KafkaConnectStandaloneExample</mainClass>
64+
</manifest>
65+
</archive>
66+
<descriptorRefs>
67+
<descriptorRef>jar-with-dependencies</descriptorRef>
68+
</descriptorRefs>
69+
</configuration>
70+
</execution>
71+
</executions>
72+
</plugin>
73+
</plugins>
74+
</build>
75+
</project>
76+
77+
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Sample line 1
2+
Sample line 2
3+
Sample line 3
4+
5+
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package ai.superstream.examples.connect;
2+
3+
import org.apache.kafka.connect.cli.ConnectStandalone;
4+
import org.slf4j.Logger;
5+
import org.slf4j.LoggerFactory;
6+
7+
/**
8+
* Minimal Kafka Connect standalone example that runs a Connect worker
9+
* and a single connector in this JVM. When launched with the Superstream agent
10+
* (-javaagent), all Kafka producers created inside Connect will be routed
11+
* through the Superstream shadow layer.
12+
*
13+
* By default this uses the configuration files from the {@code config}
14+
* directory in this module:
15+
* - connect-standalone.properties
16+
* - connect-file-source.properties
17+
*
18+
* You can override the locations via:
19+
* - System property: connect.example.config.dir (base dir)
20+
* - Env: CONNECT_WORKER_CONFIG
21+
* - Env: CONNECT_CONNECTOR_CONFIG
22+
*/
23+
public class KafkaConnectStandaloneExample {
24+
25+
private static final Logger LOG = LoggerFactory.getLogger(KafkaConnectStandaloneExample.class);
26+
27+
public static void main(String[] args) throws Exception {
28+
String workerConfig;
29+
String connectorConfig;
30+
31+
if (args.length >= 2) {
32+
workerConfig = args[0];
33+
connectorConfig = args[1];
34+
} else {
35+
String baseDir = System.getProperty(
36+
"connect.example.config.dir",
37+
"examples/kafka-connect-example/config"
38+
);
39+
workerConfig = getenvOrDefault("CONNECT_WORKER_CONFIG", baseDir + "/connect-standalone.properties");
40+
connectorConfig = getenvOrDefault("CONNECT_CONNECTOR_CONFIG", baseDir + "/connect-file-source.properties");
41+
}
42+
43+
LOG.info("Starting Kafka Connect Standalone with:");
44+
LOG.info(" worker config = {}", workerConfig);
45+
LOG.info(" connector config= {}", connectorConfig);
46+
LOG.info(" (launch this JVM with -javaagent:...superstream-clients-2.0.0.jar to enable the Superstream agent)");
47+
48+
// Launch a Kafka Connect standalone worker in this JVM.
49+
ConnectStandalone.main(new String[] { workerConfig, connectorConfig });
50+
}
51+
52+
private static String getenvOrDefault(String key, String defaultValue) {
53+
String v = System.getenv(key);
54+
return (v == null || v.isEmpty()) ? defaultValue : v;
55+
}
56+
}
57+
58+
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
log4j.rootLogger=INFO, stdout
2+
3+
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
4+
log4j.appender.stdout.Target=System.out
5+
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
6+
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c - %m%n
7+
8+

pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
<module>examples/akka-kafka-example</module>
1616
<module>examples/spring-kafka-example</module>
1717
<module>examples/pekko-kafka-example</module>
18+
<module>examples/kafka-connect-example</module>
1819
</modules>
1920

2021
<properties>

superstream-clients/dependency-reduced-pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<groupId>ai.superstream</groupId>
55
<artifactId>superstream-clients</artifactId>
66
<name>Superstream Kafka Client Optimizer</name>
7-
<version>2.0.0</version>
7+
<version>2.0.1</version>
88
<description>A Java library that dynamically optimizes Kafka client configuration based on recommendations</description>
99
<url>https://github.com/superstreamlabs/superstream-clients-java</url>
1010
<developers>

superstream-clients/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>ai.superstream</groupId>
88
<artifactId>superstream-clients</artifactId>
9-
<version>2.0.0</version>
9+
<version>2.0.1</version>
1010
<packaging>jar</packaging>
1111

1212
<name>Superstream Kafka Client Optimizer</name>

superstream-clients/src/main/java/ai/superstream/shadow/ShadowProducerManager.java

Lines changed: 129 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,16 @@ public void onCompletion(RecordMetadata shaded, Exception exception) {
657657
unMeta = constructUnshadedRecordMetadata(unRecMetaCls, unTp, shaded);
658658
}
659659

660+
// Guardrail: if we cannot construct metadata for a successful send, skip the
661+
// routed callback rather than violating the usual contract (success + null metadata).
662+
// The application's own producer callback (if any) will still see the real metadata.
663+
if (unMeta == null && exception == null) {
664+
try {
665+
logger.debug("[ROUTED-CALLBACK] onCompletion: skipping routed callback because RecordMetadata could not be constructed and no exception is present");
666+
} catch (Throwable ignoredLog) { }
667+
return;
668+
}
669+
660670
// Strategy 1: Prefer interface signature in the app classloader (most reliable)
661671
if (ifaceMethod != null) {
662672
try {
@@ -681,16 +691,35 @@ public void onCompletion(RecordMetadata shaded, Exception exception) {
681691
}
682692
}
683693

684-
// Strategy 3: Fallback - find any compatible method by name with second param assignable from Throwable/Exception
694+
// Strategy 3: Fallback - find a compatible method by name and parameter types.
695+
// We require:
696+
// - name == "onCompletion"
697+
// - exactly two parameters
698+
// - second parameter assignable from Throwable/Exception
699+
// - (when available) first parameter type compatible with the app's RecordMetadata class
685700
Method chosen = null;
686701
for (Method m : unshadedCallback.getClass().getMethods()) {
687702
if (!m.getName().equals("onCompletion")) continue;
688703
if (m.getParameterCount() != 2) continue;
689704
Class<?>[] pt = m.getParameterTypes();
690705
if (!Throwable.class.isAssignableFrom(pt[1]) && !Exception.class.isAssignableFrom(pt[1])) continue;
706+
if (unRecMetaCls != null && !pt[0].isAssignableFrom(unRecMetaCls)) continue;
691707
chosen = m;
692708
break;
693709
}
710+
if (chosen != null) {
711+
Class<?>[] pt = chosen.getParameterTypes();
712+
Object arg0 = unMeta;
713+
Object arg1 = exception;
714+
715+
// Ensure our arguments are compatible with the chosen method's parameters.
716+
if (arg0 != null && !pt[0].isInstance(arg0)) {
717+
chosen = null;
718+
} else if (arg1 != null && !pt[1].isInstance(arg1)) {
719+
chosen = null;
720+
}
721+
}
722+
694723
if (chosen != null) {
695724
try {
696725
chosen.setAccessible(true);
@@ -722,41 +751,111 @@ public void onCompletion(RecordMetadata shaded, Exception exception) {
722751
* @throws Exception if constructor invocation fails
723752
*/
724753
private static Object constructUnshadedRecordMetadata(Class<?> unRecMetaCls, Object unTopicPartition, RecordMetadata shaded) throws Exception {
725-
// Try known constructor patterns for Kafka 3.3.x and later versions
726754
Constructor<?>[] ctors = unRecMetaCls.getConstructors();
755+
756+
long offset = shaded.offset();
757+
long timestamp = shaded.timestamp();
758+
int serializedKeySize = shaded.serializedKeySize();
759+
int serializedValueSize = shaded.serializedValueSize();
760+
727761
for (Constructor<?> ctor : ctors) {
728-
Class<?>[] p = ctor.getParameterTypes();
729-
try {
730-
boolean isTopicPartition = p.length == 6
731-
&& p[0].getName().equals("org.apache.kafka.common.TopicPartition")
732-
&& p[1] == long.class
733-
&& p[2] == long.class
734-
&& (p[3] == Integer.class || p[3] == int.class || p[3] == Long.class)
735-
&& p[4] == int.class
736-
&& p[5] == int.class;
737-
if (isTopicPartition) {
738-
int sk = shaded.serializedKeySize();
739-
int sv = shaded.serializedValueSize();
740-
// Constructor: RecordMetadata(TopicPartition, long offset, long timestamp, Integer leaderEpoch, int serializedKeySize, int serializedValueSize)
741-
// Pass null for leaderEpoch to accept any epoch type (Integer, int, or Long)
742-
return ctor.newInstance(unTopicPartition, shaded.offset(), shaded.timestamp(), null, sk, sv);
762+
Class<?>[] params = ctor.getParameterTypes();
763+
if (params.length == 0) {
764+
continue;
765+
}
766+
767+
boolean firstIsTopicPartition = "org.apache.kafka.common.TopicPartition".equals(params[0].getName());
768+
if (!firstIsTopicPartition) {
769+
continue;
770+
}
771+
772+
Object[] args = new Object[params.length];
773+
args[0] = unTopicPartition;
774+
775+
int longCount = 0;
776+
777+
// Collect indices of all int/Integer parameters (positions >= 1)
778+
List<Integer> intPositions = new ArrayList<>();
779+
for (int i = 1; i < params.length; i++) {
780+
Class<?> p = params[i];
781+
if (p == int.class || p == Integer.class) {
782+
intPositions.add(i);
743783
}
744-
} catch (Throwable ignored) { }
745-
}
746-
// Fallback: try constructors by parameter count (for older Kafka versions or alternative signatures)
747-
for (Constructor<?> ctor : ctors) {
748-
if (ctor.getParameterCount() == 6) {
749-
try {
750-
return ctor.newInstance(unTopicPartition, shaded.offset(), shaded.timestamp(), null, shaded.serializedKeySize(), shaded.serializedValueSize());
751-
} catch (Throwable ignored) { }
752784
}
753-
if (ctor.getParameterCount() == 5) {
754-
try {
755-
// Try 5-parameter constructor: (TopicPartition, long offset, long timestamp, int keySize, int valueSize)
756-
return ctor.newInstance(unTopicPartition, shaded.offset(), shaded.timestamp(), shaded.serializedKeySize(), shaded.serializedValueSize());
757-
} catch (Throwable ignored) { }
785+
786+
// Decide which indices get key/value sizes: last two int positions, if present
787+
int keySizeIndex = -1;
788+
int valueSizeIndex = -1;
789+
if (!intPositions.isEmpty()) {
790+
valueSizeIndex = intPositions.get(intPositions.size() - 1);
791+
if (intPositions.size() >= 2) {
792+
keySizeIndex = intPositions.get(intPositions.size() - 2);
793+
}
794+
}
795+
796+
for (int i = 1; i < params.length; i++) {
797+
Class<?> p = params[i];
798+
799+
if (p == long.class || p == Long.class) {
800+
long value;
801+
if (longCount == 0) {
802+
value = offset;
803+
} else if (longCount == 1) {
804+
value = timestamp;
805+
} else {
806+
value = 0L;
807+
}
808+
args[i] = (p == long.class) ? value : Long.valueOf(value);
809+
longCount++;
810+
continue;
811+
}
812+
813+
if (p == int.class || p == Integer.class) {
814+
int value;
815+
if (i == keySizeIndex) {
816+
value = serializedKeySize;
817+
} else if (i == valueSizeIndex) {
818+
value = serializedValueSize;
819+
} else {
820+
// Other int fields (e.g., leaderEpoch, future extensions) get a neutral default.
821+
value = 0;
822+
}
823+
args[i] = (p == int.class) ? value : Integer.valueOf(value);
824+
continue;
825+
}
826+
827+
// For other reference types (e.g., leaderEpoch, checksum), pass null.
828+
if (!p.isPrimitive()) {
829+
args[i] = null;
830+
continue;
831+
}
832+
833+
// For any other primitive, fall back to the default zero value.
834+
if (p == boolean.class) {
835+
args[i] = false;
836+
} else if (p == byte.class) {
837+
args[i] = (byte) 0;
838+
} else if (p == short.class) {
839+
args[i] = (short) 0;
840+
} else if (p == char.class) {
841+
args[i] = (char) 0;
842+
} else if (p == float.class) {
843+
args[i] = 0.0f;
844+
} else if (p == double.class) {
845+
args[i] = 0.0d;
846+
} else {
847+
// Should not reach here, but keep it safe.
848+
args[i] = null;
849+
}
850+
}
851+
852+
try {
853+
return ctor.newInstance(args);
854+
} catch (Throwable ignored) {
855+
// Try the next constructor
758856
}
759857
}
858+
760859
// No compatible constructor found - return null (application callback will receive null metadata)
761860
return null;
762861
}

0 commit comments

Comments
 (0)