Skip to content

Commit e75a721

Browse files
mahmoud-elmorabeamahmoud-elmorabea
authored
fix: Sanitize unsupported numeric values (#569)
1 parent 0d3bd4f commit e75a721

File tree

5 files changed

+237
-80
lines changed

5 files changed

+237
-80
lines changed

‎datapipelines/src/main/kotlin/io/customer/datapipelines/extensions/JsonExtensions.kt‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package io.customer.datapipelines.extensions
22

33
import com.segment.analytics.kotlin.core.utilities.toJsonElement
4+
import io.customer.sdk.core.di.SDKComponent
5+
import io.customer.sdk.core.util.Logger
46
import kotlinx.serialization.json.JsonArray
57
import kotlinx.serialization.json.JsonElement
68
import kotlinx.serialization.json.JsonNull
@@ -45,14 +47,51 @@ private fun Any?.toSerializableJson(): JsonElement {
4547
}
4648

4749
/**
48-
* Recursively replaces all `null` values in nested structure with [JsonNull],
49-
* preserving structure and type. Useful for safely converting to `JsonElement`
50-
* using `Any.toJsonElement()` extension.
50+
* Recursively sanitizes data for JSON serialization by:
51+
* 1. Replacing all `null` values in nested structure with [JsonNull]
52+
* 2. Removing entries with NaN or infinity values from maps
53+
* This ensures the data can be safely converted to JSON.
54+
*
55+
* @param logger Logger to log when invalid numeric values are removed
5156
*/
57+
internal fun Map<String, Any?>.sanitizeForJson(logger: Logger = SDKComponent.logger): Map<String, Any?> {
58+
val resultMap = mutableMapOf<String, Any>()
59+
60+
for (entry in this.entries) {
61+
val sanitizedValue = entry.value.sanitizeValue(logger)
62+
if (sanitizedValue != null) {
63+
resultMap[entry.key] = sanitizedValue
64+
} else {
65+
logger.error("Removed invalid JSON numeric value (NaN or infinity) for key: ${entry.key}")
66+
}
67+
}
68+
69+
return resultMap
70+
}
71+
5272
@Suppress("UNCHECKED_CAST")
53-
internal fun <T> T?.sanitizeNullsForJson(): T = when (this) {
73+
private fun <T> T?.sanitizeValue(logger: Logger = SDKComponent.logger): T? = when (this) {
5474
null -> JsonNull as T
55-
is Map<*, *> -> this.mapValues { (_, v) -> v.sanitizeNullsForJson() } as T
56-
is List<*> -> this.map { it.sanitizeNullsForJson() } as T
75+
is Double, is Float -> if (isInvalidJsonNumber(this)) null else this
76+
is Map<*, *> -> (this as? Map<String, Any?>)?.sanitizeForJson(logger) as T
77+
is List<*> -> sanitizeList(logger) as T
5778
else -> this
5879
}
80+
81+
private fun List<*>.sanitizeList(logger: Logger): List<*> {
82+
return this.mapNotNull {
83+
val sanitizedValue = it.sanitizeValue(logger)
84+
if (sanitizedValue == null) {
85+
logger.error("Removed invalid JSON numeric value (NaN or infinity)")
86+
}
87+
sanitizedValue
88+
}
89+
}
90+
91+
private fun isInvalidJsonNumber(value: Any?): Boolean {
92+
return when (value) {
93+
is Float -> value.isNaN() || value.isInfinite()
94+
is Double -> value.isNaN() || value.isInfinite()
95+
else -> false
96+
}
97+
}

‎datapipelines/src/main/kotlin/io/customer/sdk/CustomerIO.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import io.customer.datapipelines.config.DataPipelinesModuleConfig
1515
import io.customer.datapipelines.di.analyticsFactory
1616
import io.customer.datapipelines.di.dataPipelinesLogger
1717
import io.customer.datapipelines.extensions.asMap
18-
import io.customer.datapipelines.extensions.sanitizeNullsForJson
18+
import io.customer.datapipelines.extensions.sanitizeForJson
1919
import io.customer.datapipelines.extensions.type
2020
import io.customer.datapipelines.extensions.updateAnalyticsConfig
2121
import io.customer.datapipelines.migration.TrackingMigrationProcessor
@@ -194,7 +194,7 @@ class CustomerIO private constructor(
194194
identify(userId = identifier, traits = value)
195195
} else {
196196
logger.debug("No user profile found, updating sanitized traits for anonymous user ${analytics.anonymousId()}")
197-
analytics.identify(traits = value.sanitizeNullsForJson())
197+
analytics.identify(traits = value.sanitizeForJson())
198198
}
199199
}
200200

‎datapipelines/src/main/kotlin/io/customer/sdk/DataPipelineInstance.kt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package io.customer.sdk
22

33
import com.segment.analytics.kotlin.core.emptyJsonObject
44
import com.segment.analytics.kotlin.core.utilities.JsonAnySerializer
5-
import io.customer.datapipelines.extensions.sanitizeNullsForJson
5+
import io.customer.datapipelines.extensions.sanitizeForJson
66
import io.customer.sdk.data.model.CustomAttributes
77
import io.customer.sdk.events.TrackMetric
88
import kotlinx.serialization.SerializationStrategy
@@ -67,7 +67,7 @@ abstract class DataPipelineInstance : CustomerIOInstance {
6767
*/
6868
fun identify(userId: String, traits: Map<String, Any?>) {
6969
// Method needed for Java interop as inline doesn't work with Java
70-
identify(userId = userId, traits = traits.sanitizeNullsForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
70+
identify(userId = userId, traits = traits.sanitizeForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
7171
}
7272

7373
/**
@@ -131,7 +131,7 @@ abstract class DataPipelineInstance : CustomerIOInstance {
131131
*/
132132
fun track(name: String, properties: Map<String, Any?>) {
133133
// Method needed for Java interop as inline doesn't work with Java
134-
track(name = name, properties = properties.sanitizeNullsForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
134+
track(name = name, properties = properties.sanitizeForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
135135
}
136136

137137
/**
@@ -204,7 +204,7 @@ abstract class DataPipelineInstance : CustomerIOInstance {
204204
*/
205205
fun screen(title: String, properties: Map<String, Any?>) {
206206
// Method needed for Java interop as inline doesn't work with Java
207-
screen(title = title, properties = properties.sanitizeNullsForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
207+
screen(title = title, properties = properties.sanitizeForJson(), serializationStrategy = JsonAnySerializer.serializersModule.serializer())
208208
}
209209

210210
/**

‎datapipelines/src/test/java/io/customer/datapipelines/util/SanitizeForJsonTest.kt‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package io.customer.datapipelines.util
2+
3+
import io.customer.datapipelines.extensions.sanitizeForJson
4+
import io.customer.datapipelines.testutils.core.JUnitTest
5+
import io.customer.sdk.core.util.CioLogLevel
6+
import io.customer.sdk.core.util.Logger
7+
import io.mockk.every
8+
import io.mockk.mockk
9+
import io.mockk.verify
10+
import kotlinx.serialization.json.JsonNull
11+
import org.amshove.kluent.shouldBeEqualTo
12+
import org.junit.jupiter.api.Test
13+
14+
/**
15+
* Tests for the sanitizeForJson function which sanitizes data for JSON serialization by:
16+
* 1. Replacing null values with JsonNull
17+
* 2. Removing entries with NaN or infinity values from maps
18+
*/
19+
class SanitizeForJsonTest : JUnitTest() {
20+
@Test
21+
fun sanitize_givenNull_expectJsonNull() {
22+
val input = mapOf("key" to null)
23+
val expected = mapOf("key" to JsonNull)
24+
input.sanitizeForJson() shouldBeEqualTo expected
25+
}
26+
27+
@Test
28+
fun sanitize_givenFlatMapWithNull_expectJsonNullReplacement() {
29+
val input = mapOf("a" to 1, "b" to null)
30+
val expected = mapOf("a" to 1, "b" to JsonNull)
31+
input.sanitizeForJson() shouldBeEqualTo expected
32+
}
33+
34+
@Test
35+
fun sanitize_givenNestedMapWithNull_expectJsonNullReplacement() {
36+
val input = mapOf("meta" to mapOf("os" to "android", "root" to false, "vendor" to null, "version" to 13))
37+
val expected = mapOf("meta" to mapOf("os" to "android", "root" to false, "vendor" to JsonNull, "version" to 13))
38+
input.sanitizeForJson() shouldBeEqualTo expected
39+
}
40+
41+
@Test
42+
fun sanitize_givenListWithNull_expectJsonNullReplacement() {
43+
val input = mapOf("key" to listOf("a", null, "b"))
44+
val expected = mapOf("key" to listOf("a", JsonNull, "b"))
45+
input.sanitizeForJson() shouldBeEqualTo expected
46+
}
47+
48+
@Test
49+
fun sanitize_givenNestedStructureWithNull_expectJsonNullReplacement() {
50+
val input = mapOf(
51+
"maps" to listOf(
52+
mapOf("key" to null),
53+
mapOf("key" to "value")
54+
),
55+
"colors" to listOf("red", null, "blue")
56+
)
57+
val expected = mapOf(
58+
"maps" to listOf(
59+
mapOf("key" to JsonNull),
60+
mapOf("key" to "value")
61+
),
62+
"colors" to listOf("red", JsonNull, "blue")
63+
)
64+
input.sanitizeForJson() shouldBeEqualTo expected
65+
}
66+
67+
@Test
68+
fun sanitize_givenMapWithoutNull_expectReturnSameStructure() {
69+
val input = mapOf("x" to 42, "y" to "hello")
70+
input.sanitizeForJson() shouldBeEqualTo input
71+
}
72+
73+
@Test
74+
fun sanitize_givenListWithoutNull_expectReturnSameStructure() {
75+
val input = mapOf("key" to listOf("apple", "banana", "cherry"))
76+
input.sanitizeForJson() shouldBeEqualTo input
77+
}
78+
79+
@Test
80+
fun sanitize_givenMapWithNaN_expectFilteredMap() {
81+
val input = mapOf("x" to 42, "y" to Float.NaN, "z" to "hello")
82+
val expected = mapOf("x" to 42, "z" to "hello")
83+
input.sanitizeForJson() shouldBeEqualTo expected
84+
}
85+
86+
@Test
87+
fun sanitize_givenMapWithPositiveInfinity_expectFilteredMap() {
88+
val input = mapOf("x" to 42, "y" to Double.POSITIVE_INFINITY, "z" to "hello")
89+
val expected = mapOf("x" to 42, "z" to "hello")
90+
input.sanitizeForJson() shouldBeEqualTo expected
91+
}
92+
93+
@Test
94+
fun sanitize_givenMapWithNegativeInfinity_expectFilteredMap() {
95+
val input = mapOf("x" to 42, "y" to Float.NEGATIVE_INFINITY, "z" to "hello")
96+
val expected = mapOf("x" to 42, "z" to "hello")
97+
input.sanitizeForJson() shouldBeEqualTo expected
98+
}
99+
100+
@Test
101+
fun sanitize_givenNestedMapWithInvalidNumbers_expectFilteredMap() {
102+
val input = mapOf(
103+
"meta" to mapOf(
104+
"os" to "android",
105+
"version" to 13,
106+
"temperature" to Double.NaN,
107+
"battery" to Float.POSITIVE_INFINITY
108+
),
109+
"data" to "valid",
110+
"list" to listOf(20, Double.NaN)
111+
)
112+
val expected = mapOf(
113+
"meta" to mapOf(
114+
"os" to "android",
115+
"version" to 13
116+
),
117+
"data" to "valid",
118+
"list" to listOf(20)
119+
)
120+
input.sanitizeForJson() shouldBeEqualTo expected
121+
}
122+
123+
@Test
124+
fun sanitize_givenComplexStructureWithNullsAndInvalidNumbers_expectCorrectFiltering() {
125+
val input = mapOf(
126+
"user" to mapOf(
127+
"name" to "John",
128+
"age" to null,
129+
"score" to Double.NaN
130+
),
131+
"metrics" to listOf(
132+
mapOf("key" to "views", "value" to 100),
133+
listOf(15.0, Float.POSITIVE_INFINITY),
134+
mapOf("key" to "conversions", "value" to null)
135+
),
136+
"settings" to mapOf(
137+
"enabled" to true,
138+
"timeout" to Float.NEGATIVE_INFINITY,
139+
"fallback" to null
140+
)
141+
)
142+
143+
val expected = mapOf(
144+
"user" to mapOf(
145+
"name" to "John",
146+
"age" to JsonNull
147+
),
148+
"metrics" to listOf(
149+
mapOf("key" to "views", "value" to 100),
150+
listOf(15.0),
151+
mapOf("key" to "conversions", "value" to JsonNull)
152+
),
153+
"settings" to mapOf(
154+
"enabled" to true,
155+
"fallback" to JsonNull
156+
)
157+
)
158+
159+
input.sanitizeForJson() shouldBeEqualTo expected
160+
}
161+
162+
@Test
163+
fun sanitize_givenInvalidNumericValues_expectLogMessages() {
164+
// Create a mock logger
165+
val mockLogger = mockk<Logger>(relaxed = true)
166+
every { mockLogger.logLevel } returns CioLogLevel.DEBUG
167+
168+
// Map with multiple invalid numeric values
169+
val input = mapOf(
170+
"nan" to Float.NaN,
171+
"posInf" to Double.POSITIVE_INFINITY,
172+
"negInf" to Float.NEGATIVE_INFINITY,
173+
"valid" to 42
174+
)
175+
176+
// Sanitize with the mock logger
177+
input.sanitizeForJson(mockLogger)
178+
179+
// Verify that debug was called 3 times (once for each invalid value)
180+
verify {
181+
mockLogger.error("Removed invalid JSON numeric value (NaN or infinity) for key: nan")
182+
mockLogger.error("Removed invalid JSON numeric value (NaN or infinity) for key: posInf")
183+
mockLogger.error("Removed invalid JSON numeric value (NaN or infinity) for key: negInf")
184+
}
185+
}
186+
}

‎datapipelines/src/test/java/io/customer/datapipelines/util/SanitizeNullsForJsonTest.kt‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

0 commit comments

Comments
 (0)