Skip to content

Commit 3f9c26b

Browse files
charlesmyucharlesmyu
committed
Update tests for meta field in Spark SQL plans
1 parent 9c19e11 commit 3f9c26b

File tree

2 files changed

+91
-7
lines changed

2 files changed

+91
-7
lines changed

‎dd-java-agent/instrumentation/spark/src/testFixtures/groovy/datadog/trace/instrumentation/spark/AbstractSpark24SqlTest.groovy‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
4747
}
4848

4949
static assertStringSQLPlanEquals(String expectedString, String actualString) {
50-
System.err.println("Checking if expected $expectedString SQL plan match actual $actualString")
50+
// System.err.println("Checking if expected $expectedString SQL plan match actual $actualString")
5151

5252
def jsonSlurper = new JsonSlurper()
5353

@@ -60,7 +60,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
6060
// Similar to assertStringSQLPlanEquals, but the actual plan can be a subset of the expected plan
6161
// This is used for spark 2.4 where the exact SQL plan is not deterministic
6262
protected static assertStringSQLPlanSubset(String expectedString, String actualString) {
63-
System.err.println("Checking if expected $expectedString SQL plan is a super set of $actualString")
63+
// System.err.println("Checking if expected $expectedString SQL plan is a super set of $actualString")
6464

6565
def jsonSlurper = new JsonSlurper()
6666

@@ -83,7 +83,11 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
8383
// Checking all keys expect children and metrics that are checked after
8484
expected.keySet().each { key ->
8585
if (!['children', 'metrics'].contains(key)) {
86-
86+
if (key == "meta") {
87+
var simpleString = actual["nodeDetailString"]
88+
var values = actual[key]
89+
System.err.println("FOUND: node=$actual.node, values=$values, simpleString=$simpleString")
90+
}
8791
// Some metric values will varies between runs
8892
// In the case, setting the expected value to "any" skips the assertion
8993
if (expected[key] != "any") {
@@ -159,6 +163,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
159163
"node": "Exchange",
160164
"nodeId": -1909876497,
161165
"nodeDetailString": "hashpartitioning(string_col#0, 2)",
166+
"meta": "any",
162167
"metrics": [
163168
{
164169
"data size total (min, med, max)": "any",
@@ -169,6 +174,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
169174
{
170175
"node": "WholeStageCodegen",
171176
"nodeId": 724251804,
177+
"meta": "any",
172178
"metrics": [
173179
{
174180
"duration total (min, med, max)": "any",
@@ -180,6 +186,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
180186
"node": "HashAggregate",
181187
"nodeId": 1128016273,
182188
"nodeDetailString": "(keys=[string_col#0], functions=[partial_avg(double_col#1)])",
189+
"meta": "any",
183190
"metrics": [
184191
{
185192
"aggregate time total (min, med, max)": "any",
@@ -198,11 +205,13 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
198205
{
199206
"node": "InputAdapter",
200207
"nodeId": 180293,
208+
"meta": "any",
201209
"children": [
202210
{
203211
"node": "LocalTableScan",
204212
"nodeId": 1632930767,
205213
"nodeDetailString": "[string_col#0, double_col#1]",
214+
"meta": "any",
206215
"metrics": [
207216
{
208217
"number of output rows": 3,
@@ -224,6 +233,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
224233
{
225234
"node": "WholeStageCodegen",
226235
"nodeId": 724251804,
236+
"meta": "any",
227237
"metrics": [
228238
{
229239
"duration total (min, med, max)": "any",
@@ -235,6 +245,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
235245
"node": "HashAggregate",
236246
"nodeId": 126020943,
237247
"nodeDetailString": "(keys=[string_col#0], functions=[avg(double_col#1)])",
248+
"meta": "any",
238249
"metrics": [
239250
{
240251
"aggregate time total (min, med, max)": "any",
@@ -256,7 +267,8 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
256267
"children": [
257268
{
258269
"node": "InputAdapter",
259-
"nodeId": 180293
270+
"nodeId": 180293,
271+
"meta": "any"
260272
}
261273
]
262274
}
@@ -325,6 +337,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
325337
"node": "Exchange",
326338
"nodeId": "any",
327339
"nodeDetailString": "hashpartitioning(string_col#25, 2)",
340+
"meta": "any",
328341
"metrics": [
329342
{
330343
"data size total (min, med, max)": "any",
@@ -336,6 +349,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
336349
"node": "LocalTableScan",
337350
"nodeId": "any",
338351
"nodeDetailString": "[string_col#25]",
352+
"meta": "any",
339353
"metrics": [
340354
{
341355
"number of output rows": "any",
@@ -351,6 +365,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
351365
"node": "Exchange",
352366
"nodeId": "any",
353367
"nodeDetailString": "hashpartitioning(string_col#21, 2)",
368+
"meta": "any",
354369
"metrics": [
355370
{
356371
"data size total (min, med, max)": "any",
@@ -362,6 +377,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
362377
"node": "LocalTableScan",
363378
"nodeId": "any",
364379
"nodeDetailString": "[string_col#21]",
380+
"meta": "any",
365381
"metrics": [
366382
{
367383
"number of output rows": "any",
@@ -377,6 +393,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
377393
"node": "Exchange",
378394
"nodeId": -1350402171,
379395
"nodeDetailString": "SinglePartition",
396+
"meta": "any",
380397
"metrics": [
381398
{
382399
"data size total (min, med, max)": "any",
@@ -387,6 +404,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
387404
{
388405
"node": "WholeStageCodegen",
389406
"nodeId": 724251804,
407+
"meta": "any",
390408
"metrics": [
391409
{
392410
"duration total (min, med, max)": "any",
@@ -398,6 +416,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
398416
"node": "HashAggregate",
399417
"nodeId": -879128980,
400418
"nodeDetailString": "(keys=[], functions=[partial_count(1)])",
419+
"meta": "any",
401420
"metrics": [
402421
{
403422
"aggregate time total (min, med, max)": "any",
@@ -412,11 +431,13 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
412431
{
413432
"node": "Project",
414433
"nodeId": 1355342585,
434+
"meta": "any",
415435
"children": [
416436
{
417437
"node": "SortMergeJoin",
418438
"nodeId": -1975876610,
419439
"nodeDetailString": "[string_col#21], [string_col#25], Inner",
440+
"meta": "any",
420441
"metrics": [
421442
{
422443
"number of output rows": "any",
@@ -427,10 +448,12 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
427448
{
428449
"node": "InputAdapter",
429450
"nodeId": 180293,
451+
"meta": "any",
430452
"children": [
431453
{
432454
"node": "WholeStageCodegen",
433455
"nodeId": 724251804,
456+
"meta": "any",
434457
"metrics": [
435458
{
436459
"duration total (min, med, max)": "any",
@@ -442,6 +465,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
442465
"node": "Sort",
443466
"nodeId": 66807398,
444467
"nodeDetailString": "[string_col#21 ASC NULLS FIRST], false, 0",
468+
"meta": "any",
445469
"metrics": [
446470
{
447471
"peak memory total (min, med, max)": "any",
@@ -455,7 +479,8 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
455479
"children": [
456480
{
457481
"node": "InputAdapter",
458-
"nodeId": 180293
482+
"nodeId": 180293,
483+
"meta": "any"
459484
}
460485
]
461486
}
@@ -466,10 +491,12 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
466491
{
467492
"node": "InputAdapter",
468493
"nodeId": 180293,
494+
"meta": "any",
469495
"children": [
470496
{
471497
"node": "WholeStageCodegen",
472498
"nodeId": 724251804,
499+
"meta": "any",
473500
"metrics": [
474501
{
475502
"duration total (min, med, max)": "any",
@@ -481,6 +508,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
481508
"node": "Sort",
482509
"nodeId": -952138782,
483510
"nodeDetailString": "[string_col#25 ASC NULLS FIRST], false, 0",
511+
"meta": "any",
484512
"metrics": [
485513
{
486514
"peak memory total (min, med, max)": "any",
@@ -490,7 +518,8 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
490518
"children": [
491519
{
492520
"node": "InputAdapter",
493-
"nodeId": 180293
521+
"nodeId": 180293,
522+
"meta": "any"
494523
}
495524
]
496525
}
@@ -513,6 +542,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
513542
{
514543
"node": "WholeStageCodegen",
515544
"nodeId": 724251804,
545+
"meta": "any",
516546
"metrics": [
517547
{
518548
"duration total (min, med, max)": "any",
@@ -524,6 +554,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
524554
"node": "HashAggregate",
525555
"nodeId": 724815342,
526556
"nodeDetailString": "(keys=[], functions=[count(1)])",
557+
"meta": "any",
527558
"metrics": [
528559
{
529560
"number of output rows": 1,
@@ -533,7 +564,8 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
533564
"children": [
534565
{
535566
"node": "InputAdapter",
536-
"nodeId": 180293
567+
"nodeId": 180293,
568+
"meta": "any"
537569
}
538570
]
539571
}

0 commit comments

Comments
 (0)