Skip to content

Commit 5fa00f7

Browse files
Ma11hewThomasMa11hewThomas
authored
Add labels to test object (#51)
1 parent f3cb5e3 commit 5fa00f7

File tree

4 files changed

+177
-17
lines changed

4 files changed

+177
-17
lines changed

‎examples/comprehensive.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
"suite": ["Authentication", "Login"],
4141
"type": "e2e",
4242
"tags": ["smoke", "critical"],
43+
"labels": {
44+
"priority": "P1",
45+
"severity": 2,
46+
"automated": true,
47+
"externalId": 45678
48+
},
4349
"filePath": "tests/auth/login.test.js",
4450
"browser": "chromium",
4551
"threadId": "worker-1",
@@ -67,6 +73,12 @@
6773
"suite": ["Checkout"],
6874
"type": "e2e",
6975
"tags": ["smoke"],
76+
"labels": {
77+
"priority": "P2",
78+
"severity": 3,
79+
"automated": true,
80+
"externalId": 45679
81+
},
7082
"filePath": "tests/checkout/checkout.test.js",
7183
"browser": "chromium",
7284
"threadId": "worker-2",
@@ -121,6 +133,12 @@
121133
"suite": ["Admin", "Dashboard"],
122134
"type": "e2e",
123135
"tags": ["regression"],
136+
"labels": {
137+
"priority": "P3",
138+
"severity": 2,
139+
"automated": true,
140+
"externalId": 45677
141+
},
124142
"filePath": "tests/admin/dashboard.test.js",
125143
"line": 42,
126144
"browser": "chromium",

‎schema/ctrf.schema.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,13 @@
196196
"type": "string"
197197
}
198198
},
199+
"labels": {
200+
"description": "Structured key-value metadata for the test case (e.g. priority, severity, external identifiers)",
201+
"type": "object",
202+
"additionalProperties": {
203+
"type": [ "string", "number", "boolean" ]
204+
}
205+
},
199206
"type": {
200207
"description": "Test classification (e.g., 'unit', 'integration', 'e2e')",
201208
"type": "string"

‎spec/ctrf.md‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -851,13 +851,24 @@ If present, it MUST be a string.
851851
### 9.14. `tags`
852852

853853
**Description:**
854-
A list of user-defined tags or labels associated with the test.
854+
A list of user-defined tags associated with the test.
855855

856856
**Requirements:**
857857
`tags` is OPTIONAL.
858858
If present, it MUST be an array of strings.
859859

860-
### 9.15. `type`
860+
### 9.15. `labels`
861+
862+
**Description:**
863+
Structured key-value metadata, commonly used for classification and external system integration (e.g. priority, severity, external identifiers).
864+
865+
**Requirements:**
866+
`labels` is OPTIONAL.
867+
If present, it MUST be an object.
868+
Each value MUST be one of: string, number, or boolean.
869+
Consumers MUST treat label keys as opaque and MUST NOT require any particular keys to be present.
870+
871+
### 9.16. `type`
861872

862873
**Description:**
863874
A classification or category for the test (for example, `"unit"`, `"integration"`, or `"e2e"`).
@@ -866,7 +877,7 @@ A classification or category for the test (for example, `"unit"`, `"integration"
866877
`type` is OPTIONAL.
867878
If present, it MUST be a string.
868879

869-
### 9.16. `filePath`
880+
### 9.17. `filePath`
870881

871882
**Description:**
872883
The path to the file that defines the test case.
@@ -875,7 +886,7 @@ The path to the file that defines the test case.
875886
`filePath` is OPTIONAL.
876887
If present, it MUST be a string.
877888

878-
### 9.17. `retries`
889+
### 9.18. `retries`
879890

880891
**Description:**
881892
The number of retries performed for this test case.
@@ -885,7 +896,7 @@ The number of retries performed for this test case.
885896
If present, it MUST be a non-negative integer.
886897
It SHOULD equal the count of entries in `retryAttempts`.
887898

888-
### 9.18. `retryAttempts`
899+
### 9.19. `retryAttempts`
889900

890901
**Description:**
891902
A list of retry attempts performed for this test case.
@@ -895,7 +906,7 @@ A list of retry attempts performed for this test case.
895906
If present, it MUST be an array.
896907
Each entry MUST follow the structure defined in Section 11 (Retry Attempt Object).
897908

898-
### 9.19. `flaky`
909+
### 9.20. `flaky`
899910

900911
**Description:**
901912
Indicates whether the test is considered flaky.
@@ -906,7 +917,7 @@ A test is considered flaky only if its final status is `passed` and it experienc
906917
`flaky` is OPTIONAL.
907918
If present, it MUST be a boolean.
908919

909-
### 9.20. `stdout`
920+
### 9.21. `stdout`
910921

911922
**Description:**
912923
Lines of standard output generated during test execution.
@@ -915,7 +926,7 @@ Lines of standard output generated during test execution.
915926
`stdout` is OPTIONAL.
916927
If present, it MUST be an array of strings.
917928

918-
### 9.21. `stderr`
929+
### 9.22. `stderr`
919930

920931
**Description:**
921932
Lines of standard error output generated during test execution.
@@ -924,7 +935,7 @@ Lines of standard error output generated during test execution.
924935
`stderr` is OPTIONAL.
925936
If present, it MUST be an array of strings.
926937

927-
### 9.22. `threadId`
938+
### 9.23. `threadId`
928939

929940
**Description:**
930941
Identifies the thread or worker on which the test executed.
@@ -933,7 +944,7 @@ Identifies the thread or worker on which the test executed.
933944
`threadId` is OPTIONAL.
934945
If present, it MUST be a string.
935946

936-
### 9.23. `browser`
947+
### 9.24. `browser`
937948

938949
**Description:**
939950
Identifies the browser used during test execution (for browser-based tests).
@@ -942,7 +953,7 @@ Identifies the browser used during test execution (for browser-based tests).
942953
`browser` is OPTIONAL.
943954
If present, it MUST be a string.
944955

945-
### 9.24. `device`
956+
### 9.25. `device`
946957

947958
**Description:**
948959
Identifies the device or device profile used during test execution.
@@ -951,7 +962,7 @@ Identifies the device or device profile used during test execution.
951962
`device` is OPTIONAL.
952963
If present, it MUST be a string.
953964

954-
### 9.25. `screenshot`
965+
### 9.26. `screenshot`
955966

956967
**Description:**
957968
A single base64-encoded screenshot captured during execution of the test case.
@@ -968,7 +979,7 @@ Producers MUST use the `attachments` array if providing additional screenshots o
968979
Screenshots are base64-encoded and may significantly increase document size.
969980
Producers SHOULD prefer file-based attachments for large images.
970981

971-
### 9.26. `attachments`
982+
### 9.27. `attachments`
972983

973984
**Description:**
974985
A list of additional artifacts associated with the test case (screenshots, logs, videos, etc.).
@@ -978,7 +989,7 @@ A list of additional artifacts associated with the test case (screenshots, logs,
978989
If present, it MUST be an array.
979990
Each entry MUST follow the structure defined in Section 12 (Attachment Object).
980991

981-
### 9.27. `parameters`
992+
### 9.28. `parameters`
982993

983994
**Description:**
984995
A set of test parameters, input values, or contextual data relevant to the execution.
@@ -988,7 +999,7 @@ A set of test parameters, input values, or contextual data relevant to the execu
988999
If present, it MUST be an object.
9891000
It MAY contain arbitrary fields.
9901001

991-
### 9.28. `steps`
1002+
### 9.29. `steps`
9921003

9931004
**Description:**
9941005
A list of test steps or sub-operations performed during the test.
@@ -998,7 +1009,7 @@ A list of test steps or sub-operations performed during the test.
9981009
If present, it MUST be an array.
9991010
Each element MUST follow the structure defined in Section 13 (Step Object).
10001011

1001-
### 9.29. `insights`
1012+
### 9.30. `insights`
10021013

10031014
**Description:**
10041015
Derived or aggregated insights specific to this test case.
@@ -1007,7 +1018,7 @@ Derived or aggregated insights specific to this test case.
10071018
`insights` is OPTIONAL.
10081019
If present, it MUST follow the structure defined in Section 15 (Test-Level Insights Object).
10091020

1010-
### 9.30. `extra`
1021+
### 9.31. `extra`
10111022

10121023
**Description:**
10131024
An extensibility object containing arbitrary metadata.
@@ -2001,6 +2012,13 @@ to this specification.
20012012
"type": "string"
20022013
}
20032014
},
2015+
"labels": {
2016+
"description": "Structured key-value metadata for the test case (e.g. priority, severity, external identifiers)",
2017+
"type": "object",
2018+
"additionalProperties": {
2019+
"type": [ "string", "number", "boolean" ]
2020+
}
2021+
},
20042022
"type": {
20052023
"description": "Test classification (e.g., 'unit', 'integration', 'e2e')",
20062024
"type": "string"

‎tests/normative/type-validation.test.json‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,123 @@
444444
}
445445
}
446446
},
447+
{
448+
"description": "Test with labels as array instead of object",
449+
"valid": false,
450+
"data": {
451+
"reportFormat": "CTRF",
452+
"specVersion": "1.0.0",
453+
"results": {
454+
"tool": { "name": "runner" },
455+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
456+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": ["priority", "P1"] }]
457+
}
458+
}
459+
},
460+
{
461+
"description": "Test with labels as string instead of object",
462+
"valid": false,
463+
"data": {
464+
"reportFormat": "CTRF",
465+
"specVersion": "1.0.0",
466+
"results": {
467+
"tool": { "name": "runner" },
468+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
469+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": "priority:P1" }]
470+
}
471+
}
472+
},
473+
{
474+
"description": "Test with labels containing invalid value type (array)",
475+
"valid": false,
476+
"data": {
477+
"reportFormat": "CTRF",
478+
"specVersion": "1.0.0",
479+
"results": {
480+
"tool": { "name": "runner" },
481+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
482+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "priority": ["P1", "P2"] } }]
483+
}
484+
}
485+
},
486+
{
487+
"description": "Test with labels containing invalid value type (object)",
488+
"valid": false,
489+
"data": {
490+
"reportFormat": "CTRF",
491+
"specVersion": "1.0.0",
492+
"results": {
493+
"tool": { "name": "runner" },
494+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
495+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "priority": { "level": "P1" } } }]
496+
}
497+
}
498+
},
499+
{
500+
"description": "Test with labels containing invalid value type (null)",
501+
"valid": false,
502+
"data": {
503+
"reportFormat": "CTRF",
504+
"specVersion": "1.0.0",
505+
"results": {
506+
"tool": { "name": "runner" },
507+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
508+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "priority": null } }]
509+
}
510+
}
511+
},
512+
{
513+
"description": "Test with valid labels containing string values",
514+
"valid": true,
515+
"data": {
516+
"reportFormat": "CTRF",
517+
"specVersion": "1.0.0",
518+
"results": {
519+
"tool": { "name": "runner" },
520+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
521+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "priority": "P1", "severity": "high" } }]
522+
}
523+
}
524+
},
525+
{
526+
"description": "Test with valid labels containing number values",
527+
"valid": true,
528+
"data": {
529+
"reportFormat": "CTRF",
530+
"specVersion": "1.0.0",
531+
"results": {
532+
"tool": { "name": "runner" },
533+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
534+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "severity": 2, "externalId": 12345 } }]
535+
}
536+
}
537+
},
538+
{
539+
"description": "Test with valid labels containing boolean values",
540+
"valid": true,
541+
"data": {
542+
"reportFormat": "CTRF",
543+
"specVersion": "1.0.0",
544+
"results": {
545+
"tool": { "name": "runner" },
546+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
547+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "automated": true, "flaky": false } }]
548+
}
549+
}
550+
},
551+
{
552+
"description": "Test with valid labels containing mixed value types",
553+
"valid": true,
554+
"data": {
555+
"reportFormat": "CTRF",
556+
"specVersion": "1.0.0",
557+
"results": {
558+
"tool": { "name": "runner" },
559+
"summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 },
560+
"tests": [{ "name": "test", "status": "passed", "duration": 100, "labels": { "priority": "P1", "severity": 2, "automated": true } }]
561+
}
562+
}
563+
},
447564
{
448565
"description": "Test with retries as string instead of integer",
449566
"valid": false,

0 commit comments

Comments
 (0)