Skip to content

Commit 4397dd6

Browse files
plutasnyyplutasnyy
authored
Add calculation of table related metrics based on table_as_cells (#2898)
This pull request add metrics that are calculated based on table_as_cells instead of text_as_html. This change is required for comprehensive metrics calculation, as previously every colspan or rowspan predicted was considered to be an incorrect predicted (even if it was correct prediction) This change has to be merged after #2892 which introduces table_as_cells field
1 parent 0cd07d7 commit 4397dd6

File tree

9 files changed

+296
-34
lines changed

9 files changed

+296
-34
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
## 0.13.7-dev7
1+
## 0.13.7-dev8
22

33
### Enhancements
44

55
* **Remove `page_number` metadata fields** for HTML partition until we have a better strategy to decide page counting.
66
* **Extract OCRAgent.get_agent().** Generalize access to the configured OCRAgent instance beyond its use for PDFs.
7+
* **Add calculation of table related metrics which take into account colspans and rowspans**
78

89
### Features
910

‎example-docs/test_evaluate_files/unstructured_output_table_structure/IRS-2023-Form-1095-A.pdf.json‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,8 @@
259259
"eng"
260260
],
261261
"page_number": 2,
262-
"text_as_html": "<table><thead><th>1</th><th>Marketplace identifier</th><th>2 Marketplace-assigned policy number</th><th colspan=\"3\">3 Policy issuer’s name</th></thead><thead><th>4</th><th>Recipient’s name</th><th></th><th>5 Recipient’s SSN</th><th>6</th><th>Recipient’s date of birth</th></thead><tr><td>7</td><td>Recipient’s spouse’s name</td><td></td><td>8 Recipient’s spouse’s SSN</td><td>9</td><td>Recipient’s spouse’s date of birth</td></tr><tr><td>10</td><td>Policy start date</td><td>11 Policy termination date</td><td colspan=\"3\">12 Street address (including apartment no.)</td></tr><tr><td>13</td><td>City or town</td><td>14 State or province</td><td colspan=\"3\">15 Country and ZIP or foreign postal code</td></tr></table>"
262+
"text_as_html": "<table><thead><th>1</th><th>Marketplace identifier</th><th>2 Marketplace-assigned policy number</th><th colspan=\"3\">3 Policy issuer’s name</th></thead><thead><th>4</th><th>Recipient’s name</th><th></th><th>5 Recipient’s SSN</th><th>6</th><th>Recipient’s date of birth</th></thead><tr><td>7</td><td>Recipient’s spouse’s name</td><td></td><td>8 Recipient’s spouse’s SSN</td><td>9</td><td>Recipient’s spouse’s date of birth</td></tr><tr><td>10</td><td>Policy start date</td><td>11 Policy termination date</td><td colspan=\"3\">12 Street address (including apartment no.)</td></tr><tr><td>13</td><td>City or town</td><td>14 State or province</td><td colspan=\"3\">15 Country and ZIP or foreign postal code</td></tr></table>",
263+
"table_as_cells": [{"x": 0, "y": 0, "w": 1, "h": 1, "content": "1"}, {"x": 0, "y": 1, "w": 1, "h": 1, "content": "4"}, {"x": 0, "y": 2, "w": 1, "h": 1, "content": "7"}, {"x": 0, "y": 3, "w": 1, "h": 1, "content": "10"}, {"x": 0, "y": 4, "w": 1, "h": 1, "content": "13"}, {"x": 1, "y": 0, "w": 1, "h": 1, "content": "Marketplace identifier"}, {"x": 1, "y": 1, "w": 1, "h": 1, "content": "Recipient\u2019s name"}, {"x": 1, "y": 2, "w": 1, "h": 1, "content": "Recipient\u2019s spouse\u2019s name"}, {"x": 1, "y": 3, "w": 1, "h": 1, "content": "Policy start date"}, {"x": 1, "y": 4, "w": 1, "h": 1, "content": "City or town"}, {"x": 2, "y": 0, "w": 1, "h": 1, "content": "2 Marketplace-assigned policy number"}, {"x": 2, "y": 1, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 2, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 3, "w": 1, "h": 1, "content": "11 Policy termination date"}, {"x": 2, "y": 4, "w": 1, "h": 1, "content": "14 State or province"}, {"x": 3, "y": 1, "w": 1, "h": 1, "content": "5 Recipient\u2019s SSN"}, {"x": 3, "y": 2, "w": 1, "h": 1, "content": "8 Recipient\u2019s spouse\u2019s SSN"}, {"x": 4, "y": 1, "w": 1, "h": 1, "content": "6"}, {"x": 4, "y": 2, "w": 1, "h": 1, "content": "9"}, {"x": 5, "y": 1, "w": 1, "h": 1, "content": "Recipient\u2019s date of birth"}, {"x": 5, "y": 2, "w": 1, "h": 1, "content": "Recipient\u2019s spouse\u2019s date of birth"}, {"x": 3, "y": 3, "w": 3, "h": 1, "content": "12 Street address (including apartment no.)"}, {"x": 3, "y": 4, "w": 3, "h": 1, "content": "15 Country and ZIP or foreign postal code"}, {"x": 3, "y": 0, "w": 3, "h": 1, "content": "3 Policy issuer\u2019s name"}]
263264
},
264265
"text": "1 Marketplace identifier 2 Marketplace-assigned policy number 3 Policy issuer’s name 4 Recipient’s name 5 Recipient’s SSN 6 Recipient’s date of birth 7 Recipient’s spouse’s name 8 Recipient’s spouse’s SSN 9 Recipient’s spouse’s date of birth 10 Policy start date 11 Policy termination date 12 Street address (including apartment no.) 13 City or town 14 State or province 15 Country and ZIP or foreign postal code",
265266
"type": "Table"
@@ -304,7 +305,8 @@
304305
"eng"
305306
],
306307
"page_number": 2,
307-
"text_as_html": "<table><thead><th>A. Covered individual name</th><th>B. Covered individual SSN</th><th>C. Covered individual date of birth</th><th>D. Coverage start date</th><th>E. Coverage termination date</th></thead><tr><td colspan=\"5\">16</td></tr><tr><td colspan=\"5\">17</td></tr><tr><td>18</td><td></td><td></td><td></td><td></td></tr><tr><td colspan=\"5\">19</td></tr><tr><td>20</td><td></td><td></td><td></td><td></td></tr></table>"
308+
"text_as_html": "<table><thead><th>A. Covered individual name</th><th>B. Covered individual SSN</th><th>C. Covered individual date of birth</th><th>D. Coverage start date</th><th>E. Coverage termination date</th></thead><tr><td colspan=\"5\">16</td></tr><tr><td colspan=\"5\">17</td></tr><tr><td>18</td><td></td><td></td><td></td><td></td></tr><tr><td colspan=\"5\">19</td></tr><tr><td>20</td><td></td><td></td><td></td><td></td></tr></table>",
309+
"table_as_cells": [{"x": 0, "y": 0, "w": 1, "h": 1, "content": "A. Covered individual name"}, {"x": 0, "y": 3, "w": 1, "h": 1, "content": "18"}, {"x": 0, "y": 5, "w": 1, "h": 1, "content": "20"}, {"x": 1, "y": 0, "w": 1, "h": 1, "content": "B. Covered individual SSN"}, {"x": 1, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 1, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 0, "w": 1, "h": 1, "content": "C. Covered individual date of birth"}, {"x": 2, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 0, "w": 1, "h": 1, "content": "D. Coverage start date"}, {"x": 3, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 0, "w": 1, "h": 1, "content": "E. Coverage termination date"}, {"x": 4, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 0, "y": 1, "w": 5, "h": 1, "content": "16"}, {"x": 0, "y": 4, "w": 5, "h": 1, "content": "19"}, {"x": 0, "y": 2, "w": 5, "h": 1, "content": "17"}]
308310
},
309311
"text": "A. Covered individual name B. Covered individual SSN C. Covered individual date of birth D. Coverage start date E. Coverage termination date 16 17 18 19 20",
310312
"type": "Table"
@@ -349,7 +351,8 @@
349351
"eng"
350352
],
351353
"page_number": 2,
352-
"text_as_html": "<table><thead><th></th><th>Month A.</th><th>Monthly enrollment</th><th>premiums B. Monthly second lowest cost silver plan (SLCSP) premium</th><th>C. Monthly advance payment of premium tax credit</th></thead><tr><td>22</td><td>February</td><td></td><td></td><td></td></tr><tr><td>23</td><td>March</td><td></td><td></td><td></td></tr><tr><td>24</td><td>April</td><td></td><td></td><td></td></tr><tr><td>25</td><td>May</td><td></td><td></td><td></td></tr><tr><td>26</td><td>June</td><td></td><td></td><td></td></tr><tr><td>27</td><td>July</td><td></td><td></td><td></td></tr><tr><td>28</td><td>August</td><td></td><td></td><td></td></tr><tr><td>29</td><td>September</td><td></td><td></td><td></td></tr><tr><td>30</td><td>October</td><td></td><td></td><td></td></tr><tr><td>31</td><td>November</td><td></td><td></td><td></td></tr><tr><td>32</td><td>December</td><td></td><td></td><td></td></tr></table>"
354+
"text_as_html": "<table><thead><th></th><th>Month A.</th><th>Monthly enrollment</th><th>premiums B. Monthly second lowest cost silver plan (SLCSP) premium</th><th>C. Monthly advance payment of premium tax credit</th></thead><tr><td>22</td><td>February</td><td></td><td></td><td></td></tr><tr><td>23</td><td>March</td><td></td><td></td><td></td></tr><tr><td>24</td><td>April</td><td></td><td></td><td></td></tr><tr><td>25</td><td>May</td><td></td><td></td><td></td></tr><tr><td>26</td><td>June</td><td></td><td></td><td></td></tr><tr><td>27</td><td>July</td><td></td><td></td><td></td></tr><tr><td>28</td><td>August</td><td></td><td></td><td></td></tr><tr><td>29</td><td>September</td><td></td><td></td><td></td></tr><tr><td>30</td><td>October</td><td></td><td></td><td></td></tr><tr><td>31</td><td>November</td><td></td><td></td><td></td></tr><tr><td>32</td><td>December</td><td></td><td></td><td></td></tr></table>",
355+
"table_as_cells": [{"x": 0, "y": 0, "w": 1, "h": 1, "content": ""}, {"x": 0, "y": 1, "w": 1, "h": 1, "content": "22"}, {"x": 0, "y": 2, "w": 1, "h": 1, "content": "23"}, {"x": 0, "y": 3, "w": 1, "h": 1, "content": "24"}, {"x": 0, "y": 4, "w": 1, "h": 1, "content": "25"}, {"x": 0, "y": 5, "w": 1, "h": 1, "content": "26"}, {"x": 0, "y": 6, "w": 1, "h": 1, "content": "27"}, {"x": 0, "y": 7, "w": 1, "h": 1, "content": "28"}, {"x": 0, "y": 8, "w": 1, "h": 1, "content": "29"}, {"x": 0, "y": 9, "w": 1, "h": 1, "content": "30"}, {"x": 0, "y": 10, "w": 1, "h": 1, "content": "31"}, {"x": 0, "y": 11, "w": 1, "h": 1, "content": "32"}, {"x": 1, "y": 0, "w": 1, "h": 1, "content": "Month A."}, {"x": 1, "y": 1, "w": 1, "h": 1, "content": "February"}, {"x": 1, "y": 2, "w": 1, "h": 1, "content": "March"}, {"x": 1, "y": 3, "w": 1, "h": 1, "content": "April"}, {"x": 1, "y": 4, "w": 1, "h": 1, "content": "May"}, {"x": 1, "y": 5, "w": 1, "h": 1, "content": "June"}, {"x": 1, "y": 6, "w": 1, "h": 1, "content": "July"}, {"x": 1, "y": 7, "w": 1, "h": 1, "content": "August"}, {"x": 1, "y": 8, "w": 1, "h": 1, "content": "September"}, {"x": 1, "y": 9, "w": 1, "h": 1, "content": "October"}, {"x": 1, "y": 10, "w": 1, "h": 1, "content": "November"}, {"x": 1, "y": 11, "w": 1, "h": 1, "content": "December"}, {"x": 2, "y": 0, "w": 1, "h": 1, "content": "Monthly enrollment"}, {"x": 2, "y": 1, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 2, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 4, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 6, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 7, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 8, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 9, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 10, "w": 1, "h": 1, "content": ""}, {"x": 2, "y": 11, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 0, "w": 1, "h": 1, "content": "premiums B. Monthly second lowest cost silver plan (SLCSP) premium"}, {"x": 3, "y": 1, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 2, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 4, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 6, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 7, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 8, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 9, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 10, "w": 1, "h": 1, "content": ""}, {"x": 3, "y": 11, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 0, "w": 1, "h": 1, "content": "C. Monthly advance payment of premium tax credit"}, {"x": 4, "y": 1, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 2, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 3, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 4, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 5, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 6, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 7, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 8, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 9, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 10, "w": 1, "h": 1, "content": ""}, {"x": 4, "y": 11, "w": 1, "h": 1, "content": ""}]
353356
},
354357
"text": "Month A. Monthly enrollment premiums B. Monthly second lowest cost silver plan (SLCSP) premium C. Monthly advance payment of premium tax credit 21 January 22 February 23 March 24 April 25 May 26 June 27 July 28 August 29 September 30 October 31 November 32 December",
355358
"type": "Table"

‎test_unstructured/metrics/test_evaluate.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def test_table_structure_evaluation():
140140
assert os.path.isfile(os.path.join(export_dir, "aggregate-table-structure-accuracy.tsv"))
141141
df = pd.read_csv(os.path.join(export_dir, "all-docs-table-structure-accuracy.tsv"), sep="\t")
142142
assert len(df) == 1
143-
assert len(df.columns) == 10
143+
assert len(df.columns) == 17
144144
assert df.iloc[0].filename == "IRS-2023-Form-1095-A.pdf"
145145

146146

‎test_unstructured/metrics/test_table_structure.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,115 @@ def test_table_eval_processor_simple():
8989
assert result.element_col_level_content_acc == 1.0
9090

9191

92+
def test_table_eval_processor_simple_when_input_as_cells():
93+
prediction = [
94+
{
95+
"type": "Table",
96+
"metadata": {
97+
"table_as_cells": [
98+
{
99+
"x": 1,
100+
"y": 1,
101+
"w": 1,
102+
"h": 1,
103+
"content": "r2c2",
104+
},
105+
{
106+
"x": 0,
107+
"y": 0,
108+
"w": 1,
109+
"h": 1,
110+
"content": "r1c1",
111+
},
112+
{
113+
"x": 0,
114+
"y": 1,
115+
"w": 1,
116+
"h": 1,
117+
"content": "r2c1",
118+
},
119+
{
120+
"x": 1,
121+
"y": 0,
122+
"w": 1,
123+
"h": 1,
124+
"content": "r1c2",
125+
},
126+
]
127+
},
128+
}
129+
]
130+
131+
ground_truth = [
132+
{
133+
"type": "Table",
134+
"text": [
135+
{
136+
"id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
137+
"x": 0,
138+
"y": 0,
139+
"w": 1,
140+
"h": 1,
141+
"content": "r1c1",
142+
},
143+
{
144+
"id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
145+
"x": 0,
146+
"y": 1,
147+
"w": 1,
148+
"h": 1,
149+
"content": "r2c1",
150+
},
151+
{
152+
"id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
153+
"x": 1,
154+
"y": 0,
155+
"w": 1,
156+
"h": 1,
157+
"content": "r1c2",
158+
},
159+
{
160+
"id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
161+
"x": 1,
162+
"y": 1,
163+
"w": 1,
164+
"h": 1,
165+
"content": "r2c2",
166+
},
167+
],
168+
}
169+
]
170+
171+
te_processor = TableEvalProcessor(prediction, ground_truth, source_type="cells")
172+
result = te_processor.process_file()
173+
assert result.total_tables == 1
174+
assert result.table_level_acc == 1.0
175+
assert result.element_row_level_index_acc == 1.0
176+
assert result.element_col_level_index_acc == 1.0
177+
assert result.element_row_level_content_acc == 1.0
178+
assert result.element_col_level_content_acc == 1.0
179+
180+
181+
def test_table_eval_processor_when_wrong_source_type():
182+
prediction = [
183+
{
184+
"type": "Table",
185+
"metadata": {"table_as_cells": []},
186+
}
187+
]
188+
189+
ground_truth = [
190+
{
191+
"type": "Table",
192+
"text": [],
193+
}
194+
]
195+
196+
te_processor = TableEvalProcessor(prediction, ground_truth, source_type="wrong_type")
197+
with pytest.raises(ValueError):
198+
te_processor.process_file()
199+
200+
92201
@pytest.mark.parametrize(
93202
"text_as_html",
94203
[

‎test_unstructured/metrics/test_text_extraction.py‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
import pytest
44

55
from unstructured.metrics import text_extraction
6+
from unstructured.metrics.table.table_extraction import (
7+
extract_cells_from_table_as_cells,
8+
extract_cells_from_text_as_html,
9+
)
610
from unstructured.partition.auto import partition
711

812

@@ -155,7 +159,7 @@ def test_calculate_edit_distance_with_filename(filename, expected_score, expecte
155159
),
156160
(
157161
"""Sometimes sentences have a dash - like this one!
158-
A hyphen connects 2 words with no gap: easy-peasy.""",
162+
A hyphen connects 2 words with no gap: easy-peasy.""",
159163
{
160164
"sometimes": 1,
161165
"sentences": 1,
@@ -216,3 +220,29 @@ def test_calculate_percent_missing_text(output_text, source_text, expected_perce
216220
text_extraction.calculate_percent_missing_text(output_text, source_text)
217221
== expected_percentage
218222
)
223+
224+
225+
def test_cells_extraction_from_prediction_when_simple_example():
226+
example_element = {
227+
"type": "Table",
228+
"metadata": {
229+
"text_as_html": "<table><thead><th>Month A.</th></thead><tr><td>22</td></tr></table>",
230+
"table_as_cells": [
231+
{"x": 0, "y": 0, "w": 1, "h": 1, "content": "Month A."},
232+
{"x": 0, "y": 1, "w": 1, "h": 1, "content": "22"},
233+
],
234+
},
235+
}
236+
expected_extraction = [
237+
{"row_index": 0, "col_index": 0, "content": "Month A."},
238+
{"row_index": 1, "col_index": 0, "content": "22"},
239+
]
240+
241+
assert extract_cells_from_text_as_html(example_element) == expected_extraction
242+
assert extract_cells_from_table_as_cells(example_element) == expected_extraction
243+
244+
245+
def test_cells_extraction_from_prediction_when_missing_prediction():
246+
example_element = {"type": "Table", "metadata": {"text_as_html": "", "table_as_cells": []}}
247+
assert extract_cells_from_text_as_html(example_element) is None
248+
assert extract_cells_from_table_as_cells(example_element) is None

‎unstructured/__version__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.13.7-dev7" # pragma: no cover
1+
__version__ = "0.13.7-dev8" # pragma: no cover

‎unstructured/metrics/evaluate.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -331,26 +331,41 @@ def measure_table_structure_accuracy(
331331
logger.warning(f"Ground truth file {ground_truth_file} does not exist, skipping")
332332
continue
333333

334-
processor = TableEvalProcessor.from_json_files(
334+
processor_from_text_as_html = TableEvalProcessor.from_json_files(
335335
prediction_file=prediction_file,
336336
ground_truth_file=ground_truth_file,
337337
cutoff=cutoff,
338+
source_type="html",
338339
)
339-
report = processor.process_file()
340+
report_from_html = processor_from_text_as_html.process_file()
341+
342+
processor_from_table_as_cells = TableEvalProcessor.from_json_files(
343+
prediction_file=prediction_file,
344+
ground_truth_file=ground_truth_file,
345+
cutoff=cutoff,
346+
source_type="cells",
347+
)
348+
report_from_cells = processor_from_table_as_cells.process_file()
349+
340350
rows.append(
341351
[
342352
out_filename,
343353
doctype,
344354
connector,
345355
]
346-
+ [getattr(report, metric) for metric in table_eval_metrics]
356+
+ [getattr(report_from_html, metric) for metric in table_eval_metrics]
357+
+ [getattr(report_from_cells, metric) for metric in table_eval_metrics]
347358
)
348359

360+
suffixed_table_eval_metrics = [f"{metric}_with_spans" for metric in table_eval_metrics]
361+
combined_table_metrics = table_eval_metrics + suffixed_table_eval_metrics
362+
349363
headers = [
350364
"filename",
351365
"doctype",
352366
"connector",
353-
] + table_eval_metrics
367+
] + combined_table_metrics
368+
354369
df = pd.DataFrame(rows, columns=headers)
355370
has_tables_df = df[df["total_tables"] > 0]
356371

@@ -360,7 +375,7 @@ def measure_table_structure_accuracy(
360375
).reset_index()
361376
else:
362377
element_metrics_results = {}
363-
for metric in table_eval_metrics:
378+
for metric in combined_table_metrics:
364379
metric_df = has_tables_df[has_tables_df[metric].notnull()]
365380
agg_metric = metric_df[metric].agg([_mean, _stdev, _pstdev, _count]).transpose()
366381
if agg_metric.empty:

0 commit comments

Comments
 (0)