Skip to content

Commit 3a06109

Browse files
blizzzblizzz
authored
Merge pull request #1446 from nextcloud/fix/1440/import-types
fix and improve detection and import of ods, xlsx and csv documents
2 parents 9ea4efb + 36179dc commit 3a06109

15 files changed

+194
-40
lines changed

‎lib/Service/ColumnTypes/DatetimeDateBusiness.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class DatetimeDateBusiness extends SuperBusiness implements IColumnTypeBusiness
1717
* @return string
1818
*/
1919
public function parseValue($value, ?Column $column = null): string {
20-
return json_encode($this->isValidDate($value, 'Y-m-d') ? $value : '');
20+
return json_encode($this->isValidDate((string)$value, 'Y-m-d') ? (string)$value : '');
2121
}
2222

2323
/**
@@ -26,7 +26,7 @@ public function parseValue($value, ?Column $column = null): string {
2626
* @return bool
2727
*/
2828
public function canBeParsed($value, ?Column $column = null): bool {
29-
return $this->isValidDate($value, 'Y-m-d');
29+
return $this->isValidDate((string)$value, 'Y-m-d');
3030
}
3131

3232
}

‎lib/Service/ColumnTypes/DatetimeTimeBusiness.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class DatetimeTimeBusiness extends SuperBusiness implements IColumnTypeBusiness
1717
* @return string
1818
*/
1919
public function parseValue($value, ?Column $column = null): string {
20-
return json_encode($this->isValidDate($value, 'H:i') ? $value : '');
20+
return json_encode($this->isValidDate((string)$value, 'H:i') ? $value : '');
2121
}
2222

2323
/**
@@ -26,7 +26,7 @@ public function parseValue($value, ?Column $column = null): string {
2626
* @return bool
2727
*/
2828
public function canBeParsed($value, ?Column $column = null): bool {
29-
return $this->isValidDate($value, 'H:i');
29+
return $this->isValidDate((string)$value, 'H:i');
3030
}
3131

3232
}

‎lib/Service/ColumnTypes/SelectionCheckBusiness.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
use OCA\Tables\Db\Column;
1111

1212
class SelectionCheckBusiness extends SuperBusiness implements IColumnTypeBusiness {
13-
public const PATTERN_POSITIVE = ['yes', '1', true, 1, 'true'];
14-
public const PATTERN_NEGATIVE = ['no', '0', false, 0, 'false'];
13+
public const PATTERN_POSITIVE = ['yes', '1', true, 1, 'true', 'TRUE'];
14+
public const PATTERN_NEGATIVE = ['no', '0', false, 0, 'false', 'FALSE'];
1515

1616
/**
1717
* @param mixed $value

‎lib/Service/ImportService.php‎

Lines changed: 96 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ private function getPreviewData(Worksheet $worksheet): array {
136136
$columns[] = [
137137
'title' => $title,
138138
'type' => $this->rawColumnDataTypes[$colIndex]['type'],
139-
'subtype' => $this->rawColumnDataTypes[$colIndex]['subtype'],
139+
'subtype' => $this->rawColumnDataTypes[$colIndex]['subtype'] ?? null,
140140
'numberDecimals' => $this->rawColumnDataTypes[$colIndex]['number_decimals'] ?? 0,
141141
'numberPrefix' => $this->rawColumnDataTypes[$colIndex]['number_prefix'] ?? '',
142142
'numberSuffix' => $this->rawColumnDataTypes[$colIndex]['number_suffix'] ?? '',
@@ -154,13 +154,26 @@ private function getPreviewData(Worksheet $worksheet): array {
154154
$cellIterator = $row->getCellIterator();
155155
$cellIterator->setIterateOnlyExistingCells(false);
156156

157-
foreach ($cellIterator as $cellIndex => $cell) {
157+
foreach ($cellIterator as $cell) {
158158
$value = $cell->getValue();
159-
$colIndex = (int) $cellIndex;
159+
// $cellIterator`s index is based on 1, not 0.
160+
$colIndex = $cellIterator->getCurrentColumnIndex() - 1;
160161
$column = $this->columns[$colIndex];
161162

162163
if (($column && $column->getType() === 'datetime') || (is_array($columns[$colIndex]) && $columns[$colIndex]['type'] === 'datetime')) {
163-
$value = Date::excelToDateTimeObject($value)->format('Y-m-d H:i');
164+
if (isset($columns[$colIndex]['subtype']) && $columns[$colIndex]['subtype'] === 'date') {
165+
$format = 'Y-m-d';
166+
} elseif (isset($columns[$colIndex]['subtype']) && $columns[$colIndex]['subtype'] === 'time') {
167+
$format = 'H:i';
168+
} else {
169+
$format = 'Y-m-d H:i';
170+
}
171+
172+
try {
173+
$value = Date::excelToDateTimeObject($value)->format($format);
174+
} catch (\TypeError) {
175+
$value = (new \DateTimeImmutable($value))->format($format);
176+
}
164177
} elseif (($column && $column->getType() === 'number' && $column->getNumberSuffix() === '%')
165178
|| (is_array($columns[$colIndex]) && $columns[$colIndex]['type'] === 'number' && $columns[$colIndex]['numberSuffix'] === '%')) {
166179
$value = $value * 100;
@@ -285,8 +298,14 @@ public function import(?int $tableId, ?int $viewId, string $path, bool $createMi
285298
* @throws PermissionError
286299
*/
287300
private function loop(Worksheet $worksheet): void {
288-
$firstRow = $worksheet->getRowIterator()->current();
289-
$secondRow = $worksheet->getRowIterator()->seek(2)->current();
301+
$rowIterator = $worksheet->getRowIterator();
302+
$firstRow = $rowIterator->current();
303+
$rowIterator->next();
304+
if (!$rowIterator->valid()) {
305+
return;
306+
}
307+
$secondRow = $rowIterator->current();
308+
unset($rowIterator);
290309
$this->getColumns($firstRow, $secondRow);
291310

292311
if (empty(array_filter($this->columns))) {
@@ -361,8 +380,32 @@ private function createRow(Row $row): void {
361380

362381
$value = $cell->getValue();
363382
$hasData = $hasData || !empty($value);
383+
364384
if ($column->getType() === 'datetime') {
365-
$value = Date::excelToDateTimeObject($value)->format('Y-m-d H:i');
385+
if ($column->getType() === 'datetime' && $column->getSubtype() === 'date') {
386+
$format = 'Y-m-d';
387+
} elseif ($column->getType() === 'datetime' && $column->getSubtype() === 'time') {
388+
$format = 'H:i';
389+
} else {
390+
$format = 'Y-m-d H:i';
391+
}
392+
try {
393+
$value = Date::excelToDateTimeObject($value)->format($format);
394+
} catch (\TypeError) {
395+
$value = (new \DateTimeImmutable($value))->format($format);
396+
}
397+
} elseif ($column->getType() === 'datetime' && $column->getSubtype() === 'date') {
398+
try {
399+
$value = Date::excelToDateTimeObject($value)->format('Y-m-d');
400+
} catch (\TypeError) {
401+
$value = (new \DateTimeImmutable($value))->format('Y-m-d');
402+
}
403+
} elseif ($column->getType() === 'datetime' && $column->getSubtype() === 'time') {
404+
try {
405+
$value = Date::excelToDateTimeObject($value)->format('H:i');
406+
} catch (\TypeError) {
407+
$value = (new \DateTimeImmutable($value))->format('H:i');
408+
}
366409
} elseif ($column->getType() === 'number' && $column->getNumberSuffix() === '%') {
367410
$value = $value * 100;
368411
} elseif ($column->getType() === 'selection' && $column->getSubtype() === 'check') {
@@ -414,6 +457,8 @@ private function getColumns(Row $firstRow, Row $secondRow): void {
414457
$index = 0;
415458
$countMatchingColumnsFromConfig = 0;
416459
$countCreatedColumnsFromConfig = 0;
460+
$lastCellWasEmpty = false;
461+
$hasGapInTitles = false;
417462
foreach ($cellIterator as $cell) {
418463
if ($cell && $cell->getValue() !== null && $cell->getValue() !== '') {
419464
$title = $cell->getValue();
@@ -437,14 +482,29 @@ private function getColumns(Row $firstRow, Row $secondRow): void {
437482

438483
// Convert data type to our data type
439484
$dataTypes[] = $this->parseColumnDataType($secondRowCellIterator->current());
485+
if ($lastCellWasEmpty) {
486+
$hasGapInTitles = true;
487+
}
488+
$lastCellWasEmpty = false;
440489
} else {
441490
$this->logger->debug('No cell given or cellValue is empty while loading columns for importing');
491+
if ($cell->getDataType() === 'null') {
492+
// LibreOffice generated XLSX doc may have more empty columns in the first row.
493+
// Continue without increasing error count, but leave a marker to detect gaps in titles.
494+
$lastCellWasEmpty = true;
495+
continue;
496+
}
442497
$this->countErrors++;
443498
}
444499
$secondRowCellIterator->next();
445500
$index++;
446501
}
447502

503+
if ($hasGapInTitles) {
504+
$this->logger->info('Imported table is having a gap in column titles');
505+
$this->countErrors++;
506+
}
507+
448508
$this->rawColumnTitles = $titles;
449509
$this->rawColumnDataTypes = $dataTypes;
450510

@@ -468,9 +528,33 @@ private function parseColumnDataType(Cell $cell): array {
468528
'subtype' => 'line',
469529
];
470530

471-
if (Date::isDateTime($cell) || $originDataType === DataType::TYPE_ISO_DATE) {
531+
try {
532+
if ($value === false) {
533+
throw new \Exception('We do not accept `false` here');
534+
}
535+
$dateValue = new \DateTimeImmutable($value);
536+
} catch (\Exception) {
537+
}
538+
539+
if (isset($dateValue)
540+
|| Date::isDateTime($cell)
541+
|| $originDataType === DataType::TYPE_ISO_DATE) {
542+
// the formatted value stems from the office document and shows the original user intent
543+
$dateAnalysis = date_parse($formattedValue);
544+
$containsDate = $dateAnalysis['year'] !== false || $dateAnalysis['month'] !== false || $dateAnalysis['day'] !== false;
545+
$containsTime = $dateAnalysis['hour'] !== false || $dateAnalysis['minute'] !== false || $dateAnalysis['second'] !== false;
546+
547+
if ($containsDate && !$containsTime) {
548+
$subType = 'date';
549+
} elseif (!$containsDate && $containsTime) {
550+
$subType = 'time';
551+
} else {
552+
$subType = '';
553+
}
554+
472555
$dataType = [
473556
'type' => 'datetime',
557+
'subtype' => $subType,
474558
];
475559
} elseif ($originDataType === DataType::TYPE_NUMERIC) {
476560
if (str_contains($formattedValue, '%')) {
@@ -514,7 +598,10 @@ private function parseColumnDataType(Cell $cell): array {
514598
'type' => 'number',
515599
];
516600
}
517-
} elseif ($originDataType === DataType::TYPE_BOOL) {
601+
} elseif ($originDataType === DataType::TYPE_BOOL
602+
|| ($originDataType === DataType::TYPE_FORMULA
603+
&& in_array($formattedValue, ['FALSE', 'TRUE'], true))
604+
) {
518605
$dataType = [
519606
'type' => 'selection',
520607
'subtype' => 'check',

‎src/modules/modals/Import.vue‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
<p class="span">
4242
{{ t('tables', 'Supported formats: xlsx, xls, csv, html, xml') }}
4343
<br>
44-
{{ t('tables', 'First row of the file must contain column headings.') }}
44+
{{ t('tables', 'First row of the file must contain column headings without gaps.') }}
4545
</p>
4646
</div>
4747
</RowFormWrapper>

‎tests/integration/features/APIv1.feature‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -187,31 +187,36 @@ Feature: APIv1
187187
Then user deletes last created row
188188
Then user "participant1" deletes table with keyword "Rows check"
189189

190-
191-
@api1
192-
Scenario: Import csv table
193-
Given file "/import.csv" exists for user "participant1" with following data
194-
| Col1 | Col2 | Col3 | num | emoji | special |
195-
| Val1 | Val2 | Val3 | 1 | 💙 | Ä |
196-
| great | news | here | 99 | ⚠️ | Ö |
197-
Given table "Import test" with emoji "👨🏻‍💻" exists for user "participant1" as "base1"
198-
When user imports file "/import.csv" into last created table
190+
@api1 @import
191+
Scenario Outline: Import a document file
192+
Given user "participant1" uploads file "<importfile>"
193+
And table "Import test" with emoji "👨🏻‍💻" exists for user "participant1" as "base1"
194+
When user imports file "/<importfile>" into last created table
199195
Then import results have the following data
200-
| found_columns_count | 6 |
201-
| created_columns_count | 6 |
202-
| inserted_rows_count | 2 |
203-
| errors_count | 0 |
204-
Then table has at least following columns
205-
| Col1 |
206-
| Col2 |
207-
| Col3 |
208-
| num |
209-
| emoji |
210-
| special |
196+
| found_columns_count | 10 |
197+
| created_columns_count | 10 |
198+
| inserted_rows_count | 2 |
199+
| errors_count | 0 |
200+
Then table has at least following typed columns
201+
| Col1 | text |
202+
| Col2 | text |
203+
| Col3 | text |
204+
| num | number |
205+
| emoji | text |
206+
| special | text |
207+
| date | datetime |
208+
| truth | selection |
211209
Then table contains at least following rows
212-
| Col1 | Col2 | Col3 | num | emoji | special |
213-
| Val1 | Val2 | Val3 | 1 | 💙 | Ä |
214-
| great | news | here | 99 | ⚠️ | Ö |
210+
| Date and Time | Col1 | Col2 | Col3 | num | emoji | special | date | truth | time |
211+
| 2022-02-20 08:42 | Val1 | Val2 | Val3 | 1 | 💙 | Ä | 2024-02-24 | false | 18:48 |
212+
| 2016-06-01 13:37 | great | news | here | 99 | ⚠ | Ö | 2016-06-01 | true | 01:23 |
213+
214+
Examples:
215+
| importfile |
216+
| import-from-libreoffice.ods |
217+
| import-from-libreoffice.xlsx |
218+
| import-from-ms365.xlsx |
219+
| import-from-libreoffice.csv |
215220

216221
@api1
217222
Scenario: Create, edit and delete views

‎tests/integration/features/bootstrap/FeatureContext.php‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
use GuzzleHttp\Cookie\CookieJar;
1414
use GuzzleHttp\Exception\ClientException;
1515
use GuzzleHttp\Exception\GuzzleException;
16+
use GuzzleHttp\Psr7\Utils;
1617
use PHPUnit\Framework\Assert;
1718
use PHPUnit\Framework\ExpectationFailedException;
1819
use Psr\Http\Message\ResponseInterface;
@@ -64,6 +65,8 @@ class FeatureContext implements Context {
6465
private array $tableData = [];
6566
private array $viewData = [];
6667

68+
private $importColumnData = null;
69+
6770
// use CommandLineTrait;
6871
private CollectionManager $collectionManager;
6972

@@ -89,6 +92,7 @@ public function setUp() {
8992
* @AfterScenario
9093
*/
9194
public function cleanupUsers() {
95+
$this->importColumnData = null;
9296
$this->collectionManager->cleanUp();
9397
foreach ($this->createdUsers as $user) {
9498
$this->deleteUser($user);
@@ -467,8 +471,21 @@ public function columnsForNodeV2(string $nodeType, string $nodeName, ?TableNode
467471
// (((((((((((((((((((((((((((( END API v2 )))))))))))))))))))))))))))))))))))
468472

469473

474+
/**
475+
* @Given user :user uploads file :file
476+
*/
477+
public function uploadFile(string $user, string $file): void {
478+
$this->setCurrentUser($user);
479+
480+
$localFilePath = __DIR__ . '/../../resources/' . $file;
481+
482+
$url = sprintf('%sremote.php/dav/files/%s/%s', $this->baseUrl, $user, $file);
483+
$body = Utils::streamFor(fopen($localFilePath, 'rb'));
470484

485+
$this->sendRequestFullUrl('PUT', $url, $body);
471486

487+
Assert::assertEquals(201, $this->response->getStatusCode());
488+
}
472489

473490
// IMPORT --------------------------
474491

@@ -574,7 +591,7 @@ public function checkRowsExists(TableNode $table): void {
574591
$allValuesForColumn[] = $row[$indexForCol];
575592
}
576593
foreach ($table->getColumn($key) as $item) {
577-
Assert::assertTrue(in_array($item, $allValuesForColumn));
594+
Assert::assertTrue(in_array($item, $allValuesForColumn), sprintf('%s not in %s', $item, implode(', ', $allValuesForColumn)));
578595
}
579596
}
580597
}
@@ -1190,6 +1207,36 @@ public function tableColumns(?TableNode $body = null): void {
11901207
}
11911208
}
11921209

1210+
/**
1211+
* @Then table has at least following typed columns
1212+
*
1213+
* @param TableNode|null $body
1214+
*/
1215+
public function tableTypedColumns(?TableNode $body = null): void {
1216+
$this->sendRequest(
1217+
'GET',
1218+
'/apps/tables/api/1/tables/'.$this->tableId.'/columns'
1219+
);
1220+
1221+
$data = $this->getDataFromResponse($this->response);
1222+
Assert::assertEquals(200, $this->response->getStatusCode());
1223+
1224+
// check if no columns exists
1225+
if ($body === null) {
1226+
Assert::assertCount(0, $data);
1227+
return;
1228+
}
1229+
1230+
$colByTitle = [];
1231+
foreach ($data as $d) {
1232+
$colByTitle[$d['title']] = $d['type'];
1233+
}
1234+
foreach ($body->getRows() as $columnData) {
1235+
Assert::assertArrayHasKey($columnData[0], $colByTitle);
1236+
Assert::assertSame($columnData[1], $colByTitle[$columnData[0]], sprintf('Column "%s" has unexpected type "%s"', $columnData[0], $colByTitle[$columnData[0]]));
1237+
}
1238+
}
1239+
11931240
/**
11941241
* @Then user deletes last created column
11951242
*/

‎tests/integration/resources/import-from-libreoffice.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Date and Time,Col1,Col2,Col3,num,emoji,special,date,truth,time
2+
2022-02-20 08:42,Val1,Val2,Val3,1,💙,Ä,2024-02-24,false,18:48
3+
2016-06-01 13:37,great,news,here,99,⚠,Ö,2016-06-01,true,01:23

‎tests/integration/resources/import-from-libreoffice.csv.license‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
2+
SPDX-License-Identifier: AGPL-3.0-or-later
3+

‎tests/integration/resources/import-from-libreoffice.ods‎

16.8 KB
Binary file not shown.

0 commit comments

Comments
 (0)