Merge pull request #10 from OpenRefine/issue_1_5

FilesExtensions: Support for directory selection, Removed fileContent, Project name generation

Author: sunilnatraj

module/langs/translation-en.json

@@ -6,7 +6,7 @@
     "files-import/no-directory-selected": "No directory selected",
     "files-import/select-one": "Select one",
     "files-import/scanning": "Scanning for files",
-    "files-parsing/project-default-name": "fileList",
+    "files-parsing/project-default-name": "folder-details",
     "files-import/warning-directory-selection": "You must select 1 or more directories.",
     "files-import/preparing": "Preparing…",
     "files-import/creating": "Creating project…",

module/scripts/index/files-importing-controller.js

@@ -58,6 +58,7 @@ Refine.FilesImportingController.prototype.startImportingDocument = function(doc)
                     self._doc = doc;
                     self._jobID = data.jobID;
                     self._options = data2.options;
+                    self._projectName = data2.projectName;
 
                     self._showParsingPanel();
                 } else {
@@ -214,13 +215,14 @@ Refine.FilesImportingController.prototype._showParsingPanel = function() {
     delete self._doc;
     delete self._jobID;
     delete self._options;
+    delete self._projectName;
 
     self._createProjectUI.showSourceSelectionPanel();
   });
 
   this._parsingPanelElmts.createProjectButton.click(function() { self._createProject(); });
 
-  this._parsingPanelElmts.projectNameInput[0].value = $.i18n('files-parsing/project-default-name');
+  this._parsingPanelElmts.projectNameInput[0].value = self._projectName ? self._projectName : $.i18n('files-parsing/project-default-name');
 
   this._createProjectUI.showCustomPanel(this._parsingPanel);
   this._updatePreview();

src/main/java/org/openrefine/extensions/files/importer/FilesImporter.java

@@ -91,7 +91,6 @@ public static void loadData(Project project, ProjectMetadata metadata, Importing
         columns.add("filePath");
         columns.add("filePermissions");
         columns.add("sha256");
-        columns.add("fileContent");
         JSONUtilities.safePut(options, "columnNames", columns);
         JSONUtilities.safePut(options, "separator", ",");
 
@@ -141,9 +140,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
                             String fileExt = getFileExt(fileName);
                             String filePermissions = getFilePermissions(file);
                             String fileChecksum = calculateFileChecksum(file, "SHA-256");
-                            String fileContent = getFileContent(file);
 
-                            csvPrinter.printRecord(fileName, fileSize, fileExt, dateModified, dateCreated, author, filePath, filePermissions, fileChecksum, fileContent);
+                            csvPrinter.printRecord(fileName, fileSize, fileExt, dateModified, dateCreated, author, filePath, filePermissions, fileChecksum);
                         }
                     } catch (Exception e) {
                         logger.info("--- importDirectory. Error processing file: " + file + " - " + e.getMessage());
@@ -212,15 +210,28 @@ private static String bytesToHex(byte[] bytes) {
     }
 
     private static String getFileContent(Path path) {
-        if (Files.exists(path)) {
-            try {
-                String content = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
-                int maxBytes = Math.min(content.length(), 1 * fileContentSizeLimit); // Max 32KB
-                if ( canIncludeFileContent(content)) {
-                    return content.substring(0, maxBytes);
+        if (Files.exists(path) && Files.isReadable(path)) {
+            try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+                StringBuilder contentBuilder = new StringBuilder();
+                char[] buffer = new char[1024];
+                int bytesRead;
+                int totalBytesRead = 0;
+                int maxBytes = fileContentSizeLimit;
+
+                while ((bytesRead = reader.read(buffer, 0, Math.min(buffer.length, maxBytes - totalBytesRead))) != -1) {
+                    contentBuilder.append(buffer, 0, bytesRead);
+                    totalBytesRead += bytesRead;
+
+                    if (totalBytesRead >= maxBytes) {
+                        break;
+                    }
                 }
-            }
-            catch (IOException e) {
+
+                String content = contentBuilder.toString();
+                if (canIncludeFileContent(content)) {
+                    return content;
+                }
+            } catch (IOException e) {
                 logger.info("--- importDirectory. Failed to read file content: " + e.getMessage());
             }
         }
@@ -323,4 +334,20 @@ private static void buildDirectoryNode(Path dir, JsonGenerator jsonGenerator) th
         }
     }
 
+    public static String generateProjectName(ArrayNode directoryInput)  {
+        if ( directoryInput == null || directoryInput.isEmpty() ) {
+            return "folder-details";
+        }
+        String folder1 = Paths.get(directoryInput.get(0).get("directory").asText()).getFileName().toString();
+        String folder2 = directoryInput.size() > 1 ? Paths.get(directoryInput.get(1).get("directory").asText()).getFileName().toString() : null;
+
+        if (folder2 == null) {
+            return String.format("folder-details_%s", folder1);
+        } else if (directoryInput.size() > 2) {
+            return String.format("folder-details_%s_%s_and_more", folder1, folder2);
+        } else {
+            return String.format("folder-details_%s_%s", folder1, folder2);
+        }
+    }
+
 }

src/main/java/org/openrefine/extensions/files/importer/FilesImportingController.java

@@ -100,10 +100,15 @@ private void doInitializeParserUI(HttpServletRequest request, HttpServletRespons
             logger.debug("::doInitializeParserUI::");
         }
 
+        ArrayNode directoryInput = ParsingUtilities.evaluateJsonStringToArrayNode(
+                parameters.getProperty("directoryJsonValue"));
+        String projectName = FilesImporter.generateProjectName(directoryInput);
+
         ObjectNode result = ParsingUtilities.mapper.createObjectNode();
         ObjectNode options = ParsingUtilities.mapper.createObjectNode();
         JSONUtilities.safePut(result, "status", "ok");
         JSONUtilities.safePut(result, "options", options);
+        JSONUtilities.safePut(result, "projectName", projectName);
 
         JSONUtilities.safePut(options, "skipDataLines", 0);
         if(logger.isDebugEnabled()) {

src/test/java/org/google/refine/filesExtension/importer/FilesImportingControllerTest.java

@@ -153,37 +153,31 @@ private void validateLocalDirectoryTestResults(Row row) {
             Assert.assertEquals(row.getCellValue(2).toString(), "csv");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertNotNull(row.getCellValue(9));
         }
         else if ( fileCellValue.startsWith("movies")) {
             Assert.assertEquals(row.getCellValue(2).toString(), "tsv");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertNotNull(row.getCellValue(9));
         }
         else if ( fileCellValue.startsWith("dates")) {
             Assert.assertEquals(row.getCellValue(2).toString(), "xls");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertEquals(row.getCellValue(9).toString().trim().length(), 0);
         }
         else if ( fileCellValue.startsWith("euc-jp")) {
             Assert.assertEquals(row.getCellValue(2).toString(), "html");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertNotNull(row.getCellValue(9));
         }
         else if ( fileCellValue.startsWith("archive")) {
             Assert.assertEquals(row.getCellValue(2).toString(), "zip");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertEquals(row.getCellValue(9).toString().trim().length(), 0);
         }
         else if ( fileCellValue.startsWith("persons")) {
             Assert.assertEquals(row.getCellValue(2).toString(), "gz");
             Assert.assertNotNull(row.getCellValue(7));
             Assert.assertNotNull(row.getCellValue(8));
-            Assert.assertEquals(row.getCellValue(9).toString().trim().length(), 0);
         }
         else {
             Assert.fail("Test failed : unknown record - " + row.toString());