ESP32: faster file and directory abstractions

Author: profezzorn

common/file_reader.h

@@ -102,7 +102,7 @@ class FileReader {
     }
 #endif
 #ifdef ENABLE_SD
-    new (&sd_file_) File;
+    new (&sd_file_) LSFS::LSFILE;
     type_ = TYPE_SD;
     sd_file_ = LSFS::Open(filename);
     if (sd_file_) {
@@ -127,7 +127,7 @@ class FileReader {
     }
 #endif
 #ifdef ENABLE_SD
-    new (&sd_file_) File;
+    new (&sd_file_) LSFS::LSFILE;
     type_ = TYPE_SD;
     sd_file_ = LSFS::OpenFast(filename);
     if (sd_file_) {
@@ -141,7 +141,7 @@ class FileReader {
   bool Create(const char* filename) {
     Close();
 #ifdef ENABLE_SD
-    new (&sd_file_) File;
+    new (&sd_file_) LSFS::LSFILE;
     sd_file_ = LSFS::OpenForWrite(filename);
     if (sd_file_) {
       type_ = TYPE_SD;
@@ -153,7 +153,7 @@ class FileReader {
   bool OpenRW(const char* filename) {
     Close();
 #ifdef ENABLE_SD
-    new (&sd_file_) File;
+    new (&sd_file_) LSFS::LSFILE;
     sd_file_ = LSFS::OpenRW(filename);
     if (sd_file_) {
       type_ = TYPE_SD;
@@ -457,7 +457,7 @@ class FileReader {
     TYPE_MEM
   } type_;
   union {
-    IF_SD(File sd_file_;)
+    IF_SD(LSFS::LSFILE sd_file_;)
     IF_SF(SerialFlashFile sf_file_;)
     MemFile mem_file_;
   };

common/lsfs.h

@@ -5,6 +5,15 @@
 
 #include "string_piece.h"
 
+#ifdef F_MAXPATH
+#define PO_MAXPATH F_MAXPATH
+#elif ESP32
+#define PO_MAXPATH 1024
+#else
+#define PO_MAXPATH 128
+#endif
+
+
 struct PathHelper {
   void Append(const char* name) {
     if (strlen(path_) && path_[strlen(path_)-1] != '/') {
@@ -56,13 +65,16 @@ struct PathHelper {
   void UndoDirname() {
     path_[strlen(path_)] = '/';
   }
+
+  void Slashify() {
+    size_t len = strlen(path_);
+    if (path_[len-1] == '/') return;
+    path_[len] = '/';
+    path_[len+1] = 0;
+  }
   operator const char*() const { return path_; }
   operator StringPiece() const { return StringPiece(path_); }
-#ifdef F_MAXPATH  
-  char path_[F_MAXPATH];
-#else
-  char path_[128];
-#endif
+  char path_[PO_MAXPATH];
 };
 
 
@@ -131,7 +143,7 @@ class File {
 
 class LSFS {
 public:
-  typedef File FILE;
+  typedef File LSFILE;
   static bool Begin() { return true; }
   static bool End() { return true; }
   static bool Exists(const char* path) {
@@ -196,7 +208,7 @@ class LSFS {
 
 class LSFS {
 public:
-  typedef File FILE;
+  typedef File LSFILE;
   static bool Begin() {
 #if defined(__MK64FX512__) || defined(__MK66FX1M0__)
     // Prefer the built-in sd card for Teensy 3.5/3.6 as it is faster.
@@ -283,7 +295,7 @@ class LSFS {
 
 class LSFS {
 public:
-  typedef File FILE;
+  typedef File LSFILE;
   static bool IsMounted() {
     return mounted_;
   }
@@ -444,27 +456,108 @@ class LSFS {
 bool LSFS::mounted_ = false;
 #elif defined(ESP32)
 
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+
 #include "FS.h"
 #include "SD_MMC.h"
 #include "SD.h"
 
+#include "linked_ptr.h"
+// Posix file primitives
+
+class DoCloseFile {
+public:
+  static void Free(FILE* f) { if(f) fclose(f); }
+};
+
+class DoCloseDir {
+public:
+  static void Free(DIR* dir) { if(dir) closedir(dir); }
+};
+
 class LSFS {
 public:
-  typedef File FILE;
+
+  class File : public Stream {
+  public:
+    File() : file_() {}
+    File(FILE* f) : file_(f) {
+      if (file_.get()) {
+	setvbuf(file_.get(), NULL, _IOFBF, 512);
+      }
+    }
+    operator bool() const { return !!file_; }
+    void close() { file_ = NULL; }
+    int read(uint8_t *dest, size_t bytes) {
+      return fread(dest, 1, bytes, file_.get());
+    }
+    int read() {
+      uint8_t c;
+      read(&c, 1);
+      return c;
+    }
+    size_t write(const uint8_t *dest, size_t bytes) override {
+      return fwrite(dest, 1, bytes, file_.get());
+    }
+    size_t write(uint8_t c) override {
+      return write(&c, 1);
+    }
+    void seek(size_t pos) {
+      fseek(file_.get(), pos, SEEK_SET);
+    }
+    uint32_t position() {
+      return ftell(file_.get());
+    }
+    // Warning: slow!
+    int available() {
+      long pos = position();
+      fseek(file_.get(), 0, SEEK_END);
+      long end = position();
+      seek(pos);
+      return end - pos;
+    }
+    // Warning: slow!
+    uint32_t size() {
+      long pos = position();
+      fseek(file_.get(), 0, SEEK_END);
+      long end = position();
+      seek(pos);
+      return end;
+    }
+    // Warning: slow!
+    int peek() {
+      long pos = position();
+      uint8_t ret;
+      read(&ret, 1);
+      seek(pos);
+      return ret;
+    }
+    LinkedPtr<FILE, DoCloseFile> file_;
+  };
+
+  typedef File LSFILE;
+
   static bool Begin() {
 
 #if 1
 #define SDCLASS SD_MMC
     SD_MMC.setPins(sdClkPin, sdCmdPin, sdD0Pin, sdD1Pin, sdD2Pin, sdD3Pin);
-    if (!SD_MMC.begin("/sdcard", false, false, SDMMC_FREQ_DEFAULT, /*maxOpenFiles=*/ 32)) return false;
+//    if (!SD_MMC.begin("/sdcard", false, false, SDMMC_FREQ_DEFAULT, /*maxOpenFiles=*/ 32)) return false;
+    if (!SD_MMC.begin("/sdcard", false, false, BOARD_MAX_SDMMC_FREQ, /*maxOpenFiles=*/ 32)) return false;
 
+    chdir("/sdcard");
 #endif
 
+    
 #if 0
 #define SDCLASS SD
     SPI.begin(sdClkPin, sdD0Pin, sdCmdPin);
     if (!SD.begin(sdD3Pin, SPI, /* frequency= */ 20000000, "/sd" , /* max_file=*/ 32)) return false;
-#endif    
+#endif
+
 
 //    if (!SDCLASS.begin()) return false;
     uint8_t cardType = SDCLASS.cardType();
@@ -476,88 +569,92 @@ class LSFS {
     return true;
   }
   static bool Exists(const char* path) {
-    return SDCLASS.exists(path);
+    PathHelper p("/sdcard", path);
+    struct stat s;
+    return stat(p, &s) == 0;
+    
+    // return SDCLASS.exists(path);
   }
   static bool Remove(const char* path) {
-    return SDCLASS.remove(path);
+    return unlink(path) == 0;
+    // return SDCLASS.remove(path);
   }
   static File Open(const char* path) {
-    if (!SDCLASS.exists(path)) return File();
-    File f = SDCLASS.open(path);
-    f.setBufferSize(512);
-    return f;
+    PathHelper p("/sdcard", path);
+    return fopen(p, "r");
   }
   static File OpenRW(const char* path) {
-    File f = SDCLASS.open(path, FILE_WRITE);
-    f.setBufferSize(512);
-    return f;
-    
+    PathHelper p("/sdcard", path);
+    File ret = fopen(p, "r+");
+    if (ret) return ret;
+    return OpenForWrite(p);
   }
   static File OpenFast(const char* path) {
-    // At some point, I put this check in here to make sure that the file
-    // exists before we try to open it, as opening directories and other
-    // weird files can cause open() to hang. However, this check takes
-    // too long, and causes audio underflows, so we're going to need a
-    // different approach to not opening directories and weird files. /Hubbe
-    // if (!SDCLASS.exists(path)) return File();
-    File f = SDCLASS.open(path);
-    f.setBufferSize(512);
-    return f;
+    PathHelper p("/sdcard", path);
+    return fopen(p, "r");
   }
   static File OpenForWrite(const char* path) {
-    File f =  SDCLASS.open(path, FILE_WRITE);
-    if (!f) {
-      PathHelper tmp(path);
-      if (tmp.Dirname()) {
-	SDCLASS.mkdir(tmp);
-	f =  SDCLASS.open(path, FILE_WRITE);
-      }
-    }
-    f.setBufferSize(512);
-    return f;
+    PathHelper p("/sdcard", path);
+    return fopen(p, "wct");
   }
+
   class Iterator {
   public:
-    explicit Iterator(const char* dirname) {
-      dir_ = SDCLASS.open(dirname);
-      if (dir_.isDirectory()) {
-	f_ = dir_.openNextFile();
+    explicit Iterator(const char* path) : path_("/sdcard", path) {
+      path_.Slashify();
+
+      dir_ = opendir(path_);
+      if (dir_.get() == nullptr) {
+	entry_ = nullptr;
+	return;
       }
+      entry_ = readdir(dir_.get());
     }
-    explicit Iterator(Iterator& other) {
-      dir_ = other.f_;
-      other.f_ = File();
-      f_ = dir_.openNextFile();
+
+    explicit Iterator(Iterator& other) : path_(other.path_, other.name()) {
+      path_.Slashify();
+      
+      dir_ = opendir(path_);
+      if (!dir_) {
+	STDERR << "Expected path does not exist: " << path_ << "\n";
+	entry_ = nullptr;
+	return;
+      }
+      entry_ = readdir(dir_.get());
     }
-    ~Iterator() {
-      dir_.close();
-      f_.close();
+
+    void operator++() { entry_ = readdir(dir_.get()); }
+    operator bool() { return !!entry_; }
+    bool isdir() {
+      struct stat s;
+      PathHelper filename(path_, entry_->d_name);
+      stat(filename, &s);
+      return S_ISDIR(s.st_mode);
     }
-    void operator++() {
-      f_.close();
-      f_ = dir_.openNextFile();
+    const char* name() { return entry_->d_name; }
+    size_t size() {
+      struct stat s;
+      PathHelper filename(path_, entry_->d_name);
+      stat(filename, &s);
+      return s.st_size;
     }
-    operator bool() { return f_; }
-    bool isdir() { return f_.isDirectory(); }
-    const char* name() { return f_.name(); }
-    size_t size() { return f_.size(); }
 
   private:
-    File dir_;
-    File f_;
+    PathHelper path_;
+    LinkedPtr<DIR, DoCloseDir> dir_;
+    dirent* entry_;
   };
 };
 
 
-
 #else
 
 // Standard arduino
 #include <SD.h>
 
 class LSFS {
 public:
-  typedef File FILE;
+  typedef File LSFILE;
   static bool Begin() {
     return SD.begin(sdCardSelectPin);
   }

common/sd_test.h

@@ -65,23 +65,24 @@ class SDTestHelper {
   void TestFile(const char* filename) {
     uint8_t block[512];
     uint32_t start_open = micros();
-    File f = LSFS::Open(filename);
+    LSFS::LSFILE f = LSFS::Open(filename);
     if (!f) {
       STDOUT << "Failed to open!";
     }
     open_histogram.count(micros() - start_open);
 
     int cnt = 0;
     uint32_t block_start = micros();
-    while (f.available()) {
+    while (true) {
       uint32_t start = micros();
-      f.read(block, 512);
+      uint32_t bytes_read = f.read(block, 512);
       uint32_t end = micros();
+      if (bytes_read < 512) break;
       histogram.count(end - start);
       if (++cnt == 128) {
         cnt = 0;
         uint32_t block_time = micros() - block_start;
-	int streams = (int)((64.0 / 88.2) / (block_time * 0.000001));
+	int streams = (int)((512 * 128 / 2 / 44100.0) / (block_time * 0.000001));
 	if (streams < 10) STDOUT << (char)('0' + streams);
 	else if (streams < 36) STDOUT << (char)('A' + streams - 10);
 	else STDOUT << '!';