From 850a1985e1735f7ee2078c07738ce42709159a2c Mon Sep 17 00:00:00 2001 From: marsman7 Date: Tue, 29 Apr 2025 23:10:39 +0200 Subject: [PATCH 1/5] add backup/restore function --- data/de_DE.json | 10 + data/en.json | 173 +++++++- lib/ZipStream/ZipStream.cpp | 784 ++++++++++++++++++++++++++++++++++++ lib/ZipStream/ZipStream.h | 247 ++++++++++++ src/hasp/hasp.cpp | 3 + src/hasp_config.cpp | 31 ++ src/hasp_filesystem.cpp | 118 +----- src/hasp_filesystem.h | 24 +- src/sys/svc/hasp_http.cpp | 186 +++++++++ 9 files changed, 1454 insertions(+), 122 deletions(-) create mode 100644 lib/ZipStream/ZipStream.cpp create mode 100644 lib/ZipStream/ZipStream.h diff --git a/data/de_DE.json b/data/de_DE.json index d88cadd2d..bb0c81b7f 100644 --- a/data/de_DE.json +++ b/data/de_DE.json @@ -50,6 +50,16 @@ "strict": "Strikt", "always": "Immer" }, + "backup": { + "title": "Datensicherung / -wiederherstellung", + "btn": "Datensicherung", + "nav": "Backup", + "backup": "Daten sichern", + "bakfile": "Datensicherungs Dateiname", + "restore": "Daten wiederherstellen", + "resfile": "Datei Wiederherstellung", + "overwrite": "Vorhandene Datei überschrieben" + }, "editor": { "title": "Datei Editor", "btn": "Datei Editor", diff --git a/data/en.json b/data/en.json index 248e4adcd..1322ba7a0 100644 --- a/data/en.json +++ b/data/en.json @@ -1 +1,172 @@ -{"en":{"language":"English","home":{"title":"Main Menu","btn":"Main Menu","nav":"Home"},"save":"Save Settings","user":"Username","pass":"Password","hasp":{"title":"HASP Design","btn":"HASP Design","theme":"UI Theme","color1":"Primary color","color2":"Secondary color","pages":"Start Layout","font":"Default Font","startpage":"Startup Page","startdim":"Startup Dim"},"screenshot":{"title":"Screenshot","btn":"Screenshot","nav":"Screenshot","prev":"Prev Page","next":"Next Page","refresh":"Refresh"},"info":{"title":"Information","btn":"Information","nav":"Information"},"config":{"title":"Configuration","btn":"Configuration","nav":"Settings"},"ota":{"title":"Firmware Update","btn":"Firmware Update","nav":"Firmware","submit":"Update Firmware","file":"Firmware File","url":"Firmware URL","redirect":"Follow Redirects","never":"Never","strict":"Strict","always":"Always"},"editor":{"title":"File Editor","btn":"File Editor","nav":"File Editor"},"reset":{"title":"Factory Reset","btn":"Factory Reset","warning":"Warning","message":"This process will reset all settings to the default values. The internal flash will be erased and the device is restarted. You may need to connect to the WiFi AP displayed on the panel to reconfigure the device before accessing it again.","fileloss":"ALL FILES WILL BE LOST!"},"reboot":{"title":"Rebooting...","btn":"Restart","nav":"Reboot","message":"The device is rebooting."},"about":{"credits":"Based on the previous work of the following open source developers:","copyright":"Copyright ","rights":"All rights reserved.","clause1":"Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:","clause2":"The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.","clause3":"THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.","mit":"MIT License","bsd":"BSD License","freebsd":"FreeBSD License","apache2":"Apache2 License"},"wifi":{"title":"Wifi Settings","btn":"Wifi Settings","ssid":"SSID"},"wg":{"title":"WireGuard Settings","btn":"WireGuard Settings","vpnip":"VPN IP","privkey":"Private Key","host":"Remote IP","port":"Remote Port","pubkey":"Remote Public Key"},"mqtt":{"title":"MQTT Settings","btn":"MQTT Settings","name":"Hostname","group":"Groupname","host":"Broker","port":"Port","node_t":"Node Topic","group_t":"Group Topic","broadcast_t":"Broadcast Topic","hass_t":"HA LWT Topic"},"http":{"title":"HTTP Settings","btn":"HTTP Settings"},"ftp":{"title":"FTP Settings","btn":"FTP Settings","port":"FTP Port","pasv":"Passive Port"},"gui":{"title":"Display Settings","btn":"Display Settings","antiburn":"Antiburn","calibrate":"Calibrate"},"gpio":"GPIO Settings","debug":{"title":"Debug Settings","btn":"Debug Settings","baud":"Baudrate","tele":"Tele Period","ansi":"Use ANSI codes","host":"Syslog Server","port":"Syslog Port","ietf":"IETF (RFC 5424)","bsd":"BSD (RFC 3164)","log":"Facility"},"time":{"title":"Time Settings","btn":"Time Settings","region":"Region","zone":"Timezone","tz":"Timezone","ntp":"NTP Servers"},"region":{"etc":"Etcetera ","continents":"Continents ","af":"Africa ","as":"Asia ","au":"Australia ","aq":"Antarctica ","eu":"Europe ","na":"North America ","sa":"South America ","islands":"Islands ","at":"Atlantic Ocean ","in":"Indian Ocean ","pa":"Pacific Ocean "}}} \ No newline at end of file +{ + "en": { + "language": "English", + "home": { + "title": "Main Menu", + "btn": "Main Menu", + "nav": "Home" + }, + "save": "Save Settings", + "user": "Username", + "pass": "Password", + "hasp": { + "title": "HASP Design", + "btn": "HASP Design", + "theme": "UI Theme", + "color1": "Primary color", + "color2": "Secondary color", + "pages": "Start Layout", + "font": "Default Font", + "startpage": "Startup Page", + "startdim": "Startup Dim" + }, + "screenshot": { + "title": "Screenshot", + "btn": "Screenshot", + "nav": "Screenshot", + "prev": "Prev Page", + "next": "Next Page", + "refresh": "Refresh" + }, + "info": { + "title": "Information", + "btn": "Information", + "nav": "Information" + }, + "config": { + "title": "Configuration", + "btn": "Configuration", + "nav": "Settings" + }, + "ota": { + "title": "Firmware Update", + "btn": "Firmware Update", + "nav": "Firmware", + "submit": "Update Firmware", + "file": "Firmware File", + "url": "Firmware URL", + "redirect": "Follow Redirects", + "never": "Never", + "strict": "Strict", + "always": "Always" + }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, + "editor": { + "title": "File Editor", + "btn": "File Editor", + "nav": "File Editor" + }, + "reset": { + "title": "Factory Reset", + "btn": "Factory Reset", + "warning": "Warning", + "message": "This process will reset all settings to the default values. The internal flash will be erased and the device is restarted. You may need to connect to the WiFi AP displayed on the panel to reconfigure the device before accessing it again.", + "fileloss": "ALL FILES WILL BE LOST!" + }, + "reboot": { + "title": "Rebooting...", + "btn": "Restart", + "nav": "Reboot", + "message": "The device is rebooting." + }, + "about": { + "credits": "Based on the previous work of the following open source developers:", + "copyright": "Copyright ", + "rights": "All rights reserved.", + "clause1": "Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files(the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:", + "clause2": "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.", + "clause3": "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.", + "mit": "MIT License", + "bsd": "BSD License", + "freebsd": "FreeBSD License", + "apache2": "Apache2 License" + }, + "wifi": { + "title": "Wifi Settings", + "btn": "Wifi Settings", + "ssid": "SSID" + }, + "wg": { + "title": "WireGuard Settings", + "btn": "WireGuard Settings", + "vpnip": "VPN IP", + "privkey": "Private Key", + "host": "Remote IP", + "port": "Remote Port", + "pubkey": "Remote Public Key" + }, + "mqtt": { + "title": "MQTT Settings", + "btn": "MQTT Settings", + "name": "Hostname", + "group": "Groupname", + "host": "Broker", + "port": "Port", + "node_t": "Node Topic", + "group_t": "Group Topic", + "broadcast_t": "Broadcast Topic", + "hass_t": "HA LWT Topic" + }, + "http": { + "title": "HTTP Settings", + "btn": "HTTP Settings" + }, + "ftp": { + "title": "FTP Settings", + "btn": "FTP Settings", + "port": "FTP Port", + "pasv": "Passive Port" + }, + "gui": { + "title": "Display Settings", + "btn": "Display Settings", + "antiburn": "Antiburn", + "calibrate": "Calibrate" + }, + "gpio": "GPIO Settings", + "debug": { + "title": "Debug Settings", + "btn": "Debug Settings", + "baud": "Baudrate", + "tele": "Tele Period", + "ansi": "Use ANSI codes", + "host": "Syslog Server", + "port": "Syslog Port", + "ietf": "IETF (RFC 5424)", + "bsd": "BSD (RFC 3164)", + "log": "Facility" + }, + "time": { + "title": "Time Settings", + "btn": "Time Settings", + "region": "Region", + "zone": "Timezone", + "tz": "Timezone", + "ntp": "NTP Servers" + }, + "region": { + "etc": "Etcetera ", + "continents": "Continents ", + "af": "Africa ", + "as": "Asia ", + "au": "Australia ", + "aq": "Antarctica ", + "eu": "Europe ", + "na": "North America ", + "sa": "South America ", + "islands": "Islands ", + "at": "Atlantic Ocean ", + "in": "Indian Ocean ", + "pa": "Pacific Ocean " + } + } + } \ No newline at end of file diff --git a/lib/ZipStream/ZipStream.cpp b/lib/ZipStream/ZipStream.cpp new file mode 100644 index 000000000..6385c16cf --- /dev/null +++ b/lib/ZipStream/ZipStream.cpp @@ -0,0 +1,784 @@ +/**************************************************************************//** + * @file ZipStream.cpp + * @brief This library pack and unpack a ZIP-archive without store the archive + * in file system. Only uncompressed ZIP archives are supported (Store + * mode). ZIP and Unzip can not use the same instance on the same time. + * + * Under linux the archive can be created with the following command : + * zip -0 foobar.zip * + * and unpacked with + * unzip foobar.zip + * + * + * ZIP : The ZIP archive is transferred to the library as a stream. + * This can be a file or a web stream. + * At the beginning, a file list in JSON format must be passed to the + * "beginZip()" function. The file list must be in the following format: + * [{"name":"foo.txt"},{"name":"config.json"},{"name":"bar.png"}] + * + * Unzip : Existing files are overwritten. + * At the beginning, the "beginUnZip()" function must be passed. + * + * + * @version 0.9.0 + * @date 2025-04-14 + * + * @copyright Copyright (c) 2025 + * + ************************************************************************** */ + + +#if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 + +#include "ZipStream.h" +#include "rom/crc.h" + +#if ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 0) +#include +#define HASP_FS LittleFS +#else +#include "LITTLEFS.h" +#include "esp_littlefs.h" +#define HASP_FS LITTLEFS +#endif // ESP_ARDUINO_VERSION + +#define ZIP_LFH_SIGNATURE 0x04034b50 +#define ZIP_CDFH_SIGNATURE 0x02014b50 +#define ZIP_EOCD_SIGNATURE 0x06054b50 +#define ZIP_DD_SIGNATURE 0x08074b50 + +#define TAG_ZIP "ZIP" +#define ZIP_STREAM_BUFFER_SIZE 512 +#define ZIP_BUFFER_SIZE 512 +#define CRC_BUFFER_SIZE 512 + +static const char zip_ok[] PROGMEM = "OK"; +static const char zip_memory_error[] PROGMEM = "Failed to allocate memory"; +static const char zip_compreession_unsupport[] PROGMEM = "Compression is unsupported"; +static const char zip_invalid_file_list_error[] PROGMEM = "File list error"; +static const char zip_file_not_found_error[] PROGMEM = "File not found"; +static const char zip_file_open_failed_error[] PROGMEM = "File open failed"; +static const char zip_filename_length_error[] PROGMEM = "Filename too long"; +static const char zip_file_invalid_data_error[] PROGMEM = "File invalid"; +static const char zip_file_crc_error[] PROGMEM = "File wrong CRC"; +static const char zip_invalid_stream_error[] PROGMEM = "Stream invalid"; +static const char zip_initialisation_error[] PROGMEM = "Initialisation wrong"; +static const char zip_unknown_error[] PROGMEM = "Unknown internal error"; + +/**************************************************************************//** + * @brief Initialises the class for a new unpacking process. Must always + * be called before the first write to the stream. + * + * @return zip_stream_error_t + ************************************************************************** */ +zip_stream_error_t ZipStream::beginUnZip() +{ + if (_streamState != ZIP_STREAM_INACTIVE) { + _lastErrorCode = ZIP_STREAM_MODE_WRONG; + _lastErrorString = String(zip_initialisation_error); + return _lastErrorCode; + } + + flush(); + + _position = 0; + _lastErrorCode = ZIP_STREAM_OK; + _lastErrorString = ""; + + _pBuffer = (uint8_t *)malloc(ZIP_BUFFER_SIZE); + + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + return _lastErrorCode; + } + + _streamState = ZIP_STREAM_EXPECT_SIGNATURE; + + return _lastErrorCode; +} + +/**************************************************************************//** + * @brief + * + * @param fileList + * @return zip_stream_error_t + ************************************************************************** */ +zip_stream_error_t ZipStream::beginZip(String *fileList) +{ + if (_streamState != ZIP_STREAM_INACTIVE) { + _lastErrorCode = ZIP_STREAM_MODE_WRONG; + _lastErrorString = String(zip_initialisation_error); + return _lastErrorCode; + } + + flush(); + + _position = 0; + _fileIdx = 0; + _lastErrorCode = ZIP_STREAM_OK; + _lastErrorString = ""; + + memset(&_pEndCentralDirRecord, 0, sizeof(zip_end_central_dir_record_t)); + + _jsonError = deserializeJson(_jsonDoc, *fileList); + if (_jsonError != DeserializationError::Ok) { + _lastErrorCode = ZIP_STREAM_FILE_LIST_FAIL; + _lastErrorString = String("JSON parse error: ") + _jsonError.c_str(); + } + if(!_jsonDoc.is() ) { // Only JsonArray is valid + _lastErrorCode = ZIP_STREAM_FILE_LIST_FAIL; + _lastErrorString = String("JSON is not an array"); + } + _jFileArray = _jsonDoc.as(); + if (!_jFileArray.size()) { + _lastErrorCode = ZIP_STREAM_FILE_LIST_FAIL; + _lastErrorString = String("No files in list"); + } + + if (_lastErrorCode != ZIP_STREAM_OK) { + _lastErrorString = String("File list error"); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return _lastErrorCode; + } + +// ESP_LOGI(TAG_FILE, F("File list size : %d | %s"), _jFileArray.size(), fileObj["name"].as().c_str()); + + _pBuffer = (uint8_t *)malloc(ZIP_BUFFER_SIZE); + + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + return _lastErrorCode; + } + + if (!_pFileDescriptorBuffer) { + _pFileDescriptorBuffer = (zip_file_discriptor_t *)malloc(sizeof(zip_file_discriptor_t) * _jFileArray.size()); + if (!_pFileDescriptorBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return _lastErrorCode; + } + } + + _streamState = ZIP_STREAM_LOCAL_HEADER; + buildBufferLocalFileHeader(); + + return ZIP_STREAM_OK; +} + +/**************************************************************************//** + * @brief Unzips a stream of data. + * + * @param zipStream - The stream to unzip e.g. a file or a web stream + * @return size_t + ************************************************************************** */ +size_t ZipStream::write(Stream &zipStream) +{ + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + return _lastErrorCode; + } + + if (_streamState <= ZIP_STREAM_INACTIVE) { + _lastErrorCode = ZIP_STREAM_MODE_WRONG; + _lastErrorString = String(zip_initialisation_error); + return 0; + } + + if(!dynamic_cast(&zipStream)) { + _lastErrorCode = ZIP_STREAM_NOT_VALID; + _lastErrorString = String(zip_invalid_stream_error); + return 0; + } + + uint8_t * _pInputBuffer = (uint8_t *)malloc(ZIP_STREAM_BUFFER_SIZE); + + if (!_pInputBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + return 1; + } + + size_t toRead = 0; + size_t bytesRead = 0; + size_t totalWritten = 0; + + while ( toRead = zipStream.available() ) { + if (toRead > ZIP_STREAM_BUFFER_SIZE) { toRead = ZIP_STREAM_BUFFER_SIZE; } + bytesRead = zipStream.readBytes(_pInputBuffer, toRead); + totalWritten += write(_pInputBuffer, bytesRead); + if (_lastErrorCode < ZIP_STREAM_OK) { // On error stop, on warning continue +// ESP_LOGE(TAG_ZIP, "Unzip error %d", _lastErrorCode); + break; + } + } + + if (_pInputBuffer) { + free(_pInputBuffer); + _pInputBuffer = NULL; + } + + return totalWritten; +} + +/**************************************************************************//** + * @brief Unzips a chunk of data. + * + * @param pInData - Pointer to incoming data of chunk + * @param size - Size of the chunk + * @return size_t + ************************************************************************** */ +size_t ZipStream::write(const uint8_t *pInData, size_t size) +{ + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + return _lastErrorCode; + } + + if (_streamState <= ZIP_STREAM_INACTIVE) { + _lastErrorCode = ZIP_STREAM_MODE_WRONG; + _lastErrorString = String(zip_initialisation_error); + return 0; + } + + size_t getBytes = 0; + size_t inPosition = 0; + + while (size > inPosition) { + switch (_streamState) { + case ZIP_STREAM_EXPECT_SIGNATURE : { + getBytes = sizeof(_signature) - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + if(_position < sizeof(_signature)) { break; } + + _signature = *(uint32_t *)_pBuffer; + _streamState = ZIP_STREAM_EXPECT_HEADER; + _position = 0; + break; + } + + case ZIP_STREAM_EXPECT_HEADER : { + switch(_signature) { + case ZIP_LFH_SIGNATURE: { // Local file header + getBytes = sizeof(zip_local_file_header_t) - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + if (_position < sizeof(zip_local_file_header_t)) { break; } + + _zipLocalHeader = *(zip_local_file_header_t *)_pBuffer; + _streamState = ZIP_STREAM_GET_FILENAME; + + if (_zipLocalHeader.compression_method != ZIP_NO_COMPRESSION) { +// ESP_LOGW(TAG_ZIP, "Compression is not supported %d", _zipLocalHeader.compression_method); + _lastErrorCode = ZIP_STREAM_COMPRESSION_ERROR; + _lastErrorString = String(zip_compreession_unsupport); + _streamState = ZIP_STREAM_SKIP_DATA; + } + + if (_zipLocalHeader.filename_length >= 255) { +// ESP_LOGW(TAG_ZIP, "Filename too long %d", _zipLocalHeader.filename_length); + _lastErrorCode = ZIP_STREAM_FILENAME_LENGTH; + _lastErrorString = String(zip_filename_length_error); + _streamState = ZIP_STREAM_SKIP_DATA; + } + + if (_streamState == ZIP_STREAM_SKIP_DATA) { + _datalen = _zipLocalHeader.filename_length + _zipLocalHeader.extra_length + _zipLocalHeader.compressed_size; + } + + _position = 0; + break; + } + + case ZIP_CDFH_SIGNATURE: { // Central directory file header + getBytes = sizeof(zip_central_dir_file_header_t) - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + + if(_position < sizeof(zip_central_dir_file_header_t)) { break; } + + zip_central_dir_file_header_t zipCentralDirHeader = *(zip_central_dir_file_header_t *)_pBuffer; + + _streamState = ZIP_STREAM_SKIP_DATA; + _datalen = zipCentralDirHeader.filename_length + zipCentralDirHeader.extra_length + zipCentralDirHeader.comment_length; + _position = 0; + break; + } + + case ZIP_EOCD_SIGNATURE: { // End of central directory record + getBytes = sizeof(zip_end_central_dir_record_t) - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + + if(_position < sizeof(zip_end_central_dir_record_t)) { break; } + + zip_end_central_dir_record_t zipEndCentralDirHeader = *(zip_end_central_dir_record_t *)_pBuffer; + + _streamState = ZIP_STREAM_INACTIVE; + _datalen = zipEndCentralDirHeader.comment_length; + _position = 0; + + return inPosition; + } + + case ZIP_DD_SIGNATURE: { // Data descriptor + getBytes = sizeof(zip_data_discriptor_t) - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + + if(_position < sizeof(zip_data_discriptor_t)) { break; } + + _streamState = ZIP_STREAM_EXPECT_SIGNATURE; + _position = 0; + break; + } + + default: { +// ESP_LOGE(TAG_ZIP, F("Invalid Data")); + _lastErrorCode = ZIP_STREAM_DATA_INVALID; + _lastErrorString = String(zip_file_invalid_data_error); + return inPosition; + } + } + break; + } + + case ZIP_STREAM_GET_FILENAME : { + getBytes = _zipLocalHeader.filename_length - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + memcpy(_pBuffer + _position, (uint8_t *)pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + + if(_position != _zipLocalHeader.filename_length) { break; } + + *(_pBuffer + _position) = 0; // add string-terminator + + String filename((char *)_pBuffer); + if (!filename.startsWith("/")) { + filename = "/" + filename; + } + + if (HASP_FS.exists(filename)) { +// ESP_LOGI(TAG_ZIP, "Overwrite existing file %s", filename.c_str()); + HASP_FS.remove(filename); + } + + _File = HASP_FS.open(filename, "w"); + if (!_File) { +// ESP_LOGE(TAG_ZIP, "File cannot created %s", filename.c_str()); + filename = String(""); + _streamState = ZIP_STREAM_SKIP_DATA; + _datalen = _zipLocalHeader.extra_length + _zipLocalHeader.compressed_size; + _position = 0; + _lastErrorCode = ZIP_STREAM_FILE_OPEN_FAILED; + _lastErrorString = String(zip_file_open_failed_error); + break; + } + + if (_zipLocalHeader.extra_length > 0) { + _streamState = ZIP_STREAM_GET_EXTRA_FIELD; + } else { + _streamState = ZIP_STREAM_STORE_DATA; + } + +// ESP_LOGI(TAG_ZIP, "Writing file %s", filename.c_str()); + filename = String(); + _position = 0; + _fileCRC32 = 0; + break; + } + + case ZIP_STREAM_GET_EXTRA_FIELD : { + getBytes = _zipLocalHeader.extra_length - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + _position += getBytes; + inPosition += getBytes; + + if (_position != _zipLocalHeader.extra_length) { break; } + + _streamState = ZIP_STREAM_STORE_DATA; + _position = 0; + break; + } + + case ZIP_STREAM_STORE_DATA : { + if (!_File) { + _lastErrorCode = ZIP_STREAM_UNKNOWN_ERROR; + _lastErrorString = String(zip_file_open_failed_error); + break; + } + + getBytes = _zipLocalHeader.compressed_size - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + _fileCRC32 = crc32_le(_fileCRC32, pInData + inPosition, getBytes); + _File.write(pInData + inPosition, getBytes); + _position += getBytes; + inPosition += getBytes; + + // Check if the file is finished + if (_position < _zipLocalHeader.compressed_size) { break; } + + _File.close(); + + if(_fileCRC32 != _zipLocalHeader.crc) { +// ESP_LOGI(TAG_ZIP, "File CRC does not match %x <=> %x", _fileCRC32, _zipLocalHeader.crc); +// _lastErrorCode = ZIP_STREAM_CRC_ERROR; +// _lastErrorString = String(zip_file_crc_error); + } + + _streamState = ZIP_STREAM_EXPECT_SIGNATURE; + _position = 0; + } + + case ZIP_STREAM_SKIP_DATA : { + getBytes = _datalen - _position; + if (getBytes > (size - inPosition)) { getBytes = (size - inPosition); } + _position += getBytes; + inPosition += getBytes; + + if (_position != _datalen) { break; } + + _streamState = ZIP_STREAM_EXPECT_SIGNATURE; + _position = 0; + break; + } + + default : { +// ESP_LOGE(TAG_ZIP, F("Invalid Data")); + _lastErrorCode = ZIP_STREAM_DATA_INVALID; + _lastErrorString = String(zip_file_invalid_data_error); + return inPosition; + } + } + } + + return inPosition; +} + +/**************************************************************************//** + * @brief Fill buffer with local file header + * + * @return zip_stream_error_t + ************************************************************************** */ +void ZipStream::buildBufferLocalFileHeader() { + _pFileDescriptorBuffer[_fileIdx].is_valid = false; + _position = 0; + +/* + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); +// ESP_LOGE(TAG_ZIP, "%s", _lastErrorString.c_str()); + return; + } +*/ + + *(uint32_t*)_pBuffer = ZIP_LFH_SIGNATURE; + _content_length = sizeof(uint32_t); + + zip_local_file_header_t *pZipLocalHeader = (zip_local_file_header_t *)(_pBuffer + _content_length); + pZipLocalHeader->min_version = 0x000A; // Version 2.0 + pZipLocalHeader->flags = 0x0000; // No flags + pZipLocalHeader->compression_method = ZIP_NO_COMPRESSION; + pZipLocalHeader->time_modified = 0x0000; // No time + pZipLocalHeader->date_modified = 0x0000; // No date + pZipLocalHeader->crc = 0; + pZipLocalHeader->extra_length = 0; + _content_length += sizeof(zip_local_file_header_t); + + String filename = _jFileArray[_fileIdx].as()["name"].as(); + if (filename.length() > 255) { + _lastErrorCode = ZIP_STREAM_FILENAME_LENGTH; + _lastErrorString = String(zip_filename_length_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + + memcpy(_pBuffer + _content_length, filename.c_str(), filename.length()); + _content_length += filename.length(); + pZipLocalHeader->filename_length = filename.length(); + + if (!filename.startsWith("/")) { filename = "/" + filename; } + + if (!HASP_FS.exists(filename)) { + _lastErrorCode = ZIP_STREAM_FILE_NOT_FOUND; + _lastErrorString = String(zip_file_not_found_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + + if (_File) { _File.close(); } + + _File = HASP_FS.open(filename, "r"); + if (!_File) { + _lastErrorCode = ZIP_STREAM_FILE_OPEN_FAILED; + _lastErrorString = String(zip_file_open_failed_error) + " " + filename; +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + + pZipLocalHeader->compressed_size = _File.size(); + pZipLocalHeader->uncompressed_size = _File.size(); + + uint32_t crc32 = 0; + size_t bytesRead = 0; + uint8_t *pCrcBuffer = (uint8_t *)malloc(CRC_BUFFER_SIZE); + + if (!pCrcBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + + while (_File.available()) { + bytesRead = _File.readBytes((char *)pCrcBuffer, sizeof(pCrcBuffer)); + crc32 = crc32_le(crc32, (uint8_t const *)pCrcBuffer, bytesRead); + } + + pZipLocalHeader->crc = crc32; + + _pFileDescriptorBuffer[_fileIdx].is_valid = true; + _pFileDescriptorBuffer[_fileIdx].crc = pZipLocalHeader->crc; + _pFileDescriptorBuffer[_fileIdx].compressed_size = pZipLocalHeader->compressed_size; + _pFileDescriptorBuffer[_fileIdx].relative_offset = _pEndCentralDirRecord.offset_central_dir_start; + + _pEndCentralDirRecord.offset_central_dir_start += _content_length + pZipLocalHeader->compressed_size; + + return; +} + +/**************************************************************************//** + * @brief Fill buffer with central directory file header + * + * @return zip_stream_error_t + ************************************************************************** */ +void ZipStream::buildBufferCentralDirFileHeader() { + if (!_pFileDescriptorBuffer[_fileIdx].is_valid) { + _lastErrorCode = ZIP_STREAM_FILE_INVALID; + _lastErrorString = String(zip_file_invalid_data_error); + return; + } + + _position = 0; + +/* + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); +// ESP_LOGE(TAG_ZIP, "%s", _lastErrorString.c_str()); + return; + } +*/ + + *(uint32_t*)_pBuffer = ZIP_CDFH_SIGNATURE; + _content_length = sizeof(uint32_t); + + zip_central_dir_file_header_t *pHeader = (zip_central_dir_file_header_t *)(_pBuffer + _content_length); + pHeader->version_made_by = 0x030A; // Unix, Zip Version 2.0 + pHeader->version_min = 0x000A; // Version 2.0 + pHeader->flags = 0x0000; // No flags + pHeader->compression_method = ZIP_NO_COMPRESSION; + pHeader->time_modified = 0x0000; // No time + pHeader->date_modified = 0x0000; // No date + pHeader->crc = _pFileDescriptorBuffer[_fileIdx].crc; + pHeader->compressed_size = _pFileDescriptorBuffer[_fileIdx].compressed_size; + pHeader->uncompressed_size = _pFileDescriptorBuffer[_fileIdx].compressed_size; + pHeader->filename_length = 0; + pHeader->extra_length = 0; + pHeader->comment_length = 0; + pHeader->disk_no = 0; + pHeader->internal_file_attr = 0; + pHeader->external_file_attr = 0; + pHeader->relative_offset = _pFileDescriptorBuffer[_fileIdx].relative_offset; + + _content_length += sizeof(zip_central_dir_file_header_t); + + String filename = _jFileArray[_fileIdx].as()["name"].as(); + if (filename.length() > 255) { + _lastErrorCode = ZIP_STREAM_FILENAME_LENGTH; + _lastErrorString = String(zip_filename_length_error) + " " + filename; +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + + memcpy(_pBuffer + _content_length, filename.c_str(), filename.length()); + _content_length += filename.length(); + pHeader->filename_length = filename.length(); + + _pEndCentralDirRecord.central_dir_records++; + _pEndCentralDirRecord.total_dir_records++; + _pEndCentralDirRecord.central_dir_size += _content_length; + + return; +} + +/**************************************************************************//** + * @brief Fill buffer with end of central directory record + * + * @return zip_stream_error_t + ************************************************************************** */ +void ZipStream::buildBufferEndCentralFileHeader() { + _position = 0; + +/* + if (!_pBuffer) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + return; + } + */ + + *(uint32_t*)_pBuffer = ZIP_EOCD_SIGNATURE; + _content_length = sizeof(uint32_t); + + memcpy(_pBuffer + _content_length, &_pEndCentralDirRecord, sizeof(zip_end_central_dir_record_t)); + + _content_length += sizeof(zip_end_central_dir_record_t); + + return; +} + +/**************************************************************************//** + * @brief read data from files in the file list, unpack them and put them + * into the buffer. + * + * @param buffer + * @param length + * @return size_t + ************************************************************************** */ +size_t ZipStream::readBytes(uint8_t *buffer, size_t length) { + if (!_pBuffer || _streamState >= ZIP_STREAM_INACTIVE) { + _lastErrorCode = ZIP_STREAM_MODE_WRONG; + _lastErrorString = String(zip_initialisation_error); + return 0; + } + + switch (_streamState) { + case ZIP_STREAM_LOCAL_HEADER: { + if (length > _content_length - _position) { length = _content_length - _position; } + + if (length) { memcpy(buffer, _pBuffer + _position, length); } + _position += length; + + if (_content_length - _position == 0) { + // Local file header is complete, continue with file data + _streamState = ZIP_STREAM_FILE_DATA; + _content_length = _File.size(); + _position = 0; + if (!_File) { + _lastErrorCode = ZIP_STREAM_FILE_OPEN_FAILED; + _lastErrorString = String(zip_file_open_failed_error); +// ESP_LOGE(TAG_ZIP, F(_lastErrorString)); + return 0; + } + + _File.seek(0, SeekSet); + } + break; + } + + case ZIP_STREAM_FILE_DATA: { + length = _File.readBytes((char *)buffer, length); + + if(!_File.available()) { + _File.close(); + + if (++_fileIdx < _jFileArray.size()) { + // continue with local file header of next file + _streamState = ZIP_STREAM_LOCAL_HEADER; + buildBufferLocalFileHeader(); + } else { + // All files processed, continue with central directory + _fileIdx = 0; + _streamState = ZIP_STREAM_CENTRAL_DIR_HEADER; + buildBufferCentralDirFileHeader(); + } + } + break; + } + + case ZIP_STREAM_CENTRAL_DIR_HEADER: { + if (length > _content_length - _position) { length = _content_length - _position; } + + if (length) { memcpy(buffer, _pBuffer + _position, length); } + _position += length; + + if (_content_length - _position == 0) { + if (++_fileIdx < _jFileArray.size()) { + // continue with central dir header of next file + _streamState = ZIP_STREAM_CENTRAL_DIR_HEADER; + buildBufferCentralDirFileHeader(); + } else { + _streamState = ZIP_STREAM_END_OF_CENTRAL_DIR; + buildBufferEndCentralFileHeader(); + } + } + break; + } + + case ZIP_STREAM_END_OF_CENTRAL_DIR: { + if (length > _content_length - _position) { length = _content_length - _position; } + + if (length) { memcpy(buffer, _pBuffer + _position, length); } + _position += length; + + if (_content_length - _position == 0) { + // All files processed, end of central directory + flush(); + } + break; + } + + default: + _lastErrorCode = ZIP_STREAM_UNKNOWN_ERROR; + _lastErrorString = String(zip_unknown_error); + return 0; + } + return length; +} + +/**************************************************************************//** + * @brief + * + ************************************************************************** */ +void ZipStream::flush() { + _streamState = ZIP_STREAM_INACTIVE; + _content_length = 0; + _position = 0; + _fileIdx = 0; + + if (_File) { + _File.close(); +// _File = fs::File(); + } + + if (_pBuffer) { + free(_pBuffer); + _pBuffer = NULL; + } + + if (_pFileDescriptorBuffer) { + free(_pFileDescriptorBuffer); + _pFileDescriptorBuffer = NULL; + } +} + +#endif // HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 \ No newline at end of file diff --git a/lib/ZipStream/ZipStream.h b/lib/ZipStream/ZipStream.h new file mode 100644 index 000000000..3ee20c6c4 --- /dev/null +++ b/lib/ZipStream/ZipStream.h @@ -0,0 +1,247 @@ +/**************************************************************************//** + * @file ZipStream.h + * @brief This library pack and unpack a ZIP-archive without store the archive + * in file system. Only uncompressed ZIP archives are supported (Store + * mode). ZIP and Unzip can not use the same instance on the same time. + * + * Under linux the archive can be created with the following command : + * zip -0 foobar.zip * + * and unpacked with + * unzip foobar.zip + * + * ZIP : The ZIP archive is transferred to the library as a stream. + * This can be a file or a web stream. + * At the beginning, a file list in JSON format must be passed to the + * "beginZip()" function. The file list must be in the following format: + * [{"name":"foo.txt"},{"name":"config.json"},{"name":"bar.png"}] + * + * Unzip : Existing files are overwritten. + * At the beginning, the "beginUnZip()" function must be passed. + * + * + * @version 0.9.0 + * @date 2025-04-14 + * + * @copyright Copyright (c) 2025 + * + ************************************************************************** */ + +#ifndef _ZIP_STREAM_H_ +#define _ZIP_STREAM_H_ + +#if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 + +#include "stdio.h" + +#include +#include "ArduinoJson.h" +#include "FS.h" + +#include +#include + +enum zip_stream_error_t { + // Errors, negative values + ZIP_STREAM_ERROR_NO_MEMORY = -1, + ZIP_STREAM_NOT_VALID = -2, + ZIP_STREAM_MODE_WRONG = -3, // Wrong mode Zip / UnZip + ZIP_STREAM_UNKNOWN_ERROR = -4, + // No error + ZIP_STREAM_OK = 0, + // Warnings, positive values + ZIP_STREAM_COMPRESSION_ERROR, + ZIP_STREAM_DATA_INVALID, + ZIP_STREAM_CRC_ERROR, + ZIP_STREAM_FILE_INVALID, + ZIP_STREAM_FILE_NOT_FOUND, + ZIP_STREAM_FILE_OPEN_FAILED, + ZIP_STREAM_FILENAME_LENGTH, + ZIP_STREAM_FILE_LIST_FAIL, + ZIP_STREAM_OVERWRITE_FILE +}; + +/**************************************************************************//** + * @brief class ZipStream + * + ************************************************************************** */ + +class ZipStream : public Stream { +private: + enum { ZIP_NO_COMPRESSION = 0, ZIP_DEFLTATE = 8 }; + typedef uint16_t zip_compression_method_t; + + enum zip_stream_state_t { + ZIP_STREAM_INACTIVE = 0, + // UnZip, positive values + ZIP_STREAM_EXPECT_SIGNATURE = 1, + ZIP_STREAM_EXPECT_HEADER, + ZIP_STREAM_HEADER_COMPLETE, + ZIP_STREAM_GET_FILENAME, + ZIP_STREAM_GET_EXTRA_FIELD, + ZIP_STREAM_STORE_DATA, + ZIP_STREAM_SKIP_DATA, + // Zip, negative values + ZIP_STREAM_LOCAL_HEADER = -1, + ZIP_STREAM_FILE_DATA = -2, + ZIP_STREAM_CENTRAL_DIR_HEADER = -3, + ZIP_STREAM_END_OF_CENTRAL_DIR = -4, + }; + + typedef struct __attribute__((packed)) + { + uint16_t min_version; + uint16_t flags; + zip_compression_method_t compression_method; + uint16_t time_modified; + uint16_t date_modified; + uint32_t crc; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_length; + uint16_t extra_length; + } zip_local_file_header_t; + + typedef struct __attribute__((packed)) + { + uint16_t version_made_by; + uint16_t version_min; + uint16_t flags; + zip_compression_method_t compression_method; + uint16_t time_modified; + uint16_t date_modified; + uint32_t crc; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_length; + uint16_t extra_length; + uint16_t comment_length; + uint16_t disk_no; + uint16_t internal_file_attr; + uint32_t external_file_attr; + uint32_t relative_offset; + } zip_central_dir_file_header_t; + + typedef struct __attribute__((packed)) + { + uint32_t crc; + uint32_t compressed_size; + uint32_t uncompressed_size; + } zip_data_discriptor_t; + + typedef struct __attribute__((packed)) + { + uint16_t disk_no_this; + uint16_t disk_dir_starts; + uint16_t central_dir_records; + uint16_t total_dir_records; + uint32_t central_dir_size; + uint32_t offset_central_dir_start; + uint16_t comment_length; + } zip_end_central_dir_record_t; + + typedef struct __attribute__((packed)) + { + bool is_valid; + uint32_t crc; + uint32_t compressed_size; + uint32_t relative_offset; + } zip_file_discriptor_t; + + + zip_stream_state_t _streamState; + zip_stream_error_t _lastErrorCode; + String _lastErrorString; + + uint8_t *_pBuffer; // Pointer to the internal buffer + size_t _position; // Current read position in the buffer + fs::File _File; // On zipping the ZIP-file; on unzipping the unpacked file + + // Unzip + size_t _datalen; + uint32_t _signature; + zip_local_file_header_t _zipLocalHeader; + uint32_t _fileCRC32; + + // Zip + StaticJsonDocument<2048> _jsonDoc; + DeserializationError _jsonError; + JsonArray _jFileArray; + + uint16_t _fileIdx; // Current file index + size_t _content_length; + zip_file_discriptor_t *_pFileDescriptorBuffer; + zip_end_central_dir_record_t _pEndCentralDirRecord; + + void buildBufferLocalFileHeader(); + void buildBufferCentralDirFileHeader(); + void buildBufferEndCentralFileHeader(); + +protected : + +public: + ZipStream() : + _streamState(ZIP_STREAM_INACTIVE), + _pBuffer(NULL), + _position(0), + + _content_length(0), + _fileIdx(0), + _pFileDescriptorBuffer(NULL), + + _signature(0), + _datalen(0), + _fileCRC32(0), + + _lastErrorCode(ZIP_STREAM_OK), + _lastErrorString("") + { } + + ~ZipStream() { flush(); } + + zip_stream_error_t getLastError() { return _lastErrorCode; } + String getLastErrorString( ) { return _lastErrorString; } + void resetLastError() { _lastErrorCode = ZIP_STREAM_OK; _lastErrorString = ""; } + + zip_stream_error_t beginZip(String *fileList); + zip_stream_error_t beginUnZip(); + + // Read packed data chunk (only for zipping) + size_t readBytes(uint8_t *buffer, size_t length); + size_t readBytes(char *buffer, size_t length) { return readBytes((uint8_t *) buffer, length); }; + + // Check how many bytes are available to read (only for zipping) + int available() override { + if (!_pBuffer || _streamState >= ZIP_STREAM_INACTIVE) { + return 0; // No buffer available + } + return _content_length - _position; + } + + // Read a single byte from the buffer (only for zipping) + // not implemented + int read() override { return -1; } + + // Peek at the next byte in the buffer without advancing the position (only for zipping) + // not implemented + int peek() override { return -1; } + + // Write a single byte to the buffer (only for unzipping) + // not implemented + size_t write(uint8_t byte) override { return 0; } + + // Unzip buffer (only for unzipping) + size_t write(const uint8_t *pInData, size_t size) override; + + // Unzip stream (only for unzipping) + size_t write(Stream &zipStream); + + // Flush the buffer (clear all data) + void flush() override; + + // Reset the read position to the beginning +// void reset() { _position = 0; } +}; + +#endif // HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 + +#endif // _ZIP_STREAM_H_ \ No newline at end of file diff --git a/src/hasp/hasp.cpp b/src/hasp/hasp.cpp index c02067648..00c669ae3 100644 --- a/src/hasp/hasp.cpp +++ b/src/hasp/hasp.cpp @@ -761,6 +761,9 @@ void hasp_get_info(JsonDocument& doc) hasp_get_sleep_payload(hasp_get_sleep_state(), size_buf); info[F("Idle")] = size_buf; info[F("Active Page")] = haspPages.get(); + snprintf(size_buf,sizeof(size_buf)-1,"%d.%d.%d",LVGL_VERSION_MAJOR,LVGL_VERSION_MINOR,LVGL_VERSION_PATCH); + buffer = size_buf; + info[F("LVGL Version")] = buffer; info = doc.createNestedObject(F(D_INFO_DEVICE_MEMORY)); Parser::format_bytes(haspDevice.get_free_heap(), size_buf, sizeof(size_buf)); diff --git a/src/hasp_config.cpp b/src/hasp_config.cpp index 75ac0bb5d..79970eaa0 100644 --- a/src/hasp_config.cpp +++ b/src/hasp_config.cpp @@ -8,6 +8,7 @@ #include "hasp_config.h" #include "hasp_debug.h" #include "hasp_gui.h" +#include "sys/net/hasp_time.h" #if HASP_TARGET_ARDUINO #include "hal/hasp_hal.h" #endif @@ -478,6 +479,26 @@ void configWrite() } #endif +#if HASP_USE_FTP > 0 + if(settings[FPSTR(FP_FTP)].as().isNull()) settings.createNestedObject(F("ftp")); + changed = ftpGetConfig(settings[FPSTR(FP_FTP)]); + if(changed) { + LOG_VERBOSE(TAG_FTP, settingsChanged.c_str()); + configOutput(settings[FPSTR(FP_FTP)], TAG_FTP); + writefile = true; + } +#endif + +#if HASP_USE_CONFIG > 0 + if(settings[FPSTR(FP_TIME)].as().isNull()) settings.createNestedObject(F("time")); + changed = timeGetConfig(settings[FPSTR(FP_TIME)]); + if(changed) { + LOG_VERBOSE(TAG_TIME, settingsChanged.c_str()); + configOutput(settings[FPSTR(FP_TIME)], TAG_TIME); + writefile = true; + } +#endif + #if HASP_USE_GPIO > 0 module = FPSTR(FP_GPIO); if(settings[module].as().isNull()) settings.createNestedObject(module); @@ -630,6 +651,16 @@ void configSetup() httpSetConfig(settings[FPSTR(FP_HTTP)]); #endif +#if HASP_USE_FTP > 0 + LOG_INFO(TAG_FTP, F("Loading FTP settings")); + ftpSetConfig(settings[FPSTR(FP_FTP)]); +#endif + +#if HASP_USE_CONFIG > 0 + LOG_INFO(TAG_TIME, F("Loading time settings")); + timeSetConfig(settings[FPSTR(FP_TIME)]); +#endif + #if HASP_USE_GPIO > 0 LOG_INFO(TAG_GPIO, F("Loading GPIO settings")); gpioSetConfig(settings[FPSTR(FP_GPIO)]); diff --git a/src/hasp_filesystem.cpp b/src/hasp_filesystem.cpp index 1f9f49532..6c4da8b38 100644 --- a/src/hasp_filesystem.cpp +++ b/src/hasp_filesystem.cpp @@ -31,124 +31,38 @@ extern const uint8_t PAGES_JSONL_END[] asm("_binary_data_pages_pages_jsonl_end") #endif #endif -#include #include "ArduinoJson.h" -#include "ArduinoLog.h" - #include "hasp_debug.h" #include "hasp_filesystem.h" +#include "ZipStream.h" + #if defined(ARDUINO_ARCH_ESP32) #include "rom/crc.h" void filesystemUnzip(const char*, const char* filename, uint8_t source) { + if (strlen(filename) < 1) { + LOG_ERROR(TAG_FILE, F("File name not available")); + return; + } + File zipfile = HASP_FS.open(filename, FILE_READ); - if(!zipfile) { + if (!zipfile) { + LOG_ERROR(TAG_FILE, F("File %s not found"), filename); return; } - int32_t head; - size_t len; - bool done = false; - - zipfile.seek(0); - while(!done) { - len = zipfile.read((uint8_t*)&head, sizeof(head)); - if(len != sizeof(head)) { - done = true; - continue; - } - - switch(head) { - case 0x04034b50: { - zip_file_header_t fh; - zipfile.seek(zipfile.position() - 2, SeekSet); // rewind for struct alignment (26-28) - len = zipfile.read((uint8_t*)(&fh), sizeof(zip_file_header_t)); - if(len != sizeof(zip_file_header_t)) { - done = true; - continue; - } - - if(fh.filename_length >= 255) { - LOG_WARNING(TAG_FILE, F("filename length too long %d"), fh.filename_length); - zipfile.seek(fh.filename_length + fh.extra_length, SeekCur); // skip extra field - continue; - // } else { - // LOG_WARNING(TAG_FILE, F("min %d - flag %d - len %d - xtra %d"), fh.min_version, fh.flags, - // fh.filename_length, fh.extra_length); - } - char name[257] = {0}; - name[0] = '/'; - - len = zipfile.read((uint8_t*)&name[1], fh.filename_length); - if(len != fh.filename_length) { - LOG_WARNING(TAG_FILE, F("filename read failed %d != %d"), fh.filename_length, len); - done = true; - continue; - } - zipfile.seek(fh.extra_length, SeekCur); // skip extra field - - if(fh.compression_method != ZIP_NO_COMPRESSION) { - LOG_WARNING(TAG_FILE, F("Compression is not supported %d"), fh.compression_method); - zipfile.seek(fh.compressed_size, SeekCur); // skip compressed file - } else { + ZipStream unzipStream; + if ( unzipStream.beginUnZip() == ZIP_STREAM_OK) { + unzipStream.write(zipfile); + } - if(HASP_FS.exists(name)) HASP_FS.remove(name); - - File f = HASP_FS.open(name, FILE_WRITE); - if(f) { - uint8_t buffer[512]; - uint32_t crc32 = 0; - - while(!done && fh.compressed_size >= 512) { - len = zipfile.readBytes((char*)&buffer, 512); - if(len != 512) done = true; - fh.compressed_size -= len; - crc32 = crc32_le(crc32, buffer, len); - f.write(buffer, len); - } - - if(!done && fh.compressed_size > 0) { - len = zipfile.readBytes((char*)&buffer, fh.compressed_size); - if(len != fh.compressed_size) done = true; - fh.compressed_size -= len; - crc32 = crc32_le(crc32, buffer, len); - f.write(buffer, len); - } - - if(crc32 != fh.crc) done = true; - - if(!done) { - Parser::format_bytes(fh.uncompressed_size, (char*)buffer, sizeof(buffer)); - LOG_VERBOSE(TAG_FILE, F(D_BULLET "%s (%s)"), name, buffer); - } else { - LOG_ERROR(TAG_FILE, F(D_FILE_SAVE_FAILED), name); - } - - f.close(); - } - } + zipfile.close(); - break; - } - case 0x02014b50: - done = true; - break; - case 0x06054b50: - // end of file - done = true; - break; - default: { - char outputString[9]; - itoa(head, outputString, 16); - LOG_WARNING(TAG_FILE, F("invalid %s"), outputString); - done = true; - } - } + if (unzipStream.getLastError() != ZIP_STREAM_OK) { + LOG_ERROR(TAG_FILE, F("Unpacking error %d - %s"), unzipStream.getLastError(), unzipStream.getLastErrorString().c_str() ); } - zipfile.close(); - LOG_VERBOSE(TAG_FILE, F("extracting %s complete"), filename); } #endif diff --git a/src/hasp_filesystem.h b/src/hasp_filesystem.h index 0299fc7f0..896847184 100644 --- a/src/hasp_filesystem.h +++ b/src/hasp_filesystem.h @@ -6,31 +6,17 @@ #include #include +#include "hasp_debug.h" -bool filesystemSetup(void); +#include +#include +#include +bool filesystemSetup(void); void filesystemList(); void filesystemInfo(); void filesystemSetupFiles(); -enum { ZIP_NO_COMPRESSION = 0, ZIP_DEFLTATE = 8 }; -typedef uint16_t zip_compression_method_t; - -typedef struct -{ - uint16_t dummy_bytes; // total struct needs to be a multiple of 4 bytes - uint16_t min_version; - uint16_t flags; - zip_compression_method_t compression_method; - uint16_t time_modified; - uint16_t date_modified; - uint32_t crc; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint16_t filename_length; // OK - uint16_t extra_length; -} zip_file_header_t; - #if defined(ARDUINO_ARCH_ESP32) #if HASP_USE_SPIFFS > 0 #include "SPIFFS.h" diff --git a/src/sys/svc/hasp_http.cpp b/src/sys/svc/hasp_http.cpp index 99f3c9816..52819d72e 100644 --- a/src/sys/svc/hasp_http.cpp +++ b/src/sys/svc/hasp_http.cpp @@ -27,6 +27,8 @@ #include "sys/net/hasp_network.h" #include "sys/net/hasp_time.h" +#include "ZipStream.h" + #if(HASP_USE_CAPTIVE_PORTAL > 0) && (HASP_USE_WIFI > 0) #include #endif @@ -440,6 +442,7 @@ static void http_handle_root() html[min(i++, len)] = R"()"; html[min(i++, len)] = R"()"; html[min(i++, len)] = R"()"; + html[min(i++, len)] = R"()"; html[min(i++, len)] = R"()"; #ifdef ARDUINO_ARCH_ESP32 html[min(i++, len)] = R"()"; @@ -2539,6 +2542,185 @@ static void webHandleFirmware() } } +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/**************************************************************************//** + * @brief Handle backup and restore + * When the backup is executed, the ‘config.json’ file is created and + * the data from this file is transferred to the configuration during + * the restore. No archive file is created during the backup in the + * file system. The archive is sent directly to the client. The archive + * is also not cached in the file system during the restore. + * + ************************************************************************** */ + +#if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 + +static void webHandleBackup() +{ + if(!http_is_authenticated("backup")) return; + + bool isError = false; + + if (webServer.method() == HTTP_POST) { +// LOG_INFO(TAG_HTTP, F("Backup : %s | %i"), webServer.uri().c_str(), webServer.args()); +// for (int i = 0; i < webServer.args(); i++) LOG_INFO(TAG_HTTP, F("Backup Arg %i = %s | %s"), i, webServer.argName(i).c_str(), webServer.arg(i).c_str()); + + if (webServer.hasArg("backup")) { + configWrite(); + + String dirjson = filesystem_list(HASP_FS, String("/").c_str(), 5); +// LOG_INFO(TAG_HTTP, F("File list : %s"), dirjson.c_str() ); + + String filename = String(haspDevice.get_hostname()) + String("_backup.zip"); + if (webServer.hasArg("filename")) { + filename = webServer.arg("filename"); + } + + uint8_t * pBuffer = (uint8_t *)malloc(1360); + if (!pBuffer) { + LOG_ERROR(TAG_HTTP, F("Failed to allocate memory")); + webServer.send(500, "Content-Type", PSTR("Failed to allocate memory")); + return; + } + + LOG_INFO(TAG_HTTP, F("Backup name : %s"), filename.c_str() ); + + webServer.sendHeader("content-disposition", "attachment; filename=" + filename); + webServer.sendHeader("Content-Type", "application/x-zip"); + webServer.sendHeader("Transfer-Encoding", "chunked"); + webServer.send(200, "", ""); // Send initial headers + + ZipStream zipstream; + zipstream.beginZip(&dirjson); + + size_t bytesRead; + + // Write data chunks into the zip stream + while (zipstream.available()) { + bytesRead = zipstream.readBytes(pBuffer, 1360); + if (bytesRead > 0) { + String chunkHeader = String(bytesRead, HEX) + "\r\n"; + webServer.sendContent(chunkHeader); + webServer.sendContent((const char*)pBuffer, bytesRead); + webServer.sendContent("\r\n"); + } + } + + webServer.sendContent("0\r\n\r\n"); + free(pBuffer); + pBuffer = NULL; + zipstream.flush(); + LOG_INFO(TAG_HTTP, F("Backup success")); + return; + } + + if (webServer.hasArg("restore")) { + LOG_INFO(TAG_HTTP, F("Restore success")); + if (HASP_FS.exists(FPSTR(FP_HASP_CONFIG_FILE))) { + configSetup(); + } + isError = true; + } + } + + const char* html[20]; + int i = 0; + int len = (sizeof(html) / sizeof(html[0])) - 1; + + html[min(i++, len)] = "

"; + html[min(i++, len)] = haspDevice.get_hostname(); + html[min(i++, len)] = "

"; + html[min(i++, len)] = R"( +

+ +
+
+
+
+
+
+ + +
+
+ +
+
+
+
+
+ + +
+
+ +
+
+ +)"; + html[min(i++, len)] = R"()"; + +/* + // error message + if (true) { + html[min(i++, len)] = R"( + + )"; + } + */ + http_send_content(html, min(i, len)); +} + +static void webHandleRestoreUpload() +{ + static ZipStream *unzipstream = NULL; + + if(!http_is_authenticated()) return; + + upload = &webServer.upload(); + + switch (upload->status) { + case UPLOAD_FILE_START : { + if (unzipstream) { delete unzipstream; } + unzipstream = new ZipStream(); + unzipstream->beginUnZip(); + break; + } + case UPLOAD_FILE_WRITE : { + if (unzipstream && unzipstream->getLastError() >= ZIP_STREAM_OK) { + unzipstream->write(upload->buf, upload->currentSize); + } + break; + } + case UPLOAD_FILE_END : { + if (unzipstream && unzipstream->getLastError() != ZIP_STREAM_OK) { + LOG_ERROR(TAG_FILE, F("Restore error %d - %s"), unzipstream->getLastError(), unzipstream->getLastErrorString().c_str() ); + } + + if (unzipstream) { + delete unzipstream; + unzipstream = NULL; + } + break; + } + default: { + LOG_WARNING(TAG_HTTP, "Restore aborted"); + if (unzipstream) { + delete unzipstream; + unzipstream = NULL; + } + webServer.send(400, "text/plain", PSTR("Upload aborted")); + } + } +} +#endif // HASP_USE_SPIFFS || HASP_USE_LITTLEFS + //////////////////////////////////////////////////////////////////////////////////////////////////// #if HASP_USE_CONFIG > 0 @@ -2723,6 +2905,10 @@ void httpSetup() webHandleFirmwareUpload); #endif +#if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 + webServer.on("/backup", HTTP_ANY, webHandleBackup, webHandleRestoreUpload); +#endif + #ifdef HTTP_LEGACY webServer.on("/config", http_handle_config); #endif From 7e5525f496dae7d0613d481f3bc535761d580237 Mon Sep 17 00:00:00 2001 From: marsman7 Date: Wed, 30 Apr 2025 09:50:53 +0200 Subject: [PATCH 2/5] memory utilisation improved and source code tidied up --- lib/ZipStream/ZipStream.cpp | 68 +++++++++++++++---------------------- lib/ZipStream/ZipStream.h | 3 +- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/lib/ZipStream/ZipStream.cpp b/lib/ZipStream/ZipStream.cpp index 6385c16cf..6f83005d2 100644 --- a/lib/ZipStream/ZipStream.cpp +++ b/lib/ZipStream/ZipStream.cpp @@ -119,8 +119,6 @@ zip_stream_error_t ZipStream::beginZip(String *fileList) _lastErrorCode = ZIP_STREAM_OK; _lastErrorString = ""; - memset(&_pEndCentralDirRecord, 0, sizeof(zip_end_central_dir_record_t)); - _jsonError = deserializeJson(_jsonDoc, *fileList); if (_jsonError != DeserializationError::Ok) { _lastErrorCode = ZIP_STREAM_FILE_LIST_FAIL; @@ -138,7 +136,6 @@ zip_stream_error_t ZipStream::beginZip(String *fileList) if (_lastErrorCode != ZIP_STREAM_OK) { _lastErrorString = String("File list error"); -// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); return _lastErrorCode; } @@ -149,6 +146,7 @@ zip_stream_error_t ZipStream::beginZip(String *fileList) if (!_pBuffer) { _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; _lastErrorString = String(zip_memory_error); + flush(); return _lastErrorCode; } @@ -157,11 +155,23 @@ zip_stream_error_t ZipStream::beginZip(String *fileList) if (!_pFileDescriptorBuffer) { _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; _lastErrorString = String(zip_memory_error); -// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); + flush(); + return _lastErrorCode; + } + } + + if (!_pEndCentralDirRecord) { + _pEndCentralDirRecord = (zip_end_central_dir_record_t *)malloc(sizeof(zip_end_central_dir_record_t)); + if (!_pEndCentralDirRecord) { + _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; + _lastErrorString = String(zip_memory_error); + flush(); return _lastErrorCode; } } + memset(_pEndCentralDirRecord, 0, sizeof(zip_end_central_dir_record_t)); + _streamState = ZIP_STREAM_LOCAL_HEADER; buildBufferLocalFileHeader(); @@ -199,7 +209,7 @@ size_t ZipStream::write(Stream &zipStream) if (!_pInputBuffer) { _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; _lastErrorString = String(zip_memory_error); - return 1; + return 0; } size_t toRead = 0; @@ -438,8 +448,8 @@ size_t ZipStream::write(const uint8_t *pInData, size_t size) if(_fileCRC32 != _zipLocalHeader.crc) { // ESP_LOGI(TAG_ZIP, "File CRC does not match %x <=> %x", _fileCRC32, _zipLocalHeader.crc); -// _lastErrorCode = ZIP_STREAM_CRC_ERROR; -// _lastErrorString = String(zip_file_crc_error); + _lastErrorCode = ZIP_STREAM_CRC_ERROR; + _lastErrorString = String(zip_file_crc_error); } _streamState = ZIP_STREAM_EXPECT_SIGNATURE; @@ -480,15 +490,6 @@ void ZipStream::buildBufferLocalFileHeader() { _pFileDescriptorBuffer[_fileIdx].is_valid = false; _position = 0; -/* - if (!_pBuffer) { - _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; - _lastErrorString = String(zip_memory_error); -// ESP_LOGE(TAG_ZIP, "%s", _lastErrorString.c_str()); - return; - } -*/ - *(uint32_t*)_pBuffer = ZIP_LFH_SIGNATURE; _content_length = sizeof(uint32_t); @@ -557,9 +558,9 @@ void ZipStream::buildBufferLocalFileHeader() { _pFileDescriptorBuffer[_fileIdx].is_valid = true; _pFileDescriptorBuffer[_fileIdx].crc = pZipLocalHeader->crc; _pFileDescriptorBuffer[_fileIdx].compressed_size = pZipLocalHeader->compressed_size; - _pFileDescriptorBuffer[_fileIdx].relative_offset = _pEndCentralDirRecord.offset_central_dir_start; + _pFileDescriptorBuffer[_fileIdx].relative_offset = _pEndCentralDirRecord->offset_central_dir_start; - _pEndCentralDirRecord.offset_central_dir_start += _content_length + pZipLocalHeader->compressed_size; + _pEndCentralDirRecord->offset_central_dir_start += _content_length + pZipLocalHeader->compressed_size; return; } @@ -578,15 +579,6 @@ void ZipStream::buildBufferCentralDirFileHeader() { _position = 0; -/* - if (!_pBuffer) { - _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; - _lastErrorString = String(zip_memory_error); -// ESP_LOGE(TAG_ZIP, "%s", _lastErrorString.c_str()); - return; - } -*/ - *(uint32_t*)_pBuffer = ZIP_CDFH_SIGNATURE; _content_length = sizeof(uint32_t); @@ -622,9 +614,9 @@ void ZipStream::buildBufferCentralDirFileHeader() { _content_length += filename.length(); pHeader->filename_length = filename.length(); - _pEndCentralDirRecord.central_dir_records++; - _pEndCentralDirRecord.total_dir_records++; - _pEndCentralDirRecord.central_dir_size += _content_length; + _pEndCentralDirRecord->central_dir_records++; + _pEndCentralDirRecord->total_dir_records++; + _pEndCentralDirRecord->central_dir_size += _content_length; return; } @@ -637,19 +629,10 @@ void ZipStream::buildBufferCentralDirFileHeader() { void ZipStream::buildBufferEndCentralFileHeader() { _position = 0; -/* - if (!_pBuffer) { - _lastErrorCode = ZIP_STREAM_ERROR_NO_MEMORY; - _lastErrorString = String(zip_memory_error); -// ESP_LOGE(TAG_ZIP, F(_lastErrorString.c_str())); - return; - } - */ - *(uint32_t*)_pBuffer = ZIP_EOCD_SIGNATURE; _content_length = sizeof(uint32_t); - memcpy(_pBuffer + _content_length, &_pEndCentralDirRecord, sizeof(zip_end_central_dir_record_t)); + memcpy(_pBuffer + _content_length, _pEndCentralDirRecord, sizeof(zip_end_central_dir_record_t)); _content_length += sizeof(zip_end_central_dir_record_t); @@ -779,6 +762,11 @@ void ZipStream::flush() { free(_pFileDescriptorBuffer); _pFileDescriptorBuffer = NULL; } + + if (_pEndCentralDirRecord) { + free(_pEndCentralDirRecord); + _pEndCentralDirRecord = NULL; + } } #endif // HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 \ No newline at end of file diff --git a/lib/ZipStream/ZipStream.h b/lib/ZipStream/ZipStream.h index 3ee20c6c4..7a26d0bdd 100644 --- a/lib/ZipStream/ZipStream.h +++ b/lib/ZipStream/ZipStream.h @@ -170,7 +170,7 @@ class ZipStream : public Stream { uint16_t _fileIdx; // Current file index size_t _content_length; zip_file_discriptor_t *_pFileDescriptorBuffer; - zip_end_central_dir_record_t _pEndCentralDirRecord; + zip_end_central_dir_record_t *_pEndCentralDirRecord; void buildBufferLocalFileHeader(); void buildBufferCentralDirFileHeader(); @@ -187,6 +187,7 @@ protected : _content_length(0), _fileIdx(0), _pFileDescriptorBuffer(NULL), + _pEndCentralDirRecord(NULL), _signature(0), _datalen(0), From 75d05a50379ecda61385997575b3e2d18f394d80 Mon Sep 17 00:00:00 2001 From: marsman7 Date: Wed, 30 Apr 2025 09:59:30 +0200 Subject: [PATCH 3/5] add strings in language files but not translation --- data/da_DK.json | 10 ++++++++++ data/es_ES.json | 10 ++++++++++ data/fr_FR.json | 10 ++++++++++ data/hu_HU.json | 10 ++++++++++ data/nl_NL.json | 10 ++++++++++ data/pt_BR.json | 10 ++++++++++ data/pt_PT.json | 10 ++++++++++ data/ro_RO.json | 10 ++++++++++ data/sv_SE.json | 10 ++++++++++ data/zh_CN.json | 10 ++++++++++ 10 files changed, 100 insertions(+) diff --git a/data/da_DK.json b/data/da_DK.json index 6e48f802f..ced4e3502 100644 --- a/data/da_DK.json +++ b/data/da_DK.json @@ -50,6 +50,16 @@ "strict": "Streng", "always": "Altid" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Fil Editor", "btn": "Fil Editor", diff --git a/data/es_ES.json b/data/es_ES.json index 0906e4046..0bfd5af6d 100644 --- a/data/es_ES.json +++ b/data/es_ES.json @@ -50,6 +50,16 @@ "strict": "Estricto", "always": "Siempre" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Editor de Archivos", "btn": "Editor de Archivos", diff --git a/data/fr_FR.json b/data/fr_FR.json index aacaead09..9f2c46d05 100644 --- a/data/fr_FR.json +++ b/data/fr_FR.json @@ -50,6 +50,16 @@ "strict": "Stricte", "always": "Toujours" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Éditeur de fichiers", "btn": "Éditeur de fichiers", diff --git a/data/hu_HU.json b/data/hu_HU.json index 16a2e736d..f4633c4ec 100644 --- a/data/hu_HU.json +++ b/data/hu_HU.json @@ -50,6 +50,16 @@ "strict": "Szigorú", "always": "Mindig" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Fájlkezelő", "btn": "Fájlkezelő", diff --git a/data/nl_NL.json b/data/nl_NL.json index 834935723..9692f611e 100644 --- a/data/nl_NL.json +++ b/data/nl_NL.json @@ -50,6 +50,16 @@ "strict": "Strikt", "always": "Altijd" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Bestandseditor", "btn": "Bestandseditor", diff --git a/data/pt_BR.json b/data/pt_BR.json index 2717d9ae2..e9fdfa8a8 100644 --- a/data/pt_BR.json +++ b/data/pt_BR.json @@ -50,6 +50,16 @@ "strict": "Estrito", "always": "Sempre" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Editor de ficheiros", "btn": "Editor de ficheiros", diff --git a/data/pt_PT.json b/data/pt_PT.json index 64f447e7f..c12c04b82 100644 --- a/data/pt_PT.json +++ b/data/pt_PT.json @@ -50,6 +50,16 @@ "strict": "Estrito", "always": "Sempre" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Editor de ficheiros", "btn": "Editor de ficheiros", diff --git a/data/ro_RO.json b/data/ro_RO.json index bd92c3b87..17a76b579 100644 --- a/data/ro_RO.json +++ b/data/ro_RO.json @@ -50,6 +50,16 @@ "strict": "Strict", "always": "Mereu" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Browser de fișiere", "btn": "Browser de fișiere", diff --git a/data/sv_SE.json b/data/sv_SE.json index f308542f0..c45a78513 100644 --- a/data/sv_SE.json +++ b/data/sv_SE.json @@ -50,6 +50,16 @@ "strict": "Strikt", "always": "Alltid" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "Filredigerare", "btn": "Filredigerare", diff --git a/data/zh_CN.json b/data/zh_CN.json index aeced869e..494d6c304 100644 --- a/data/zh_CN.json +++ b/data/zh_CN.json @@ -50,6 +50,16 @@ "strict": "严格", "always": "总是" }, + "backup": { + "title": "Backup / Restore", + "btn": "Backup / Restore", + "nav": "Backup", + "backup": "Backup Data", + "bakfile": "Backup to file", + "restore": "Restore Data", + "resfile": "Restore file", + "overwrite": "Overwrite existing files" + }, "editor": { "title": "文件编辑器", "btn": "文件编辑器", From 93c9ffce6230a9cde330de9912819f2bcf29ba91 Mon Sep 17 00:00:00 2001 From: marsman7 Date: Thu, 1 May 2025 11:37:03 +0200 Subject: [PATCH 4/5] add old unzip sourcecode, add HASP_USE_BACKUP --- include/user_config_override-template.h | 1 + src/hasp_filesystem.cpp | 113 +++++++++++++++++++++++- src/hasp_filesystem.h | 28 +++++- src/sys/svc/hasp_http.cpp | 9 +- 4 files changed, 145 insertions(+), 6 deletions(-) diff --git a/include/user_config_override-template.h b/include/user_config_override-template.h index 113acc7e8..8ba823c18 100644 --- a/include/user_config_override-template.h +++ b/include/user_config_override-template.h @@ -157,5 +157,6 @@ //#define HASP_DEBUG_OBJ_TREE // Output all objects to the log on page changes //#define HASP_LOG_LEVEL LOG_LEVEL_VERBOSE // LOG_LEVEL_* can be DEBUG, VERBOSE, TRACE, INFO, WARNING, ERROR, CRITICAL, ALERT, FATAL, SILENT //#define HASP_LOG_TASKS // Also log the Taskname and watermark of ESP32 tasks +//#define HASP_USE_BACKUP 1 #endif // HASP_USER_CONFIG_OVERRIDE_H diff --git a/src/hasp_filesystem.cpp b/src/hasp_filesystem.cpp index 6c4da8b38..e4731cb4c 100644 --- a/src/hasp_filesystem.cpp +++ b/src/hasp_filesystem.cpp @@ -34,7 +34,10 @@ extern const uint8_t PAGES_JSONL_END[] asm("_binary_data_pages_pages_jsonl_end") #include "ArduinoJson.h" #include "hasp_debug.h" #include "hasp_filesystem.h" + +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) #include "ZipStream.h" +#endif #if defined(ARDUINO_ARCH_ESP32) @@ -53,18 +56,126 @@ void filesystemUnzip(const char*, const char* filename, uint8_t source) return; } +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) ZipStream unzipStream; if ( unzipStream.beginUnZip() == ZIP_STREAM_OK) { unzipStream.write(zipfile); } +#else + + int32_t head; + size_t len; + bool done = false; + + zipfile.seek(0); + while(!done) { + len = zipfile.read((uint8_t*)&head, sizeof(head)); + if(len != sizeof(head)) { + done = true; + continue; + } + + switch(head) { + case 0x04034b50: { + zip_file_header_t fh; + zipfile.seek(zipfile.position() - 2, SeekSet); // rewind for struct alignment (26-28) + len = zipfile.read((uint8_t*)(&fh), sizeof(zip_file_header_t)); + if(len != sizeof(zip_file_header_t)) { + done = true; + continue; + } + + if(fh.filename_length >= 255) { + LOG_WARNING(TAG_FILE, F("filename length too long %d"), fh.filename_length); + zipfile.seek(fh.filename_length + fh.extra_length, SeekCur); // skip extra field + continue; + // } else { + // LOG_WARNING(TAG_FILE, F("min %d - flag %d - len %d - xtra %d"), fh.min_version, fh.flags, + // fh.filename_length, fh.extra_length); + } + char name[257] = {0}; + name[0] = '/'; + + len = zipfile.read((uint8_t*)&name[1], fh.filename_length); + if(len != fh.filename_length) { + LOG_WARNING(TAG_FILE, F("filename read failed %d != %d"), fh.filename_length, len); + done = true; + continue; + } + zipfile.seek(fh.extra_length, SeekCur); // skip extra field + + if(fh.compression_method != ZIP_NO_COMPRESSION) { + LOG_WARNING(TAG_FILE, F("Compression is not supported %d"), fh.compression_method); + zipfile.seek(fh.compressed_size, SeekCur); // skip compressed file + } else { + + if(HASP_FS.exists(name)) HASP_FS.remove(name); + + File f = HASP_FS.open(name, FILE_WRITE); + if(f) { + uint8_t buffer[512]; + uint32_t crc32 = 0; + + while(!done && fh.compressed_size >= 512) { + len = zipfile.readBytes((char*)&buffer, 512); + if(len != 512) done = true; + fh.compressed_size -= len; + crc32 = crc32_le(crc32, buffer, len); + f.write(buffer, len); + } + + if(!done && fh.compressed_size > 0) { + len = zipfile.readBytes((char*)&buffer, fh.compressed_size); + if(len != fh.compressed_size) done = true; + fh.compressed_size -= len; + crc32 = crc32_le(crc32, buffer, len); + f.write(buffer, len); + } + + if(crc32 != fh.crc) done = true; + + if(!done) { + Parser::format_bytes(fh.uncompressed_size, (char*)buffer, sizeof(buffer)); + LOG_VERBOSE(TAG_FILE, F(D_BULLET "%s (%s)"), name, buffer); + } else { + LOG_ERROR(TAG_FILE, F(D_FILE_SAVE_FAILED), name); + } + + f.close(); + } + } + + break; + } + case 0x02014b50: + done = true; + break; + case 0x06054b50: + // end of file + done = true; + break; + default: { + char outputString[9]; + itoa(head, outputString, 16); + LOG_WARNING(TAG_FILE, F("invalid %s"), outputString); + done = true; + } + } + } + +#endif zipfile.close(); +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) if (unzipStream.getLastError() != ZIP_STREAM_OK) { LOG_ERROR(TAG_FILE, F("Unpacking error %d - %s"), unzipStream.getLastError(), unzipStream.getLastErrorString().c_str() ); } -} +#else + LOG_VERBOSE(TAG_FILE, F("extracting %s complete"), filename); #endif +} +#endif #if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 void filesystemInfo() { // Get all information of your SPIFFS diff --git a/src/hasp_filesystem.h b/src/hasp_filesystem.h index 896847184..f3128bfc3 100644 --- a/src/hasp_filesystem.h +++ b/src/hasp_filesystem.h @@ -6,17 +6,37 @@ #include #include -#include "hasp_debug.h" +//#include "hasp_debug.h" -#include -#include -#include +//#include +//#include +//#include bool filesystemSetup(void); void filesystemList(); void filesystemInfo(); void filesystemSetupFiles(); +#if (!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) + enum { ZIP_NO_COMPRESSION = 0, ZIP_DEFLTATE = 8 }; + typedef uint16_t zip_compression_method_t; + + typedef struct + { + uint16_t dummy_bytes; // total struct needs to be a multiple of 4 bytes + uint16_t min_version; + uint16_t flags; + zip_compression_method_t compression_method; + uint16_t time_modified; + uint16_t date_modified; + uint32_t crc; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_length; // OK + uint16_t extra_length; + } zip_file_header_t; +#endif + #if defined(ARDUINO_ARCH_ESP32) #if HASP_USE_SPIFFS > 0 #include "SPIFFS.h" diff --git a/src/sys/svc/hasp_http.cpp b/src/sys/svc/hasp_http.cpp index 52819d72e..40ad25028 100644 --- a/src/sys/svc/hasp_http.cpp +++ b/src/sys/svc/hasp_http.cpp @@ -27,7 +27,9 @@ #include "sys/net/hasp_network.h" #include "sys/net/hasp_time.h" +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) #include "ZipStream.h" +#endif #if(HASP_USE_CAPTIVE_PORTAL > 0) && (HASP_USE_WIFI > 0) #include @@ -442,7 +444,9 @@ static void http_handle_root() html[min(i++, len)] = R"()"; html[min(i++, len)] = R"()"; html[min(i++, len)] = R"()"; +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) html[min(i++, len)] = R"()"; +#endif html[min(i++, len)] = R"()"; #ifdef ARDUINO_ARCH_ESP32 html[min(i++, len)] = R"()"; @@ -2544,6 +2548,7 @@ static void webHandleFirmware() //////////////////////////////////////////////////////////////////////////////////////////////////// +#if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) /**************************************************************************//** * @brief Handle backup and restore * When the backup is executed, the ‘config.json’ file is created and @@ -2719,6 +2724,8 @@ static void webHandleRestoreUpload() } } } +#endif // #if !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) + #endif // HASP_USE_SPIFFS || HASP_USE_LITTLEFS //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -2905,7 +2912,7 @@ void httpSetup() webHandleFirmwareUpload); #endif -#if HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0 +#if (HASP_USE_SPIFFS > 0 || HASP_USE_LITTLEFS > 0) && !(!defined(HASP_USE_BACKUP) || HASP_USE_BACKUP < 1) webServer.on("/backup", HTTP_ANY, webHandleBackup, webHandleRestoreUpload); #endif From 20f5ca90c1a92406f9aa312135637661f5311479 Mon Sep 17 00:00:00 2001 From: marsman7 Date: Mon, 12 May 2025 22:26:56 +0200 Subject: [PATCH 5/5] fix: crash in timeSetup --- src/sys/net/hasp_time.cpp | 16 ++++++++++------ src/sys/net/hasp_time.h | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/sys/net/hasp_time.cpp b/src/sys/net/hasp_time.cpp index 3a1930492..ac2886755 100644 --- a/src/sys/net/hasp_time.cpp +++ b/src/sys/net/hasp_time.cpp @@ -34,12 +34,13 @@ void timeSyncCallback(struct timeval* tv) } #endif -void timeSetup() +void timeSetup(bool websync) { #if defined(ARDUINO_ARCH_ESP8266) LOG_WARNING(TAG_TIME, F("TIMEZONE: %s"), MYTZ); configTzTime(MYTZ, NTPSERVER1, NTPSERVER2, NTPSERVER3); // literal string #endif + #if defined(ARDUINO_ARCH_ESP32) Preferences preferences; nvs_user_begin(preferences, "time", true); @@ -57,9 +58,12 @@ void timeSetup() LOG_VERBOSE(TAG_TIME, F("%s => %s"), zone.c_str(), mytz.c_str()); LOG_VERBOSE(TAG_TIME, F("NTP: %s %s %s"), ntp1.c_str(), ntp2.c_str(), ntp3.c_str()); - sntp_set_time_sync_notification_cb(timeSyncCallback); - configTzTime(mytz.c_str(), ntp1.c_str(), ntp2.c_str(), ntp3.c_str()); - sntp_servermode_dhcp(enable && dhcp ? 1 : 0); + if (websync) { + sntp_set_time_sync_notification_cb(timeSyncCallback); + configTzTime(mytz.c_str(), ntp1.c_str(), ntp2.c_str(), ntp3.c_str()); + sntp_servermode_dhcp(enable && dhcp ? 1 : 0); + } + preferences.end(); #endif } @@ -697,8 +701,8 @@ bool timeSetConfig(const JsonObject& settings) changed |= nvsUpdateString(preferences, "ntp3", settings["ntp"][2]); preferences.end(); - timeSetup(); - + timeSetup(false); + return changed; } #endif \ No newline at end of file diff --git a/src/sys/net/hasp_time.h b/src/sys/net/hasp_time.h index 0448fd295..2d7803a21 100644 --- a/src/sys/net/hasp_time.h +++ b/src/sys/net/hasp_time.h @@ -7,7 +7,7 @@ #include "hasplib.h" /* ===== Default Event Processors ===== */ -void timeSetup(); +void timeSetup(bool websync = true); /* ===== Special Event Processors ===== */