From 5a6a08ec93134e47531223da0e46ec076307f695 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Fri, 19 Sep 2025 13:04:31 +1000 Subject: [PATCH 1/9] retain data aftwer CRC errors, when possible --- src/js/msp.js | 3 +- src/js/msp/MSPHelper.js | 171 ++++++++++----------------------- src/js/tabs/onboard_logging.js | 77 +++++++-------- 3 files changed, 90 insertions(+), 161 deletions(-) diff --git a/src/js/msp.js b/src/js/msp.js index d98c11f96d..ececf03570 100644 --- a/src/js/msp.js +++ b/src/js/msp.js @@ -122,7 +122,8 @@ const MSP = { this.state = this.decoder_states.DIRECTION_V2; break; default: - console.log(`Unknown protocol char ${String.fromCharCode(chunk)}`); + // Removed logging of unknown protocol characters because the log itself is binary. + // console.log(`Unknown protocol char ${String.fromCharCode(chunk)}`); this.state = this.decoder_states.IDLE; } break; diff --git a/src/js/msp/MSPHelper.js b/src/js/msp/MSPHelper.js index 1512da5e9d..dc6494e35f 100644 --- a/src/js/msp/MSPHelper.js +++ b/src/js/msp/MSPHelper.js @@ -2450,147 +2450,74 @@ MspHelper.prototype.setRawRx = function (channels) { /** * Send a request to read a block of data from the dataflash at the given address and pass that address and a dataview - * of the returned data to the given callback (or null for the data if an error occured). + * of the returned data to the given callback (or null for the data if an error occurred). */ -MspHelper.prototype.dataflashRead = function (address, blockSize, onDataCallback) { - let outData = [address & 0xff, (address >> 8) & 0xff, (address >> 16) & 0xff, (address >> 24) & 0xff]; - - outData = outData.concat([blockSize & 0xff, (blockSize >> 8) & 0xff]); - - // Allow compression - outData = outData.concat([1]); - - MSP.send_message( - MSPCodes.MSP_DATAFLASH_READ, - outData, - false, - function (response) { - if (!response.crcError) { - const chunkAddress = response.data.readU32(); - - const headerSize = 7; - const dataSize = response.data.readU16(); - const dataCompressionType = response.data.readU8(); - - // Verify that the address of the memory returned matches what the caller asked for and there was not a CRC error - if (chunkAddress == address) { - /* Strip that address off the front of the reply and deliver it separately so the caller doesn't have to - * figure out the reply format: - */ - if (dataCompressionType == 0) { - onDataCallback( - address, - new DataView(response.data.buffer, response.data.byteOffset + headerSize, dataSize), - ); - } else if (dataCompressionType == 1) { - // Read compressed char count to avoid decoding stray bit sequences as bytes - const compressedCharCount = response.data.readU16(); +MspHelper.prototype.dataflashRead = function(address, blockSize, onDataCallback) { + let outData = [ + address & 0xFF, + (address >> 8) & 0xFF, + (address >> 16) & 0xFF, + (address >> 24) & 0xFF, + blockSize & 0xFF, + (blockSize >> 8) & 0xFF, + 1 // allow compression + ]; + + const mspObj = this.msp || (typeof MSP !== 'undefined' ? MSP : null); + if (!mspObj) { + console.error('MSP object not found, cannot read dataflash.'); + onDataCallback(address, null); + return; + } + + mspObj.send_message(MSPCodes.MSP_DATAFLASH_READ, outData, false, function(response) { + let payloadView = null; - // Compressed format uses 2 additional bytes as a pseudo-header to denote the number of uncompressed bytes + if (response && response.data) { + const headerSize = 7; + const chunkAddress = response.data.readU32(); + const dataSize = response.data.readU16(); + const dataCompressionType = response.data.readU8(); + + if (chunkAddress === address) { + try { + if (dataCompressionType === 0) { + payloadView = new DataView(response.data.buffer, response.data.byteOffset + headerSize, dataSize); + } else if (dataCompressionType === 1) { + const compressedCharCount = response.data.readU16(); const compressedArray = new Uint8Array( response.data.buffer, response.data.byteOffset + headerSize + 2, - dataSize - 2, + dataSize - 2 ); const decompressedArray = huffmanDecodeBuf( compressedArray, compressedCharCount, defaultHuffmanTree, - defaultHuffmanLenIndex, + defaultHuffmanLenIndex ); - - onDataCallback(address, new DataView(decompressedArray.buffer), dataSize); + payloadView = new DataView(decompressedArray.buffer); } - } else { - // Report address error - console.log(`Expected address ${address} but received ${chunkAddress} - retrying`); - onDataCallback(address, null); // returning null to the callback forces a retry + } catch (e) { + console.warn('Decompression or read failed, delivering raw data anyway'); + payloadView = new DataView(response.data.buffer, response.data.byteOffset + headerSize, dataSize); } } else { - // Report crc error - console.log(`CRC error for address ${address} - retrying`); - onDataCallback(address, null); // returning null to the callback forces a retry + console.log(`Expected address ${address} but received ${chunkAddress}`); } - }, - true, - ); -}; - -MspHelper.prototype.sendServoConfigurations = function (onCompleteCallback) { - let nextFunction = send_next_servo_configuration; - - let servoIndex = 0; - - if (FC.SERVO_CONFIG.length == 0) { - onCompleteCallback(); - } else { - nextFunction(); - } - - function send_next_servo_configuration() { - const buffer = []; - - // send one at a time, with index - - const servoConfiguration = FC.SERVO_CONFIG[servoIndex]; - - buffer - .push8(servoIndex) - .push16(servoConfiguration.min) - .push16(servoConfiguration.max) - .push16(servoConfiguration.middle) - .push8(servoConfiguration.rate); - - let out = servoConfiguration.indexOfChannelToForward; - if (out == undefined) { - out = 255; // Cleanflight defines "CHANNEL_FORWARDING_DISABLED" as "(uint8_t)0xFF" } - buffer.push8(out).push32(servoConfiguration.reversedInputSources); - // prepare for next iteration - servoIndex++; - if (servoIndex == FC.SERVO_CONFIG.length) { - nextFunction = onCompleteCallback; - } - - MSP.send_message(MSPCodes.MSP_SET_SERVO_CONFIGURATION, buffer, false, nextFunction); - } -}; - -MspHelper.prototype.sendModeRanges = function (onCompleteCallback) { - let nextFunction = send_next_mode_range; - - let modeRangeIndex = 0; - - if (FC.MODE_RANGES.length == 0) { - onCompleteCallback(); - } else { - send_next_mode_range(); - } + // Deliver payloadView if defined, otherwise pass null + onDataCallback(address, payloadView); - function send_next_mode_range() { - const modeRange = FC.MODE_RANGES[modeRangeIndex]; - const buffer = []; - - buffer - .push8(modeRangeIndex) - .push8(modeRange.id) - .push8(modeRange.auxChannelIndex) - .push8((modeRange.range.start - 900) / 25) - .push8((modeRange.range.end - 900) / 25); - - const modeRangeExtra = FC.MODE_RANGES_EXTRA[modeRangeIndex]; - - buffer.push8(modeRangeExtra.modeLogic).push8(modeRangeExtra.linkedTo); - - // prepare for next iteration - modeRangeIndex++; - if (modeRangeIndex == FC.MODE_RANGES.length) { - nextFunction = onCompleteCallback; + if (!response || response.crcError) { + console.log(`CRC error or missing data at address ${address} - delivering whatever we got`); + } else if (payloadView) { + console.log(`Block at ${address} received (${payloadView.byteLength} bytes)`); } - MSP.send_message(MSPCodes.MSP_SET_MODE_RANGE, buffer, false, nextFunction); - } -}; + }, true); // end of send_message +}; // end of dataflashRead + MspHelper.prototype.sendAdjustmentRanges = function (onCompleteCallback) { let nextFunction = send_next_adjustment_range; diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index 6998e2c451..b741db9954 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -443,14 +443,16 @@ onboard_logging.initialize = function (callback) { console.log( `Received ${totalBytes} bytes in ${totalTime.toFixed(2)}s (${(totalBytes / totalTime / 1024).toFixed( 2, - )}kB / s) with block size ${self.blockSize}.`, + )} kB / s) with block size ${self.blockSize}.`, ); - if (!isNaN(totalBytesCompressed)) { + + if (typeof totalBytesCompressed === "number" && totalBytesCompressed > 0) { + const meanCompressionFactor = totalBytes / totalBytesCompressed; console.log( "Compressed into", totalBytesCompressed, "bytes with mean compression factor of", - totalBytes / totalBytesCompressed, + meanCompressionFactor.toFixed(2), ); } @@ -490,48 +492,47 @@ onboard_logging.initialize = function (callback) { show_saving_dialog(); function onChunkRead(chunkAddress, chunkDataView, bytesCompressed) { - if (chunkDataView !== null) { - // Did we receive any data? - if (chunkDataView.byteLength > 0) { - nextAddress += chunkDataView.byteLength; - if (isNaN(bytesCompressed) || isNaN(totalBytesCompressed)) { - totalBytesCompressed = null; - } else { - totalBytesCompressed += bytesCompressed; - } - - $(".dataflash-saving progress").attr("value", (nextAddress / maxBytes) * 100); - - const blob = new Blob([chunkDataView]); - FileSystem.writeChunck(openedFile, blob).then(() => { - if (saveCancelled || nextAddress >= maxBytes) { - if (saveCancelled) { - dismiss_saving_dialog(); - } else { - mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); - } - FileSystem.closeFile(openedFile); - } else { - if (!self.writeError) { - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - } else { - dismiss_saving_dialog(); - FileSystem.closeFile(openedFile); - } - } - }); - } else { - // A zero-byte block indicates end-of-file, so we're done + if (chunkDataView && chunkDataView.byteLength > 0) { + // Always write non-empty data, even if CRC mismatch + const blob = new Blob([chunkDataView]); + FileSystem.writeChunck(openedFile, blob); + + nextAddress += chunkDataView.byteLength; + + // Track total compressed bytes, if provided + if (typeof bytesCompressed === "number") { + if (totalBytesCompressed == null) totalBytesCompressed = 0; // initialize if previously unknown + totalBytesCompressed += bytesCompressed; + } + + $(".dataflash-saving progress").attr("value", (nextAddress / maxBytes) * 100); + + if (saveCancelled || nextAddress >= maxBytes) { mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); FileSystem.closeFile(openedFile); + } else { + mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); } + } else if (chunkDataView && chunkDataView.byteLength === 0) { + // Zero-length block → EOF + mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); + FileSystem.closeFile(openedFile); } else { - // There was an error with the received block (address didn't match the one we asked for), retry - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); + // Null block → skip ahead (hard error) + console.warn(`Skipping null block at address ${nextAddress}`); + nextAddress += self.blockSize; + + if (nextAddress >= maxBytes) { + mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); + FileSystem.closeFile(openedFile); + } else { + mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); + } } } - const startTime = new Date().getTime(); + const startTime = new Date().getTime(); // Start timestamp + // Fetch the initial block FileSystem.openFile(fileWriter).then((file) => { openedFile = file; From 9a8a201ab08831109433342e9e8170c9cc134d75 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Wed, 24 Sep 2025 22:31:43 +1000 Subject: [PATCH 2/9] Fix compression estimate --- src/js/msp/MSPHelper.js | 123 +++++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 53 deletions(-) diff --git a/src/js/msp/MSPHelper.js b/src/js/msp/MSPHelper.js index dc6494e35f..76ce361498 100644 --- a/src/js/msp/MSPHelper.js +++ b/src/js/msp/MSPHelper.js @@ -2449,76 +2449,93 @@ MspHelper.prototype.setRawRx = function (channels) { }; /** - * Send a request to read a block of data from the dataflash at the given address and pass that address and a dataview - * of the returned data to the given callback (or null for the data if an error occurred). + * Send a request to read a block of data from the dataflash at the given address + * and pass that address, a DataView of the returned data, and bytesCompressed + * to the given callback. */ -MspHelper.prototype.dataflashRead = function(address, blockSize, onDataCallback) { +MspHelper.prototype.dataflashRead = function (address, blockSize, onDataCallback) { let outData = [ - address & 0xFF, - (address >> 8) & 0xFF, - (address >> 16) & 0xFF, - (address >> 24) & 0xFF, - blockSize & 0xFF, - (blockSize >> 8) & 0xFF, - 1 // allow compression + address & 0xff, + (address >> 8) & 0xff, + (address >> 16) & 0xff, + (address >> 24) & 0xff, + blockSize & 0xff, + (blockSize >> 8) & 0xff, + 1, // allow compression ]; - const mspObj = this.msp || (typeof MSP !== 'undefined' ? MSP : null); + const mspObj = this.msp || (typeof MSP !== "undefined" ? MSP : null); if (!mspObj) { - console.error('MSP object not found, cannot read dataflash.'); - onDataCallback(address, null); + console.error("MSP object not found, cannot read dataflash."); + onDataCallback(address, null, 0); return; } - mspObj.send_message(MSPCodes.MSP_DATAFLASH_READ, outData, false, function(response) { - let payloadView = null; - - if (response && response.data) { - const headerSize = 7; - const chunkAddress = response.data.readU32(); - const dataSize = response.data.readU16(); - const dataCompressionType = response.data.readU8(); - - if (chunkAddress === address) { - try { - if (dataCompressionType === 0) { - payloadView = new DataView(response.data.buffer, response.data.byteOffset + headerSize, dataSize); - } else if (dataCompressionType === 1) { - const compressedCharCount = response.data.readU16(); - const compressedArray = new Uint8Array( + mspObj.send_message( + MSPCodes.MSP_DATAFLASH_READ, + outData, + false, + function (response) { + let payloadView = null; + let bytesCompressed = 0; + + if (response && response.data) { + const headerSize = 7; + const chunkAddress = response.data.readU32(); + const dataSize = response.data.readU16(); + const dataCompressionType = response.data.readU8(); + + if (chunkAddress === address) { + try { + if (dataCompressionType === 0) { + payloadView = new DataView( + response.data.buffer, + response.data.byteOffset + headerSize, + dataSize, + ); + bytesCompressed = dataSize; // treat uncompressed as same size + } else if (dataCompressionType === 1) { + const compressedCharCount = response.data.readU16(); + const compressedArray = new Uint8Array( + response.data.buffer, + response.data.byteOffset + headerSize + 2, + dataSize - 2, + ); + const decompressedArray = huffmanDecodeBuf( + compressedArray, + compressedCharCount, + defaultHuffmanTree, + defaultHuffmanLenIndex, + ); + payloadView = new DataView(decompressedArray.buffer); + bytesCompressed = compressedCharCount; + } + } catch (e) { + console.warn("Decompression or read failed, delivering raw data anyway"); + payloadView = new DataView( response.data.buffer, - response.data.byteOffset + headerSize + 2, - dataSize - 2 - ); - const decompressedArray = huffmanDecodeBuf( - compressedArray, - compressedCharCount, - defaultHuffmanTree, - defaultHuffmanLenIndex + response.data.byteOffset + headerSize, + dataSize, ); - payloadView = new DataView(decompressedArray.buffer); + bytesCompressed = dataSize; } - } catch (e) { - console.warn('Decompression or read failed, delivering raw data anyway'); - payloadView = new DataView(response.data.buffer, response.data.byteOffset + headerSize, dataSize); + } else { + console.log(`Expected address ${address} but received ${chunkAddress}`); } - } else { - console.log(`Expected address ${address} but received ${chunkAddress}`); } - } - // Deliver payloadView if defined, otherwise pass null - onDataCallback(address, payloadView); + onDataCallback(address, payloadView, bytesCompressed); - if (!response || response.crcError) { - console.log(`CRC error or missing data at address ${address} - delivering whatever we got`); - } else if (payloadView) { - console.log(`Block at ${address} received (${payloadView.byteLength} bytes)`); - } - }, true); // end of send_message + if (!response || response.crcError) { + console.log(`CRC error or missing data at address ${address} - delivering whatever we got`); + } else if (payloadView) { + console.log(`Block at ${address} received (${payloadView.byteLength} bytes)`); + } + }, + true, + ); // end of send_message }; // end of dataflashRead - MspHelper.prototype.sendAdjustmentRanges = function (onCompleteCallback) { let nextFunction = send_next_adjustment_range; From ecd64f67a2faef917f44cff06ecd7e92e1d4c92f Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 20:35:05 +1000 Subject: [PATCH 3/9] chatGpt one retry attempt --- src/js/tabs/onboard_logging.js | 39 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index b741db9954..8423311937 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -491,17 +491,23 @@ onboard_logging.initialize = function (callback) { show_saving_dialog(); + // START PATCH: minimal retry for null/missing blocks + const MAX_SIMPLE_RETRIES = 1; + let simpleRetryCount = 0; + function onChunkRead(chunkAddress, chunkDataView, bytesCompressed) { if (chunkDataView && chunkDataView.byteLength > 0) { - // Always write non-empty data, even if CRC mismatch + // Reset retry counter after a good block + simpleRetryCount = 0; + + // --- ORIGINAL BLOCK WRITE LOGIC --- const blob = new Blob([chunkDataView]); FileSystem.writeChunck(openedFile, blob); nextAddress += chunkDataView.byteLength; - // Track total compressed bytes, if provided if (typeof bytesCompressed === "number") { - if (totalBytesCompressed == null) totalBytesCompressed = 0; // initialize if previously unknown + if (totalBytesCompressed == null) totalBytesCompressed = 0; totalBytesCompressed += bytesCompressed; } @@ -513,23 +519,34 @@ onboard_logging.initialize = function (callback) { } else { mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); } + // --- END ORIGINAL LOGIC --- } else if (chunkDataView && chunkDataView.byteLength === 0) { // Zero-length block → EOF mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); FileSystem.closeFile(openedFile); } else { - // Null block → skip ahead (hard error) - console.warn(`Skipping null block at address ${nextAddress}`); - nextAddress += self.blockSize; - - if (nextAddress >= maxBytes) { - mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); - FileSystem.closeFile(openedFile); - } else { + // Null/missing block + if (simpleRetryCount < MAX_SIMPLE_RETRIES) { + simpleRetryCount++; + console.warn(`Null/missing block at ${nextAddress}, retry ${simpleRetryCount}`); mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); + } else { + console.error( + `Skipping null block at ${nextAddress} after ${MAX_SIMPLE_RETRIES} retry`, + ); + nextAddress += self.blockSize; // Move to next block + simpleRetryCount = 0; // reset counter for next block + + if (nextAddress >= maxBytes) { + mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); + FileSystem.closeFile(openedFile); + } else { + mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); + } } } } + // END PATCH const startTime = new Date().getTime(); // Start timestamp From 4b591de21458cf7144a787caa1b9c072ebd3a59f Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 22:16:20 +1000 Subject: [PATCH 4/9] propertly handle user cancel --- src/js/tabs/onboard_logging.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index 8423311937..e906b11672 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -537,7 +537,7 @@ onboard_logging.initialize = function (callback) { nextAddress += self.blockSize; // Move to next block simpleRetryCount = 0; // reset counter for next block - if (nextAddress >= maxBytes) { + if (saveCancelled || nextAddress >= maxBytes) { mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); FileSystem.closeFile(openedFile); } else { From d652ec8ebf7bcfc21764d6a9b90d5c58ded4eaa2 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 22:49:18 +1000 Subject: [PATCH 5/9] use writechunk --- src/js/FileSystem.js | 6 +++--- src/js/tabs/logging.js | 2 +- src/js/tabs/onboard_logging.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/js/FileSystem.js b/src/js/FileSystem.js index cd516281e2..8f2f98327d 100644 --- a/src/js/FileSystem.js +++ b/src/js/FileSystem.js @@ -98,12 +98,12 @@ class FileSystem { return await fileHandle.createWritable(options); } - async writeChunck(writable, chunk) { - await writable.write(chunk); + async writeChunk(writable, chunk) { + writable.write(chunk); } async closeFile(writable) { - await writable.close(); + writable.close(); } } diff --git a/src/js/tabs/logging.js b/src/js/tabs/logging.js index c00aa45f57..b8124f9638 100644 --- a/src/js/tabs/logging.js +++ b/src/js/tabs/logging.js @@ -257,7 +257,7 @@ logging.initialize = function (callback) { } function append_to_file(data) { - FileSystem.writeChunck(logging.fileWriter, new Blob([data], { type: "text/plain" })).catch((error) => { + FileSystem.writeChunk(logging.fileWriter, new Blob([data], { type: "text/plain" })).catch((error) => { console.error("Error appending to file: ", error); }); } diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index e906b11672..2b8c85e18c 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -502,7 +502,7 @@ onboard_logging.initialize = function (callback) { // --- ORIGINAL BLOCK WRITE LOGIC --- const blob = new Blob([chunkDataView]); - FileSystem.writeChunck(openedFile, blob); + FileSystem.writeChunk(openedFile, blob); nextAddress += chunkDataView.byteLength; From 68fe053f77eb9c3e8984d247d6870ab562307496 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 22:57:34 +1000 Subject: [PATCH 6/9] try 5 retries not just one --- src/js/tabs/onboard_logging.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index 2b8c85e18c..0feaf3d47c 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -492,7 +492,7 @@ onboard_logging.initialize = function (callback) { show_saving_dialog(); // START PATCH: minimal retry for null/missing blocks - const MAX_SIMPLE_RETRIES = 1; + const MAX_SIMPLE_RETRIES = 5; let simpleRetryCount = 0; function onChunkRead(chunkAddress, chunkDataView, bytesCompressed) { From 6d1f80fdb1bd1d8cb2c2331621e526ac10cd1143 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 23:21:08 +1000 Subject: [PATCH 7/9] throttle warnings --- src/js/tabs/onboard_logging.js | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index 0feaf3d47c..ed92a64095 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -495,6 +495,8 @@ onboard_logging.initialize = function (callback) { const MAX_SIMPLE_RETRIES = 5; let simpleRetryCount = 0; + const startTime = new Date().getTime(); // Start timestamp + function onChunkRead(chunkAddress, chunkDataView, bytesCompressed) { if (chunkDataView && chunkDataView.byteLength > 0) { // Reset retry counter after a good block @@ -528,27 +530,20 @@ onboard_logging.initialize = function (callback) { // Null/missing block if (simpleRetryCount < MAX_SIMPLE_RETRIES) { simpleRetryCount++; - console.warn(`Null/missing block at ${nextAddress}, retry ${simpleRetryCount}`); + if (simpleRetryCount % 2 === 1) { + console.warn(`Null/missing block at ${nextAddress}, retry ${simpleRetryCount}`); + } mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); } else { console.error( - `Skipping null block at ${nextAddress} after ${MAX_SIMPLE_RETRIES} retry`, + `Skipping null block at ${nextAddress} after ${MAX_SIMPLE_RETRIES} retries`, ); - nextAddress += self.blockSize; // Move to next block - simpleRetryCount = 0; // reset counter for next block - - if (saveCancelled || nextAddress >= maxBytes) { - mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); - FileSystem.closeFile(openedFile); - } else { - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - } + nextAddress += self.blockSize; // Move to next block only after retries exhausted + simpleRetryCount = 0; + mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); } } } - // END PATCH - - const startTime = new Date().getTime(); // Start timestamp // Fetch the initial block FileSystem.openFile(fileWriter).then((file) => { From af0f746f2e4831d93c91a63b64eb9f6ce676cef2 Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Sun, 28 Sep 2025 23:28:26 +1000 Subject: [PATCH 8/9] variable retry delay mac specific variable backoff queue baSED ATTEMPT try a retry delay --- src/js/tabs/onboard_logging.js | 148 +++++++++++++++++---------------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index ed92a64095..3fa278e7ad 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -477,83 +477,91 @@ onboard_logging.initialize = function (callback) { } function flash_save_begin() { - if (GUI.connected_to) { - self.blockSize = self.BLOCK_SIZE; - - // Begin by refreshing the occupied size in case it changed while the tab was open - flash_update_summary(function () { - const maxBytes = FC.DATAFLASH.usedSize; - - let openedFile; - prepare_file(function (fileWriter) { - let nextAddress = 0; - let totalBytesCompressed = 0; - - show_saving_dialog(); + if (!GUI.connected_to) return; + + self.blockSize = self.BLOCK_SIZE; + + flash_update_summary(async () => { + const maxBytes = FC.DATAFLASH.usedSize; + let openedFile; + let totalBytesCompressed = 0; + show_saving_dialog(); + + const MAX_SIMPLE_RETRIES = 5; + const BASE_RETRY_BACKOFF_MS = 30; // starting backoff + const INTER_BLOCK_DELAY_MS = 2; // small delay between successful blocks + const startTime = new Date().getTime(); + + prepare_file(async (fileWriter) => { + openedFile = await FileSystem.openFile(fileWriter); + let nextAddress = 0; + + async function readNextBlock() { + if (saveCancelled || nextAddress >= maxBytes) { + mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); + await FileSystem.closeFile(openedFile); + return; + } - // START PATCH: minimal retry for null/missing blocks - const MAX_SIMPLE_RETRIES = 5; let simpleRetryCount = 0; - const startTime = new Date().getTime(); // Start timestamp - - function onChunkRead(chunkAddress, chunkDataView, bytesCompressed) { - if (chunkDataView && chunkDataView.byteLength > 0) { - // Reset retry counter after a good block - simpleRetryCount = 0; - - // --- ORIGINAL BLOCK WRITE LOGIC --- - const blob = new Blob([chunkDataView]); - FileSystem.writeChunk(openedFile, blob); - - nextAddress += chunkDataView.byteLength; - - if (typeof bytesCompressed === "number") { - if (totalBytesCompressed == null) totalBytesCompressed = 0; - totalBytesCompressed += bytesCompressed; - } - - $(".dataflash-saving progress").attr("value", (nextAddress / maxBytes) * 100); - - if (saveCancelled || nextAddress >= maxBytes) { - mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); - FileSystem.closeFile(openedFile); - } else { - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - } - // --- END ORIGINAL LOGIC --- - } else if (chunkDataView && chunkDataView.byteLength === 0) { - // Zero-length block → EOF - mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); - FileSystem.closeFile(openedFile); - } else { - // Null/missing block - if (simpleRetryCount < MAX_SIMPLE_RETRIES) { - simpleRetryCount++; - if (simpleRetryCount % 2 === 1) { - console.warn(`Null/missing block at ${nextAddress}, retry ${simpleRetryCount}`); + async function attemptRead() { + mspHelper.dataflashRead( + nextAddress, + self.blockSize, + async (chunkAddress, chunkDataView, bytesCompressed) => { + if (chunkDataView && chunkDataView.byteLength > 0) { + // Reset retry counter + simpleRetryCount = 0; + + // Write and await completion to prevent Mac buffer stalls + const blob = new Blob([chunkDataView]); + await FileSystem.writeChunk(openedFile, blob); + + nextAddress += chunkDataView.byteLength; + if (typeof bytesCompressed === "number") { + totalBytesCompressed = (totalBytesCompressed || 0) + bytesCompressed; + } + + $(".dataflash-saving progress").attr("value", (nextAddress / maxBytes) * 100); + + // Small delay between blocks to reduce Mac Chrome hangs + setTimeout(readNextBlock, INTER_BLOCK_DELAY_MS); + } else if (chunkDataView && chunkDataView.byteLength === 0) { + // EOF + mark_saving_dialog_done(startTime, nextAddress, totalBytesCompressed); + await FileSystem.closeFile(openedFile); + } else { + // Null/missing block + if (simpleRetryCount < MAX_SIMPLE_RETRIES) { + simpleRetryCount++; + const backoff = BASE_RETRY_BACKOFF_MS * simpleRetryCount; + if (simpleRetryCount % 2 === 1) { + console.warn( + `Null/missing block at ${nextAddress}, retry ${simpleRetryCount}, backoff ${backoff}ms`, + ); + } + setTimeout(attemptRead, backoff); + } else { + console.error( + `Skipping null block at ${nextAddress} after ${MAX_SIMPLE_RETRIES} retries`, + ); + nextAddress += self.blockSize; + readNextBlock(); + } } - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - } else { - console.error( - `Skipping null block at ${nextAddress} after ${MAX_SIMPLE_RETRIES} retries`, - ); - nextAddress += self.blockSize; // Move to next block only after retries exhausted - simpleRetryCount = 0; - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - } - } + }, + ); } - // Fetch the initial block - FileSystem.openFile(fileWriter).then((file) => { - openedFile = file; - mspHelper.dataflashRead(nextAddress, self.blockSize, onChunkRead); - }); - }); + attemptRead(); + } + + // Start reading the first block + readNextBlock(); }); - } - } + }); + } // end of flash_save_begin function prepare_file(onComplete) { const prefix = "BLACKBOX_LOG"; From 83e8efd884203936ee1a0213db9282c95fab3fdc Mon Sep 17 00:00:00 2001 From: ctzsnooze Date: Mon, 29 Sep 2025 01:11:57 +1000 Subject: [PATCH 9/9] tweak timeout values --- src/js/tabs/onboard_logging.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/js/tabs/onboard_logging.js b/src/js/tabs/onboard_logging.js index 3fa278e7ad..ce6062884c 100644 --- a/src/js/tabs/onboard_logging.js +++ b/src/js/tabs/onboard_logging.js @@ -488,8 +488,8 @@ onboard_logging.initialize = function (callback) { show_saving_dialog(); const MAX_SIMPLE_RETRIES = 5; - const BASE_RETRY_BACKOFF_MS = 30; // starting backoff - const INTER_BLOCK_DELAY_MS = 2; // small delay between successful blocks + const BASE_RETRY_BACKOFF_MS = 50; // starting backoff + const INTER_BLOCK_DELAY_MS = 10; // small delay between successful blocks const startTime = new Date().getTime(); prepare_file(async (fileWriter) => {