fix(Comm): increase json buffer size to handle large thumbnail responses from DSF

Author: AndyEveritt

src/Comm/JsonDecoder.cpp

@@ -464,7 +464,27 @@ namespace Comm
 		}
 	}
 
-	// Look for combining characters in the string value and convert them if possible
+	/**
+	 * @brief Convert Unicode combining characters in the string value to their precomposed equivalents if possible.
+	 *
+	 * Some text editors save accented characters as a combination of a base character and a combining diacritical mark.
+	 * For example, "á" might be saved as "a" followed by a combining acute accent. This function looks for such
+	 * combinations in the string value and converts them to their precomposed equivalents if possible. This is done by
+	 * looking for UTF-8 encoded Unicode characters in the string value and checking if they are combining diacritical
+	 * marks that we handle. If they are, we look at the previous character and see if it can be combined with the
+	 * diacritical mark to form a precomposed character. If it can, we replace the previous character with the
+	 * precomposed character and remove the diacritical mark from the string.
+	 *
+	 * @note This function is causes significant slow downs when processing large JSON responses (SBC thumbnails). It is
+	 * currently disabled since LVGL has some limited support for rendering multi codepoint graphemes, and the main
+	 * issue with not converting is that some diacritical marks are rendered as separate characters instead of being
+	 * combined with the previous character. If this becomes a bigger issue, we could consider only converting certain
+	 * common combinations of characters and diacritical marks, or we could look into optimizing the function to reduce
+	 * the performance impact.
+	 *
+	 * @note Since the Json parser is due to be reworked anyway. It will be disabled until then unless there is an
+	 * explicit complaint.
+	 */
 	void JsonDecoder::ConvertUnicode()
 	{
 		ZoneScoped;
@@ -852,7 +872,7 @@ namespace Comm
 					switch (c)
 					{
 					case '"':
-						ConvertUnicode();
+						// ConvertUnicode();
 						ProcessField();
 						m_state = jsEndVal;
 						break;

src/Comm/Thumbnail.cpp

@@ -335,7 +335,7 @@ bool DeleteCachedThumbnail(std::string_view filepath)
 {
 	LOG_INFO("Deleting thumbnail for {:s}", filepath);
 	bool ret = true;
-	ret &= std::filesystem::remove(GetThumbnailPath(filepath, false));
+	// ret &= std::filesystem::remove(GetThumbnailPath(filepath, false));
 	ret &= std::filesystem::remove(GetThumbnailPath(filepath, true));
 	return ret;
 }

src/Configuration.h

@@ -57,7 +57,7 @@ constexpr std::string_view DEFAULT_FILAMENTS_FILE = "filaments.csv";
 constexpr std::string_view DEFAULT_HEIGHTMAPS_FILE = "heightmaps.csv";
 
 /* Thumbnails */
-constexpr std::chrono::milliseconds FILE_CACHE_REQUEST_TIMEOUT = 5000ms;
+constexpr std::chrono::milliseconds FILE_CACHE_REQUEST_TIMEOUT = 10000ms;
 constexpr size_t MAX_THUMBNAIL_CACHE_PIXELS = 64; // Largest pixel width/height thumbnail that is allowed to be cached
 constexpr std::chrono::milliseconds BACKGROUND_FILE_CACHE_POLL_INTERVAL = 500ms;
 constexpr size_t MAX_FILEINFO_REQUESTS = 1;
@@ -68,7 +68,7 @@ constexpr size_t MAX_ARRAY_NESTING = 4;
 constexpr size_t MAX_JSON_ID_LENGTH = 200;
 // 4096 is the largest needed for a Duet in standalone mode. But in
 // SBC mode, network responses can be much larger. This is most evident with `rr_thumbnail`
-constexpr size_t MAX_JSON_VALUE_LENGTH = 4096 * 5;
+constexpr size_t MAX_JSON_VALUE_LENGTH = 4096 * 20;
 
 /* Network */
 // Duet 2 seems to only support 3 concurrent connections. We need 1 connection for synchronous requests, so we can

src/image/png.cpp

@@ -2,7 +2,9 @@
 
 #include "png.h"
 #include <algorithm>
+#include <errno.h>
 #include <stdio.h>
+#include <unistd.h>
 
 PNG::PNG()
 	: m_imageFileName(nullptr)
@@ -46,6 +48,22 @@ bool PNG::Close()
 		return true;
 	}
 	LOG_DBG("Closing file {:s}", m_imageFileName);
+	if (fflush(m_imageFile) != 0)
+	{
+		LOG_ERROR("Failed to fflush file {:s}: {}", m_imageFileName, errno);
+		/* attempt to continue to close */
+	}
+
+	int fd = fileno(m_imageFile);
+	if (fd != -1)
+	{
+		if (fsync(fd) != 0)
+		{
+			LOG_ERROR("Failed to fsync file {:s}: {}", m_imageFileName, errno);
+			/* attempt to continue to close */
+		}
+	}
+
 	if (fclose(m_imageFile) != 0)
 	{
 		LOG_ERROR("Failed to close file {:s}", m_imageFileName);
@@ -62,6 +80,27 @@ size_t PNG::appendData(unsigned char data[], int size)
 		LOG_WARN("File {:s} not open", m_imageFileName);
 		return 0;
 	}
-	dataSize += size;
-	return fwrite(data, 1, size, m_imageFile);
+	if (size <= 0)
+	{
+		return 0;
+	}
+
+	size_t totalWritten = 0;
+	while (totalWritten < static_cast<size_t>(size))
+	{
+		size_t toWrite = static_cast<size_t>(size) - totalWritten;
+		size_t written = fwrite(data + totalWritten, 1, toWrite, m_imageFile);
+		if (written == 0)
+		{
+			if (ferror(m_imageFile))
+			{
+				LOG_ERROR("Failed to write PNG data to {:s}: {}", m_imageFileName, errno);
+			}
+			break;
+		}
+		totalWritten += written;
+	}
+
+	dataSize += totalWritten;
+	return totalWritten;
 }

tests/ref_imgs/diacritic/combined_vs_combining.png

@@ -0,0 +1,129 @@
+/*
+ * test_diacritic.cpp
+ *
+ * Created on: 2026-02-23
+ * Author: automated-test
+ */
+
+#include "Debug.h"
+#include "UI/Components/LVGL/LvLabel.h"
+#include "test_utils/UiTestSuite.h"
+#include <cstring>
+#include <gtest/gtest.h>
+#include <string>
+
+using namespace UI;
+
+class TestDiacritic : public UiTestSuite
+{
+  public:
+	TestDiacritic() {}
+};
+
+TEST_F(TestDiacritic, CombinedVsCombining)
+{
+	// Create two labels on the test screen
+	LvLabel label_precomposed("label_pre", screen);
+	LvLabel label_decomposed("label_decomp", screen);
+
+	// Examples: pairs of (precomposed, decomposed)
+	// This list covers common diacritics used across European languages.
+	struct Pair
+	{
+		const char* pre;
+		const char* decomp;
+		const char* desc;
+	} pairs[] = {
+		{"\u00E1", "a\u0301", "a + acute -> á"}, // á
+		{"\u00E0", "a\u0300", "a + grave -> à"}, // à
+		{"\u00E2", "a\u0302", "a + circumflex -> â"},
+		{"\u00E3", "a\u0303", "a + tilde -> ã"},
+		{"\u00E4", "a\u0308", "a + diaeresis -> ä"},
+		{"\u00E5", "a\u030A", "a + ring -> å"},
+		{"\u0105", "a\u0328", "a + ogonek -> ą"},
+
+		{"\u00E9", "e\u0301", "e + acute -> é"}, // é
+		{"\u00E8", "e\u0300", "e + grave -> è"}, // è
+		{"\u00EA", "e\u0302", "e + circumflex -> ê"},
+		{"\u00EB", "e\u0308", "e + diaeresis -> ë"},
+		{"\u0119", "e\u0328", "e + ogonek -> ę"},
+		{"\u011B", "e\u030C", "e + caron -> ě"},
+
+		{"\u00ED", "i\u0301", "i + acute -> í"},
+		{"\u00EC", "i\u0300", "i + grave -> ì"},
+		{"\u00EE", "i\u0302", "i + circumflex -> î"},
+		{"\u00EF", "i\u0308", "i + diaeresis -> ï"},
+		{"\u012B", "i\u0304", "i + macron -> ī"},
+
+		{"\u00F3", "o\u0301", "o + acute -> ó"},
+		{"\u00F2", "o\u0300", "o + grave -> ò"},
+		{"\u00F4", "o\u0302", "o + circumflex -> ô"},
+		{"\u00F5", "o\u0303", "o + tilde -> õ"},
+		{"\u00F6", "o\u0308", "o + diaeresis -> ö"},
+		{"\u00F8", "o\u0338", "o + stroke (approx) -> ø"},
+
+		{"\u00F1", "n\u0303", "n + tilde -> ñ"},
+		{"\u0144", "n\u0301", "n + acute -> ń"},
+
+		{"\u00E7", "c\u0327", "c + cedilla -> ç"},
+		{"\u0107", "c\u0301", "c + acute -> ć"},
+		{"\u010D", "c\u030C", "c + caron -> č"},
+
+		{"\u015B", "s\u0301", "s + acute -> ś"},
+		{"\u0161", "s\u030C", "s + caron -> š"},
+
+		{"\u017A", "z\u0301", "z + acute -> ź"},
+		{"\u017C", "z\u0307", "z + dot -> ż"},
+		{"\u017E", "z\u030C", "z + caron -> ž"},
+
+		{"\u0142", "l\u0335", "l + stroke (approx) -> ł"},
+		{"\u0159", "r\u030C", "r + caron -> ř"},
+		{"\u0165", "t\u030C", "t + caron -> ť"},
+
+		{"\u0117", "e\u0307", "e + dot -> ė"},
+		{"\u012F", "i\u0328", "i + ogonek -> į"},
+		{"\u0173", "u\u0328", "u + ogonek -> ų"},
+
+		{"\u00FC", "u\u0308", "u + diaeresis -> ü"},
+		{"\u016B", "u\u0304", "u + macron -> ū"},
+	};
+
+	// Create labels for each pair and verify the stored text matches what we set
+	for (size_t i = 0; i < std::size(pairs); ++i)
+	{
+		std::string name_pre = std::string("pre_") + std::to_string(i);
+		std::string name_de = std::string("de_") + std::to_string(i);
+
+		LvLabel lp(name_pre, screen);
+		LvLabel ld(name_de, screen);
+
+		lp.setText(pairs[i].pre);
+		ld.setText(pairs[i].decomp);
+
+		ASSERT_STREQ(lp.getText(), pairs[i].pre) << "precomposed failed for " << pairs[i].desc;
+		ASSERT_STREQ(ld.getText(), pairs[i].decomp) << "decomposed failed for " << pairs[i].desc;
+
+		// Precomposed and decomposed byte lengths will commonly differ
+		EXPECT_NE(std::strlen(lp.getText()), std::strlen(ld.getText())) << pairs[i].desc;
+	}
+
+	// Compose all pairs into single strings for a combined display on the two top-level labels
+	std::string combined_pre;
+	std::string combined_decomp;
+	for (size_t i = 0; i < std::size(pairs); ++i)
+	{
+		if (!combined_pre.empty())
+		{
+			combined_pre += " ";
+			combined_decomp += " ";
+		}
+		combined_pre += pairs[i].pre;
+		combined_decomp += pairs[i].decomp;
+	}
+
+	// Use the labels created at the top of the test to show the combined strings
+	label_precomposed.setText(combined_pre);
+	label_decomposed.setText(combined_decomp);
+
+	EXPECT_EQUAL_SCREENSHOT("diacritic/combined_vs_combining.png");
+}