Don't apply any text encoding to binary input data

vkrause · vkrause · commit bd3dba04ba5d · 2023-04-26T17:48:24.000+02:00
This fixes barcode generation for arbitrary binary input failing entirely
or producing incomplete/incorrect results.

Observed for example with Hungarian domestic railway tickets (zlib
compressed data in PDF417).
diff --git a/core/src/TextDecoder.cpp b/core/src/TextDecoder.cpp
@@ -29,6 +29,12 @@ void TextDecoder::Append(std::string& str, const uint8_t* bytes, size_t length,
 	if (eci == -1)
 		eci = 899; // Binary
 
+	if (eci == 899) {
+		str.reserve(str_len + length);
+		std::copy(bytes, bytes + length, std::back_inserter(str));
+		return;
+	}
+
 	int error_number = zueci_dest_len_utf8(eci, bytes, bytes_len, replacement, flags, &utf8_len);
 	if (error_number >= ZUECI_ERROR)
 		throw std::runtime_error("zueci_dest_len_utf8 failed");
@@ -46,9 +52,14 @@ void TextDecoder::Append(std::string& str, const uint8_t* bytes, size_t length,
 
 void TextDecoder::Append(std::wstring& str, const uint8_t* bytes, size_t length, CharacterSet charset)
 {
-	std::string u8str;
-	Append(u8str, bytes, length, charset);
-	str.append(FromUtf8(u8str));
+	if (charset == CharacterSet::BINARY) {
+		str.reserve(str.size() + length);
+		std::copy(bytes, bytes + length, std::back_inserter(str));
+	} else {
+		std::string u8str;
+		Append(u8str, bytes, length, charset);
+		str.append(FromUtf8(u8str));
+	}
 }
 
 /**
diff --git a/core/src/TextEncoder.cpp b/core/src/TextEncoder.cpp
@@ -24,6 +24,11 @@ void TextEncoder::GetBytes(const std::string& str, CharacterSet charset, std::st
 	if (eci == -1)
 		eci = 899; // Binary
 
+	if (eci == 899) {
+		bytes = str;
+		return;
+	}
+
 	bytes.clear();
 
 	int error_number = zueci_dest_len_eci(eci, reinterpret_cast<const unsigned char *>(str.data()), str_len, &eci_len);
@@ -44,7 +49,13 @@ void TextEncoder::GetBytes(const std::string& str, CharacterSet charset, std::st
 
 void TextEncoder::GetBytes(const std::wstring& str, CharacterSet charset, std::string& bytes)
 {
-	GetBytes(ToUtf8(str), charset, bytes);
+	if (charset == CharacterSet::BINARY) {
+		bytes.clear();
+		bytes.reserve(str.size());
+		std::copy(str.begin(), str.end(), std::back_inserter(bytes));
+	} else {
+		GetBytes(ToUtf8(str), charset, bytes);
+	}
 }
 
 } // ZXing
diff --git a/test/unit/TextEncoderTest.cpp b/test/unit/TextEncoderTest.cpp
@@ -60,7 +60,7 @@ TEST(TextEncoderTest, FullCycleEncodeDecode)
 	EnDeCode(CharacterSet::UTF32BE, u8"\u20AC", std::string("\x00\x00\x20\xAC", 4)); // EURO SIGN
 	EnDeCode(CharacterSet::UTF32LE, u8"\u20AC", std::string("\xAC\x20\x00\x00", 4)); // EURO SIGN
 //	EnDeCode(CharacterSet::ISO646_Inv, "%", "%");
-	EnDeCode(CharacterSet::BINARY, u8"\u0080\u00FF", "\x80\xFF");
-	EnDeCode(CharacterSet::Unknown, u8"\u0080", "\x80"); // Treated as binary
-	EnDeCode(CharacterSet::EUC_JP, u8"\u0080", "\x80"); // Not supported, treated as binary
+	EnDeCode(CharacterSet::BINARY, u8"\u0080\u00FF", "\xC2\x80\xC3\xBF");
+	EnDeCode(CharacterSet::Unknown, u8"\u0080", "\xC2\x80"); // Treated as binary
+	EnDeCode(CharacterSet::EUC_JP, u8"\u0080", "\xC2\x80"); // Not supported, treated as binary
 }
