Line data Source code
1 : /**
2 : * @file Utf8Sanitizer.cpp
3 : * @brief Implementation of UTF-8 validation and CP1252→UTF-8 transcoding.
4 : *
5 : * The CP1252→Unicode mapping table is derived from the official Microsoft mapping:
6 : * https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT
7 : *
8 : * Undefined slots at 0x81, 0x8D, 0x8F, 0x90, 0x9D map to U+FFFD.
9 : */
10 :
11 : #include "Utf8Sanitizer.h"
12 :
13 : #include <google/protobuf/descriptor.h>
14 : #include <google/protobuf/message.h>
15 :
16 : #include <cstdint>
17 : #include <cstdio>
18 : #include <cstdlib>
19 : #include <string>
20 :
21 : namespace Odb::Lib::Text
22 : {
23 : namespace
24 : {
25 : // clang-format off
26 : /**
27 : * @brief Windows-1252 to Unicode code point mapping table.
28 : *
29 : * Index is the CP1252 byte value (0x00–0xFF).
30 : * Value is the corresponding Unicode code point.
31 : *
32 : * - 0x00–0x7F: ASCII (identity mapping)
33 : * - 0x80–0x9F: CP1252 special characters (differs from ISO-8859-1)
34 : * - 0xA0–0xFF: ISO-8859-1 upper half (mostly Latin-1 supplement)
35 : */
36 : constexpr uint32_t kCp1252ToUnicode[256] = {
37 : // 0x00-0x7F: ASCII (identity)
38 : 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
39 : 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
40 : 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
41 : 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F,
42 : 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
43 : 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
44 : 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
45 : 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
46 : 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
47 : 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
48 : 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
49 : 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
50 : 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
51 : 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
52 : 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
53 : 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F,
54 :
55 : // 0x80-0x9F: CP1252 special characters
56 : 0x20AC, // 0x80: EURO SIGN
57 : 0xFFFD, // 0x81: UNDEFINED → REPLACEMENT CHARACTER
58 : 0x201A, // 0x82: SINGLE LOW-9 QUOTATION MARK
59 : 0x0192, // 0x83: LATIN SMALL LETTER F WITH HOOK
60 : 0x201E, // 0x84: DOUBLE LOW-9 QUOTATION MARK
61 : 0x2026, // 0x85: HORIZONTAL ELLIPSIS
62 : 0x2020, // 0x86: DAGGER
63 : 0x2021, // 0x87: DOUBLE DAGGER
64 : 0x02C6, // 0x88: MODIFIER LETTER CIRCUMFLEX ACCENT
65 : 0x2030, // 0x89: PER MILLE SIGN
66 : 0x0160, // 0x8A: LATIN CAPITAL LETTER S WITH CARON
67 : 0x2039, // 0x8B: SINGLE LEFT-POINTING ANGLE QUOTATION MARK
68 : 0x0152, // 0x8C: LATIN CAPITAL LIGATURE OE
69 : 0xFFFD, // 0x8D: UNDEFINED → REPLACEMENT CHARACTER
70 : 0x017D, // 0x8E: LATIN CAPITAL LETTER Z WITH CARON
71 : 0xFFFD, // 0x8F: UNDEFINED → REPLACEMENT CHARACTER
72 : 0xFFFD, // 0x90: UNDEFINED → REPLACEMENT CHARACTER
73 : 0x2018, // 0x91: LEFT SINGLE QUOTATION MARK
74 : 0x2019, // 0x92: RIGHT SINGLE QUOTATION MARK
75 : 0x201C, // 0x93: LEFT DOUBLE QUOTATION MARK
76 : 0x201D, // 0x94: RIGHT DOUBLE QUOTATION MARK
77 : 0x2022, // 0x95: BULLET
78 : 0x2013, // 0x96: EN DASH
79 : 0x2014, // 0x97: EM DASH
80 : 0x02DC, // 0x98: SMALL TILDE
81 : 0x2122, // 0x99: TRADE MARK SIGN
82 : 0x0161, // 0x9A: LATIN SMALL LETTER S WITH CARON
83 : 0x203A, // 0x9B: SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
84 : 0x0153, // 0x9C: LATIN SMALL LIGATURE OE
85 : 0xFFFD, // 0x9D: UNDEFINED → REPLACEMENT CHARACTER
86 : 0x017E, // 0x9E: LATIN SMALL LETTER Z WITH CARON
87 : 0x0178, // 0x9F: LATIN CAPITAL LETTER Y WITH DIAERESIS
88 :
89 : // 0xA0-0xFF: ISO-8859-1 upper half (Latin-1 Supplement)
90 : 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
91 : 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
92 : 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
93 : 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
94 : 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
95 : 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
96 : 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
97 : 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
98 : 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
99 : 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
100 : 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
101 : 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF
102 : };
103 : // clang-format on
104 :
105 : /**
106 : * @brief Encodes a Unicode code point as UTF-8.
107 : * @param codepoint The Unicode code point (U+0000 to U+10FFFF)
108 : * @param output Pointer to output buffer (must have space for 4 bytes)
109 : * @return Number of bytes written (1-4)
110 : */
111 77 : inline int EncodeUtf8Codepoint(uint32_t codepoint, char* output) noexcept
112 : {
113 77 : if (codepoint <= 0x7F)
114 : {
115 0 : output[0] = static_cast<char>(codepoint);
116 0 : return 1;
117 : }
118 77 : else if (codepoint <= 0x7FF)
119 : {
120 53 : output[0] = static_cast<char>(0xC0 | (codepoint >> 6));
121 53 : output[1] = static_cast<char>(0x80 | (codepoint & 0x3F));
122 53 : return 2;
123 : }
124 24 : else if (codepoint <= 0xFFFF)
125 : {
126 24 : output[0] = static_cast<char>(0xE0 | (codepoint >> 12));
127 24 : output[1] = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
128 24 : output[2] = static_cast<char>(0x80 | (codepoint & 0x3F));
129 24 : return 3;
130 : }
131 : else
132 : {
133 0 : output[0] = static_cast<char>(0xF0 | (codepoint >> 18));
134 0 : output[1] = static_cast<char>(0x80 | ((codepoint >> 12) & 0x3F));
135 0 : output[2] = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
136 0 : output[3] = static_cast<char>(0x80 | (codepoint & 0x3F));
137 0 : return 4;
138 : }
139 : }
140 :
141 : /**
142 : * @brief Returns the length in bytes (1-4) of the valid UTF-8 sequence
143 : * starting at bytes[i], or 0 if no valid sequence starts there.
144 : *
145 : * Shared by validation (IsValidUtf8) and repair (ToUtf8) so both use
146 : * exactly the same acceptance rules: continuation byte ranges, no
147 : * overlong encodings, no surrogate halves, no code points > U+10FFFF.
148 : */
149 616343063 : std::size_t Utf8SequenceLengthAt(const uint8_t* bytes, std::size_t i, std::size_t size) noexcept
150 : {
151 616343063 : const uint8_t byte = bytes[i];
152 :
153 : // Single byte: 0xxxxxxx
154 616343063 : if (byte <= 0x7F)
155 : {
156 616342800 : return 1;
157 : }
158 :
159 : // Two bytes: 110xxxxx 10xxxxxx
160 263 : if ((byte & 0xE0) == 0xC0)
161 : {
162 155 : if (i + 1 >= size) return 0;
163 134 : if ((bytes[i + 1] & 0xC0) != 0x80) return 0;
164 :
165 : // Check for overlong encoding
166 93 : const uint32_t codepoint = ((byte & 0x1F) << 6) | (bytes[i + 1] & 0x3F);
167 93 : if (codepoint < 0x80) return 0;
168 :
169 90 : return 2;
170 : }
171 :
172 : // Three bytes: 1110xxxx 10xxxxxx 10xxxxxx
173 108 : if ((byte & 0xF0) == 0xE0)
174 : {
175 18 : if (i + 2 >= size) return 0;
176 16 : if ((bytes[i + 1] & 0xC0) != 0x80 || (bytes[i + 2] & 0xC0) != 0x80)
177 0 : return 0;
178 :
179 16 : const uint32_t codepoint =
180 16 : ((byte & 0x0F) << 12) | ((bytes[i + 1] & 0x3F) << 6) | (bytes[i + 2] & 0x3F);
181 :
182 : // Check for overlong encoding
183 16 : if (codepoint < 0x800) return 0;
184 :
185 : // Reject surrogate halves U+D800..U+DFFF
186 16 : if (codepoint >= 0xD800 && codepoint <= 0xDFFF) return 0;
187 :
188 13 : return 3;
189 : }
190 :
191 : // Four bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
192 90 : if ((byte & 0xF8) == 0xF0)
193 : {
194 38 : if (i + 3 >= size) return 0;
195 38 : if ((bytes[i + 1] & 0xC0) != 0x80 || (bytes[i + 2] & 0xC0) != 0x80 ||
196 2 : (bytes[i + 3] & 0xC0) != 0x80)
197 36 : return 0;
198 :
199 2 : const uint32_t codepoint =
200 2 : ((byte & 0x07) << 18) | ((bytes[i + 1] & 0x3F) << 12) |
201 2 : ((bytes[i + 2] & 0x3F) << 6) | (bytes[i + 3] & 0x3F);
202 :
203 : // Check for overlong encoding
204 2 : if (codepoint < 0x10000) return 0;
205 :
206 : // Reject code points above U+10FFFF
207 2 : if (codepoint > 0x10FFFF) return 0;
208 :
209 2 : return 4;
210 : }
211 :
212 : // Invalid leading byte (includes bare continuation bytes)
213 52 : return 0;
214 : }
215 :
216 : #ifndef NDEBUG
217 : /**
218 : * @brief Reports an invalid UTF-8 value found in a message field, then aborts.
219 : *
220 : * Prints the field path and a hex-escaped sample of the offending value to
221 : * stderr. Portable replacement for glibc's __assert_fail (not available on
222 : * MSVC/Windows).
223 : */
224 0 : [[noreturn]] void FailOnInvalidUtf8(const std::string& fieldPath, const std::string& value)
225 : {
226 0 : std::fprintf(stderr, "Utf8Sanitizer assertion failed: invalid UTF-8 in %s: \"", fieldPath.c_str());
227 0 : constexpr std::size_t kMaxSampleLen = 32;
228 0 : const bool truncated = value.size() > kMaxSampleLen;
229 0 : const std::size_t sampleLen = truncated ? kMaxSampleLen : value.size();
230 0 : for (std::size_t i = 0; i < sampleLen; ++i)
231 : {
232 0 : const unsigned char c = static_cast<unsigned char>(value[i]);
233 0 : if (c >= 0x20 && c < 0x7F)
234 : {
235 0 : std::fputc(c, stderr);
236 : }
237 : else
238 : {
239 0 : std::fprintf(stderr, "\\x%02X", c);
240 : }
241 : }
242 0 : if (truncated)
243 : {
244 0 : std::fprintf(stderr, "...");
245 : }
246 0 : std::fprintf(stderr, "\"\n");
247 0 : std::abort();
248 : }
249 :
250 : void AssertMessageStringsAreValidUtf8(const google::protobuf::Message& msg, std::string& fieldPath);
251 :
252 : /**
253 : * @brief Validates a single string value, aborting (in debug builds) if invalid.
254 : */
255 213841123 : void AssertStringIsValidUtf8(const std::string& value, const std::string& fieldPath)
256 : {
257 213841123 : if (!value.empty() && !IsValidUtf8(value))
258 : {
259 0 : FailOnInvalidUtf8(fieldPath, value);
260 : }
261 213841123 : }
262 :
263 : /**
264 : * @brief Recursively validates every `string` field reachable from @p msg.
265 : *
266 : * Handles singular strings, repeated strings, and strings nested in
267 : * sub-messages. Map fields are covered implicitly: protobuf exposes a
268 : * map<K,V> field as a repeated message of synthetic MapEntry messages,
269 : * so recursing into repeated message fields validates map keys and values.
270 : * `bytes` fields are skipped (arbitrary binary data is allowed there).
271 : *
272 : * @param msg The message to validate
273 : * @param fieldPath Dot-separated path of the current message, extended as we recurse
274 : */
275 131047659 : void AssertMessageStringsAreValidUtf8(const google::protobuf::Message& msg, std::string& fieldPath)
276 : {
277 131047659 : const google::protobuf::Descriptor* descriptor = msg.GetDescriptor();
278 131047659 : const google::protobuf::Reflection* reflection = msg.GetReflection();
279 :
280 1235127149 : for (int i = 0; i < descriptor->field_count(); ++i)
281 : {
282 1104079490 : const google::protobuf::FieldDescriptor* field = descriptor->field(i);
283 1104079490 : const std::size_t basePathLen = fieldPath.size();
284 1104079490 : fieldPath.append(".").append(field->name());
285 :
286 1104079490 : if (field->is_repeated())
287 : {
288 63668262 : const int count = reflection->FieldSize(msg, field);
289 63668262 : if (field->type() == google::protobuf::FieldDescriptor::TYPE_STRING)
290 : {
291 18007 : for (int j = 0; j < count; ++j)
292 : {
293 3897 : AssertStringIsValidUtf8(reflection->GetRepeatedString(msg, field, j), fieldPath);
294 : }
295 : }
296 63654152 : else if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE)
297 : {
298 188012714 : for (int j = 0; j < count; ++j)
299 : {
300 128098501 : const std::size_t indexPathLen = fieldPath.size();
301 128098501 : fieldPath.append("[").append(std::to_string(j)).append("]");
302 128098501 : AssertMessageStringsAreValidUtf8(reflection->GetRepeatedMessage(msg, field, j), fieldPath);
303 128098501 : fieldPath.resize(indexPathLen);
304 : }
305 : }
306 : }
307 : else
308 : {
309 : // Skip unset fields when presence tracking is available
310 1040411228 : if (field->has_presence() && !reflection->HasField(msg, field))
311 : {
312 119073219 : fieldPath.resize(basePathLen);
313 119073219 : continue;
314 : }
315 :
316 921338009 : if (field->type() == google::protobuf::FieldDescriptor::TYPE_STRING)
317 : {
318 213837226 : AssertStringIsValidUtf8(reflection->GetString(msg, field), fieldPath);
319 : }
320 707500783 : else if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE)
321 : {
322 2949001 : AssertMessageStringsAreValidUtf8(reflection->GetMessage(msg, field), fieldPath);
323 : }
324 : }
325 :
326 985006271 : fieldPath.resize(basePathLen);
327 : }
328 131047659 : }
329 : #endif // !NDEBUG
330 : } // anonymous namespace
331 :
332 274574185 : bool IsValidUtf8(const char* data, std::size_t size) noexcept
333 : {
334 274574185 : if (data == nullptr && size > 0) return false;
335 :
336 274574185 : const auto* bytes = reinterpret_cast<const uint8_t*>(data);
337 274574185 : std::size_t i = 0;
338 :
339 890916594 : while (i < size)
340 : {
341 616342490 : const std::size_t sequenceLength = Utf8SequenceLengthAt(bytes, i, size);
342 616342490 : if (sequenceLength == 0) return false;
343 616342409 : i += sequenceLength;
344 : }
345 :
346 274574104 : return true;
347 : }
348 :
349 274574185 : bool IsValidUtf8(std::string_view s) noexcept
350 : {
351 274574185 : return IsValidUtf8(s.data(), s.size());
352 : }
353 :
354 124377997 : std::string ToUtf8(std::string_view input)
355 : {
356 : // Fast path: already valid UTF-8
357 124377997 : if (IsValidUtf8(input))
358 : {
359 248755852 : return std::string(input);
360 : }
361 :
362 : // Repair path: keep every valid UTF-8 sequence as-is and transcode each
363 : // invalid byte individually as CP1252. Transcoding the whole string as
364 : // CP1252 would corrupt the already-valid parts of mixed input into
365 : // mojibake (e.g. a valid "Ö" would become "Ö").
366 : // Worst case: each byte becomes 3 UTF-8 bytes (CP1252 0x80-0x9F map to
367 : // BMP code points that encode as 3-byte UTF-8 sequences).
368 71 : std::string result;
369 71 : result.reserve(input.size() * 3);
370 :
371 71 : const auto* bytes = reinterpret_cast<const uint8_t*>(input.data());
372 : char utf8Buf[4];
373 71 : std::size_t i = 0;
374 :
375 644 : while (i < input.size())
376 : {
377 573 : const std::size_t sequenceLength = Utf8SequenceLengthAt(bytes, i, input.size());
378 573 : if (sequenceLength > 0)
379 : {
380 496 : result.append(input.data() + i, sequenceLength);
381 496 : i += sequenceLength;
382 : }
383 : else
384 : {
385 77 : const uint32_t codepoint = kCp1252ToUnicode[bytes[i]];
386 77 : const int len = EncodeUtf8Codepoint(codepoint, utf8Buf);
387 77 : result.append(utf8Buf, len);
388 77 : ++i;
389 : }
390 : }
391 :
392 71 : return result;
393 71 : }
394 :
395 2 : void SanitizeToUtf8(std::string& s)
396 : {
397 2 : if (!IsValidUtf8(s))
398 : {
399 1 : s = ToUtf8(s);
400 : }
401 2 : }
402 :
403 157 : void AssertAllStringFieldsAreValidUtf8(const google::protobuf::Message& msg, std::string_view msgName)
404 : {
405 : #ifndef NDEBUG
406 157 : std::string fieldPath(msgName);
407 157 : AssertMessageStringsAreValidUtf8(msg, fieldPath);
408 : #else
409 : (void)msg;
410 : (void)msgName;
411 : #endif
412 157 : }
413 :
414 : } // namespace Odb::Lib::Text
|