Line data Source code
1 : #pragma once
2 :
3 : #include <string>
4 : #include "../Text/Utf8Sanitizer.h"
5 : #include "Logger.h"
6 :
7 : namespace Odb::Lib::ProductModel
8 : {
9 : // Copies entries into a protobuf map<string, TMessage> field, sanitizing each
10 : // key with Odb::Lib::Text::ToUtf8. Distinct raw keys can sanitize to the same
11 : // UTF-8 key — most commonly through the five undefined CP1252 slots (0x81,
12 : // 0x8D, 0x8F, 0x90, 0x9D), which all map to U+FFFD, but also when a repaired
13 : // key ("A\xC3" -> "AÃ") matches an already-valid UTF-8 key, or a key already
14 : // containing U+FFFD matches one whose undefined byte mapped to U+FFFD. The
15 : // find() check handles every collision uniformly: the first entry wins and
16 : // the collision is logged, so no entry is ever silently overwritten.
17 : template <typename TPbMap, typename TEntries>
18 79848 : void FillProtobufMapWithSanitizedKeys(TPbMap& pbMap, const TEntries& entries)
19 : {
20 5631938 : for (const auto& kv : entries)
21 : {
22 2776045 : auto sanitizedKey = Odb::Lib::Text::ToUtf8(kv.first);
23 2776045 : if (pbMap.find(sanitizedKey) != pbMap.end())
24 : {
25 0 : logwarn("UTF-8 sanitization collision in protobuf map: key \"" + kv.first +
26 : "\" sanitizes to a key that is already present; keeping the first entry");
27 0 : continue;
28 : }
29 2776045 : pbMap[sanitizedKey] = *kv.second->to_protobuf();
30 : }
31 79848 : }
32 : }
|