OdbDesign
OdbDesign ODB++ Parser - C++ library, gRPC/REST server, CLI, tests
 
Loading...
Searching...
No Matches
DesignStreamWalker.cpp
Go to the documentation of this file.
2#include <Logger.h>
4
5#include <map>
6
7namespace OdbDesignServer
8{
9 namespace Services
10 {
12
13 // Streams one Design ByName map as NormalizedMapBatch chunks with the
14 // same first-wins sanitized-key semantics as
15 // FillProtobufMapWithSanitizedKeys. The collision set is local to the
16 // map (refDes and net names legitimately overlap across the four
17 // collections; the monolithic build dedupes within each map field
18 // independently).
19 template <typename TEntries, typename TMapGetter>
20 bool StreamNormalizedMap(const TEntries& entries,
21 std::size_t mapBatchSize,
22 TMapGetter getMapField,
23 const char* mapName,
25 {
26 Odb::Grpc::DesignChunk chunk;
27 auto* batch = chunk.mutable_normalized_maps();
28 std::set<std::string> emittedKeys;
29 std::size_t count = 0;
30 for (const auto& kv : entries)
31 {
32 const auto key = Odb::Lib::Text::ToUtf8(kv.first);
33 if (emittedKeys.count(key) != 0)
34 {
35 logwarn(std::string("UTF-8 sanitization collision in streamed protobuf map (") + mapName +
36 "): key \"" + kv.first + "\" sanitizes to a key that is already present; keeping the first entry");
37 continue;
38 }
39 emittedKeys.insert(key);
40 auto* mapField = ((*batch).*getMapField)();
41 (*mapField)[key] = *kv.second->to_protobuf();
42 ++count;
43 if (count >= mapBatchSize)
44 {
45 if (!emit(chunk))
46 {
47 return false;
48 }
49 chunk.Clear();
50 batch = chunk.mutable_normalized_maps();
51 count = 0;
52 }
53 }
54 if (count > 0)
55 {
56 return emit(chunk);
57 }
58 return true;
59 }
60
61 bool DesignStreamWalker::Walk(const Design& design,
62 bool includeNormalizedLists,
63 const DesignStreamOptions& options,
64 const ChunkSink& sink,
65 Stats* stats)
66 {
67 auto emit = [&sink, stats](const Odb::Grpc::DesignChunk& chunk)
68 {
69 if (stats != nullptr)
70 {
71 ++stats->chunks;
72 stats->wireBytes += chunk.ByteSizeLong();
73 }
74 return sink(chunk);
75 };
76
77 // ---- 1. DesignHeader ----
78 {
79 Odb::Grpc::DesignChunk chunk;
80 auto* header = chunk.mutable_header();
81 header->set_product_model(Odb::Lib::Text::ToUtf8(design.GetProductModel()));
82 header->set_name(Odb::Lib::Text::ToUtf8(design.GetName()));
83 const auto fileModel = design.GetFileModel();
84 if (fileModel != nullptr)
85 {
86 auto fileArchiveMsg = fileModel->to_protobuf_skeleton();
87 header->mutable_file_archive()->Swap(fileArchiveMsg.get());
88 }
89 if (!emit(chunk))
90 {
91 return false;
92 }
93 }
94
95 const auto fileModel = design.GetFileModel();
96 if (fileModel != nullptr)
97 {
98 // ---- 2. StepSkeletons (stepsByName key order — the same
99 // order FileArchive::to_protobuf inserts the map entries in) ----
100 for (const auto& kvStep : fileModel->GetStepsByName())
101 {
102 Odb::Grpc::DesignChunk chunk;
103 auto* stepChunk = chunk.mutable_step();
104 stepChunk->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
105 auto stepMsg = kvStep.second->to_protobuf_skeleton();
106 stepChunk->mutable_step()->Swap(stepMsg.get());
107 if (!emit(chunk))
108 {
109 return false;
110 }
111 }
112
113 // ---- 3. FeatureBatches (step order, layer order, record
114 // order — FeaturesFile::to_protobuf's exact traversal).
115 // Sanitized-key collisions: the monolith inserts with map
116 // assignment, so the LAST raw step/layer holding a
117 // sanitized key survives (FileArchive::to_protobuf /
118 // StepDirectory::FillProtobufMessage). StepSkeleton
119 // reassembly reproduces that by assignment, but
120 // FeatureBatches must be emitted for surviving raw entries
121 // ONLY — a shadowed entry's records would otherwise append
122 // into the survivor (duplicating features the monolith
123 // dropped) or address a layer the survivor lacks (failing
124 // reassembly). Mirrors the guard the design-level ByName
125 // maps carry (ProtobufMapHelpers.h). ----
126 std::map<std::string, Odb::Lib::FileModel::Design::StepDirectory*> survivingSteps;
127 for (const auto& kvStep : fileModel->GetStepsByName())
128 {
129 auto stepKey = Odb::Lib::Text::ToUtf8(kvStep.first);
130 if (survivingSteps.count(stepKey) != 0)
131 {
132 logwarn("UTF-8 sanitization collision in streamed stepsByName: raw step \"" +
133 kvStep.first + "\" sanitizes to a key already held; keeping the last entry (monolith last-wins)");
134 }
135 survivingSteps[stepKey] = kvStep.second.get();
136 }
137
138 for (const auto& kvStep : fileModel->GetStepsByName())
139 {
140 if (survivingSteps[Odb::Lib::Text::ToUtf8(kvStep.first)] != kvStep.second.get())
141 {
142 continue; // shadowed raw step: the monolith's overwrite drops it entirely
143 }
144
145 std::map<std::string, Odb::Lib::FileModel::Design::LayerDirectory*> survivingLayers;
146 for (const auto& kvLayer : kvStep.second->GetLayersByName())
147 {
148 auto layerKey = Odb::Lib::Text::ToUtf8(kvLayer.first);
149 if (survivingLayers.count(layerKey) != 0)
150 {
151 logwarn("UTF-8 sanitization collision in streamed layersByName (step \"" +
152 kvStep.first + "\"): raw layer \"" + kvLayer.first +
153 "\" sanitizes to a key already held; keeping the last entry (monolith last-wins)");
154 }
155 survivingLayers[layerKey] = kvLayer.second.get();
156 }
157
158 for (const auto& kvLayer : kvStep.second->GetLayersByName())
159 {
160 if (survivingLayers[Odb::Lib::Text::ToUtf8(kvLayer.first)] != kvLayer.second.get())
161 {
162 continue; // shadowed raw layer: the monolith's overwrite drops it
163 }
164
165 const auto& records = kvLayer.second->GetFeaturesFile().GetFeatureRecords();
166 if (records.empty())
167 {
168 continue;
169 }
170
171 Odb::Grpc::DesignChunk chunk;
172 auto* batch = chunk.mutable_features();
173 batch->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
174 batch->set_layer_name(Odb::Lib::Text::ToUtf8(kvLayer.first));
175 for (const auto& pRecord : records)
176 {
177 // Identical to FeaturesFile::to_protobuf: no null
178 // filtering, same per-record serialization.
179 batch->add_records()->CopyFrom(*pRecord->to_protobuf());
180 if (static_cast<std::size_t>(batch->records_size()) >= options.featureBatchSize)
181 {
182 if (!emit(chunk))
183 {
184 return false;
185 }
186 chunk.Clear();
187 batch = chunk.mutable_features();
188 batch->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
189 batch->set_layer_name(Odb::Lib::Text::ToUtf8(kvLayer.first));
190 }
191 }
192 if (batch->records_size() > 0)
193 {
194 if (!emit(chunk))
195 {
196 return false;
197 }
198 }
199 }
200 }
201 }
202
203 if (includeNormalizedLists)
204 {
205 // ---- 4. Design-level repeated batches (same order and
206 // per-element serialization as Design::to_protobuf) ----
207 {
208 Odb::Grpc::DesignChunk chunk;
209 auto* batch = chunk.mutable_nets();
210 for (const auto& pNet : design.GetNets())
211 {
212 batch->add_nets()->CopyFrom(*pNet->to_protobuf());
213 if (static_cast<std::size_t>(batch->nets_size()) >= options.entryBatchSize)
214 {
215 if (!emit(chunk)) return false;
216 chunk.Clear();
217 batch = chunk.mutable_nets();
218 }
219 }
220 if (batch->nets_size() > 0)
221 {
222 if (!emit(chunk)) return false;
223 }
224 }
225 {
226 Odb::Grpc::DesignChunk chunk;
227 auto* batch = chunk.mutable_packages();
228 for (const auto& pPackage : design.GetPackages())
229 {
230 batch->add_packages()->CopyFrom(*pPackage->to_protobuf());
231 if (static_cast<std::size_t>(batch->packages_size()) >= options.entryBatchSize)
232 {
233 if (!emit(chunk)) return false;
234 chunk.Clear();
235 batch = chunk.mutable_packages();
236 }
237 }
238 if (batch->packages_size() > 0)
239 {
240 if (!emit(chunk)) return false;
241 }
242 }
243 {
244 Odb::Grpc::DesignChunk chunk;
245 auto* batch = chunk.mutable_components();
246 for (const auto& pComponent : design.GetComponents())
247 {
248 batch->add_components()->CopyFrom(*pComponent->to_protobuf());
249 if (static_cast<std::size_t>(batch->components_size()) >= options.entryBatchSize)
250 {
251 if (!emit(chunk)) return false;
252 chunk.Clear();
253 batch = chunk.mutable_components();
254 }
255 }
256 if (batch->components_size() > 0)
257 {
258 if (!emit(chunk)) return false;
259 }
260 }
261 {
262 Odb::Grpc::DesignChunk chunk;
263 auto* batch = chunk.mutable_parts();
264 for (const auto& pPart : design.GetParts())
265 {
266 batch->add_parts()->CopyFrom(*pPart->to_protobuf());
267 if (static_cast<std::size_t>(batch->parts_size()) >= options.entryBatchSize)
268 {
269 if (!emit(chunk)) return false;
270 chunk.Clear();
271 batch = chunk.mutable_parts();
272 }
273 }
274 if (batch->parts_size() > 0)
275 {
276 if (!emit(chunk)) return false;
277 }
278 }
279 }
280
281 // ---- 5. Connectivity batches (sections in FillConnectivityMessage
282 // order: nets, packages, netMembers, components; entries emitted by
283 // the same Design::AddConnectivity* single-entry emitters) ----
284 {
285 const auto index = design.BuildConnectivityIndex();
286
287 // FillConnectivityMessage runs unconditionally (D4), so the
288 // monolith always serializes field 12 — a design with no
289 // nets/packages/components as present-but-empty. Track whether
290 // any ConnectivityBatch was emitted so a no-net design still
291 // emits one (empty) batch and reassembly creates the
292 // submessage; otherwise the reassembled message would lack
293 // field 12 entirely and diverge from the monolithic bytes.
294 bool connectivityEmitted = false;
295 auto emitConnectivity = [&emit, &connectivityEmitted](const Odb::Grpc::DesignChunk& chunk)
296 {
297 connectivityEmitted = true;
298 return emit(chunk);
299 };
300
301 // nets
302 {
303 Odb::Grpc::DesignChunk chunk;
304 auto* entries = chunk.mutable_connectivity()->mutable_entries();
305 for (std::size_t i = 0; i < design.GetNets().size(); ++i)
306 {
307 design.AddConnectivityNetEntry(i, *entries);
308 if (static_cast<std::size_t>(entries->nets_size()) >= options.entryBatchSize)
309 {
310 if (!emitConnectivity(chunk)) return false;
311 chunk.Clear();
312 entries = chunk.mutable_connectivity()->mutable_entries();
313 }
314 }
315 if (entries->nets_size() > 0)
316 {
317 if (!emitConnectivity(chunk)) return false;
318 }
319 }
320 // packages
321 {
322 Odb::Grpc::DesignChunk chunk;
323 auto* entries = chunk.mutable_connectivity()->mutable_entries();
324 for (std::size_t i = 0; i < design.GetPackages().size(); ++i)
325 {
326 design.AddConnectivityPackagePinEntry(i, *entries);
327 if (static_cast<std::size_t>(entries->packages_size()) >= options.entryBatchSize)
328 {
329 if (!emitConnectivity(chunk)) return false;
330 chunk.Clear();
331 entries = chunk.mutable_connectivity()->mutable_entries();
332 }
333 }
334 if (entries->packages_size() > 0)
335 {
336 if (!emitConnectivity(chunk)) return false;
337 }
338 }
339 // netMembers
340 {
341 Odb::Grpc::DesignChunk chunk;
342 auto* entries = chunk.mutable_connectivity()->mutable_entries();
343 for (auto membersIt = index.membersByNet.begin(); membersIt != index.membersByNet.end(); ++membersIt)
344 {
345 design.AddConnectivityNetMemberEntry(index, membersIt, *entries);
346 if (static_cast<std::size_t>(entries->netmembers_size()) >= options.entryBatchSize)
347 {
348 if (!emitConnectivity(chunk)) return false;
349 chunk.Clear();
350 entries = chunk.mutable_connectivity()->mutable_entries();
351 }
352 }
353 if (entries->netmembers_size() > 0)
354 {
355 if (!emitConnectivity(chunk)) return false;
356 }
357 }
358 // components
359 {
360 Odb::Grpc::DesignChunk chunk;
361 auto* entries = chunk.mutable_connectivity()->mutable_entries();
362 for (std::size_t i = 0; i < design.GetComponents().size(); ++i)
363 {
364 design.AddConnectivityPlacedComponentEntry(index, i, *entries);
365 if (static_cast<std::size_t>(entries->components_size()) >= options.entryBatchSize)
366 {
367 if (!emitConnectivity(chunk)) return false;
368 chunk.Clear();
369 entries = chunk.mutable_connectivity()->mutable_entries();
370 }
371 }
372 if (entries->components_size() > 0)
373 {
374 if (!emitConnectivity(chunk)) return false;
375 }
376 }
377
378 if (!connectivityEmitted)
379 {
380 // Present-but-empty parity: ApplyDesignChunk's
381 // mutable_connectivity() creates the submessage even for an
382 // empty entries payload, matching the monolith's tag+len(0).
383 Odb::Grpc::DesignChunk chunk;
384 chunk.mutable_connectivity();
385 if (!emit(chunk))
386 {
387 return false;
388 }
389 }
390 }
391
392 if (includeNormalizedLists)
393 {
394 // ---- 6. NormalizedMapBatch chunks. Same first-wins sanitized
395 // key semantics as FillProtobufMapWithSanitizedKeys: entries
396 // are emitted in std::map (sorted-key) order — the order the
397 // monolithic build inserts them — and a raw key whose sanitized
398 // form collides with an already-emitted key is skipped. The
399 // collision set is PER MAP (refDes and net names legitimately
400 // overlap across the four collections; the monolithic build
401 // dedupes within each map field independently). ----
402 if (!StreamNormalizedMap(design.GetNetsByName(), options.mapBatchSize,
403 &Odb::Grpc::NormalizedMapBatch::mutable_nets_by_name, "netsByName", emit))
404 {
405 return false;
406 }
408 &Odb::Grpc::NormalizedMapBatch::mutable_packages_by_name, "packagesByName", emit))
409 {
410 return false;
411 }
413 &Odb::Grpc::NormalizedMapBatch::mutable_components_by_name, "componentsByName", emit))
414 {
415 return false;
416 }
417 if (!StreamNormalizedMap(design.GetPartsByName(), options.mapBatchSize,
418 &Odb::Grpc::NormalizedMapBatch::mutable_parts_by_name, "partsByName", emit))
419 {
420 return false;
421 }
422 }
423
424 return true;
425 }
426
427 bool ApplyDesignChunk(Odb::Lib::Protobuf::ProductModel::Design& design,
428 const Odb::Grpc::DesignChunk& chunk)
429 {
430 switch (chunk.payload_case())
431 {
432 case Odb::Grpc::DesignChunk::kHeader:
433 design.set_productmodel(chunk.header().product_model());
434 design.set_name(chunk.header().name());
435 // Presence parity with Design::to_protobuf: the monolith sets
436 // filemodel only when the design has a file model, so the
437 // header chunk carries it only then (Walk emits none for a
438 // null/clipped file model) — never create an empty one.
439 if (chunk.header().has_file_archive())
440 {
441 design.mutable_filemodel()->CopyFrom(chunk.header().file_archive());
442 }
443 return true;
444
445 case Odb::Grpc::DesignChunk::kStep:
446 {
447 if (!design.has_filemodel())
448 {
449 return false;
450 }
451 const auto& step = chunk.step();
452 (*design.mutable_filemodel()->mutable_stepsbyname())[step.step_name()].CopyFrom(step.step());
453 return true;
454 }
455
456 case Odb::Grpc::DesignChunk::kFeatures:
457 {
458 if (!design.has_filemodel())
459 {
460 return false;
461 }
462 const auto& batch = chunk.features();
463 auto* stepsByName = design.mutable_filemodel()->mutable_stepsbyname();
464 // Linear scans, not find(): #599 diagnosed an address-layout
465 // dependent protobuf-Map lookup bug (entry present under
466 // iteration, absent under find() on the same object), never
467 // root-caused. These maps hold a handful of entries, so
468 // iteration is free — and the reference reassembly models
469 // the recipe M6.4 clients will copy.
470 auto stepIt = stepsByName->end();
471 for (auto it = stepsByName->begin(); it != stepsByName->end(); ++it)
472 {
473 if (it->first == batch.step_name())
474 {
475 stepIt = it;
476 break;
477 }
478 }
479 if (stepIt == stepsByName->end())
480 {
481 return false;
482 }
483 auto* layersByName = stepIt->second.mutable_layersbyname();
484 auto layerIt = layersByName->end();
485 for (auto it = layersByName->begin(); it != layersByName->end(); ++it)
486 {
487 if (it->first == batch.layer_name())
488 {
489 layerIt = it;
490 break;
491 }
492 }
493 if (layerIt == layersByName->end())
494 {
495 return false;
496 }
497 auto* featureFile = layerIt->second.mutable_featurefile();
498 for (const auto& record : batch.records())
499 {
500 featureFile->add_featurerecords()->CopyFrom(record);
501 }
502 return true;
503 }
504
505 case Odb::Grpc::DesignChunk::kNets:
506 for (const auto& net : chunk.nets().nets())
507 {
508 design.add_nets()->CopyFrom(net);
509 }
510 return true;
511
512 case Odb::Grpc::DesignChunk::kPackages:
513 for (const auto& packageEntry : chunk.packages().packages())
514 {
515 design.add_packages()->CopyFrom(packageEntry);
516 }
517 return true;
518
519 case Odb::Grpc::DesignChunk::kComponents:
520 for (const auto& component : chunk.components().components())
521 {
522 design.add_components()->CopyFrom(component);
523 }
524 return true;
525
526 case Odb::Grpc::DesignChunk::kParts:
527 for (const auto& part : chunk.parts().parts())
528 {
529 design.add_parts()->CopyFrom(part);
530 }
531 return true;
532
533 case Odb::Grpc::DesignChunk::kConnectivity:
534 design.mutable_connectivity()->MergeFrom(chunk.connectivity().entries());
535 return true;
536
537 case Odb::Grpc::DesignChunk::kNormalizedMaps:
538 {
539 const auto& maps = chunk.normalized_maps();
540 for (const auto& kv : maps.nets_by_name())
541 {
542 (*design.mutable_netsbyname())[kv.first] = kv.second;
543 }
544 for (const auto& kv : maps.packages_by_name())
545 {
546 (*design.mutable_packagesbyname())[kv.first] = kv.second;
547 }
548 for (const auto& kv : maps.components_by_name())
549 {
550 (*design.mutable_componentsbyname())[kv.first] = kv.second;
551 }
552 for (const auto& kv : maps.parts_by_name())
553 {
554 (*design.mutable_partsbyname())[kv.first] = kv.second;
555 }
556 return true;
557 }
558
559 case Odb::Grpc::DesignChunk::PAYLOAD_NOT_SET:
560 default:
561 return false;
562 }
563 }
564 }
565}
#define logwarn(s)
Definition Logger.h:20
UTF-8 validation and Windows-1252 to UTF-8 transcoding utilities.
static bool Walk(const Odb::Lib::ProductModel::Design &design, bool includeNormalizedLists, const DesignStreamOptions &options, const ChunkSink &sink, Stats *stats=nullptr)
std::function< bool(const Odb::Grpc::DesignChunk &)> ChunkSink
const Component::Vector & GetComponents() const
Definition Design.cpp:64
const Part::StringMap & GetPartsByName() const
Definition Design.cpp:90
const Net::Vector & GetNets() const
Definition Design.cpp:44
const Package::StringMap & GetPackagesByName() const
Definition Design.cpp:59
void AddConnectivityNetEntry(std::size_t netIndex, Protobuf::ProductModel::Connectivity &connectivityMsg) const
Definition Design.cpp:317
const Component::StringMap & GetComponentsByName() const
Definition Design.cpp:69
ConnectivityIndex BuildConnectivityIndex() const
Definition Design.cpp:236
void AddConnectivityPackagePinEntry(std::size_t packageIndex, Protobuf::ProductModel::Connectivity &connectivityMsg) const
Definition Design.cpp:329
const Package::Vector & GetPackages() const
Definition Design.cpp:54
void AddConnectivityNetMemberEntry(const ConnectivityIndex &index, ConnectivityIndex::MembersByNetConstIt netMembersIt, Protobuf::ProductModel::Connectivity &connectivityMsg) const
Definition Design.cpp:345
const Part::Vector & GetParts() const
Definition Design.cpp:85
std::shared_ptr< FileModel::Design::FileArchive > GetFileModel() const
Definition Design.cpp:150
const Net::StringMap GetNetsByName() const
Definition Design.cpp:49
const std::string & GetName() const
Definition Design.cpp:34
const std::string & GetProductModel() const
Definition Design.cpp:39
void AddConnectivityPlacedComponentEntry(const ConnectivityIndex &index, std::size_t componentIndex, Protobuf::ProductModel::Connectivity &connectivityMsg) const
Definition Design.cpp:360
bool ApplyDesignChunk(Odb::Lib::Protobuf::ProductModel::Design &design, const Odb::Grpc::DesignChunk &chunk)
bool StreamNormalizedMap(const TEntries &entries, std::size_t mapBatchSize, TMapGetter getMapField, const char *mapName, const DesignStreamWalker::ChunkSink &emit)
std::string ToUtf8(std::string_view input)
Converts input to valid UTF-8.