Line data Source code
1 : #include "DesignStreamWalker.h"
2 : #include <Logger.h>
3 : #include <Text/Utf8Sanitizer.h>
4 :
5 : #include <map>
6 :
7 : namespace OdbDesignServer
8 : {
9 : namespace Services
10 : {
11 : using Odb::Lib::ProductModel::Design;
12 :
13 : // Streams one Design ByName map as NormalizedMapBatch chunks with the
14 : // same first-wins sanitized-key semantics as
15 : // FillProtobufMapWithSanitizedKeys. The collision set is local to the
16 : // map (refDes and net names legitimately overlap across the four
17 : // collections; the monolithic build dedupes within each map field
18 : // independently).
19 : template <typename TEntries, typename TMapGetter>
20 8 : bool StreamNormalizedMap(const TEntries& entries,
21 : std::size_t mapBatchSize,
22 : TMapGetter getMapField,
23 : const char* mapName,
24 : const DesignStreamWalker::ChunkSink& emit)
25 : {
26 8 : Odb::Grpc::DesignChunk chunk;
27 8 : auto* batch = chunk.mutable_normalized_maps();
28 8 : std::set<std::string> emittedKeys;
29 8 : std::size_t count = 0;
30 6692 : for (const auto& kv : entries)
31 : {
32 3342 : const auto key = Odb::Lib::Text::ToUtf8(kv.first);
33 3342 : if (emittedKeys.count(key) != 0)
34 : {
35 0 : logwarn(std::string("UTF-8 sanitization collision in streamed protobuf map (") + mapName +
36 : "): key \"" + kv.first + "\" sanitizes to a key that is already present; keeping the first entry");
37 0 : continue;
38 : }
39 3342 : emittedKeys.insert(key);
40 3342 : auto* mapField = ((*batch).*getMapField)();
41 3342 : (*mapField)[key] = *kv.second->to_protobuf();
42 3342 : ++count;
43 3342 : if (count >= mapBatchSize)
44 : {
45 4 : if (!emit(chunk))
46 : {
47 0 : return false;
48 : }
49 4 : chunk.Clear();
50 4 : batch = chunk.mutable_normalized_maps();
51 4 : count = 0;
52 : }
53 : }
54 8 : if (count > 0)
55 : {
56 8 : return emit(chunk);
57 : }
58 0 : return true;
59 8 : }
60 :
61 17 : bool DesignStreamWalker::Walk(const Design& design,
62 : bool includeNormalizedLists,
63 : const DesignStreamOptions& options,
64 : const ChunkSink& sink,
65 : Stats* stats)
66 : {
67 9920 : auto emit = [&sink, stats](const Odb::Grpc::DesignChunk& chunk)
68 : {
69 9920 : if (stats != nullptr)
70 : {
71 9920 : ++stats->chunks;
72 9920 : stats->wireBytes += chunk.ByteSizeLong();
73 : }
74 9920 : return sink(chunk);
75 17 : };
76 :
77 : // ---- 1. DesignHeader ----
78 : {
79 17 : Odb::Grpc::DesignChunk chunk;
80 17 : auto* header = chunk.mutable_header();
81 34 : header->set_product_model(Odb::Lib::Text::ToUtf8(design.GetProductModel()));
82 34 : header->set_name(Odb::Lib::Text::ToUtf8(design.GetName()));
83 17 : const auto fileModel = design.GetFileModel();
84 17 : if (fileModel != nullptr)
85 : {
86 16 : auto fileArchiveMsg = fileModel->to_protobuf_skeleton();
87 16 : header->mutable_file_archive()->Swap(fileArchiveMsg.get());
88 16 : }
89 17 : if (!emit(chunk))
90 : {
91 0 : return false;
92 : }
93 17 : }
94 :
95 17 : const auto fileModel = design.GetFileModel();
96 17 : if (fileModel != nullptr)
97 : {
98 : // ---- 2. StepSkeletons (stepsByName key order — the same
99 : // order FileArchive::to_protobuf inserts the map entries in) ----
100 33 : for (const auto& kvStep : fileModel->GetStepsByName())
101 : {
102 17 : Odb::Grpc::DesignChunk chunk;
103 17 : auto* stepChunk = chunk.mutable_step();
104 34 : stepChunk->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
105 17 : auto stepMsg = kvStep.second->to_protobuf_skeleton();
106 17 : stepChunk->mutable_step()->Swap(stepMsg.get());
107 17 : if (!emit(chunk))
108 : {
109 0 : return false;
110 : }
111 17 : }
112 :
113 : // ---- 3. FeatureBatches (step order, layer order, record
114 : // order — FeaturesFile::to_protobuf's exact traversal).
115 : // Sanitized-key collisions: the monolith inserts with map
116 : // assignment, so the LAST raw step/layer holding a
117 : // sanitized key survives (FileArchive::to_protobuf /
118 : // StepDirectory::FillProtobufMessage). StepSkeleton
119 : // reassembly reproduces that by assignment, but
120 : // FeatureBatches must be emitted for surviving raw entries
121 : // ONLY — a shadowed entry's records would otherwise append
122 : // into the survivor (duplicating features the monolith
123 : // dropped) or address a layer the survivor lacks (failing
124 : // reassembly). Mirrors the guard the design-level ByName
125 : // maps carry (ProtobufMapHelpers.h). ----
126 16 : std::map<std::string, Odb::Lib::FileModel::Design::StepDirectory*> survivingSteps;
127 33 : for (const auto& kvStep : fileModel->GetStepsByName())
128 : {
129 17 : auto stepKey = Odb::Lib::Text::ToUtf8(kvStep.first);
130 17 : if (survivingSteps.count(stepKey) != 0)
131 : {
132 3 : logwarn("UTF-8 sanitization collision in streamed stepsByName: raw step \"" +
133 : kvStep.first + "\" sanitizes to a key already held; keeping the last entry (monolith last-wins)");
134 : }
135 17 : survivingSteps[stepKey] = kvStep.second.get();
136 17 : }
137 :
138 32 : for (const auto& kvStep : fileModel->GetStepsByName())
139 : {
140 17 : if (survivingSteps[Odb::Lib::Text::ToUtf8(kvStep.first)] != kvStep.second.get())
141 : {
142 1 : continue; // shadowed raw step: the monolith's overwrite drops it entirely
143 : }
144 :
145 16 : std::map<std::string, Odb::Lib::FileModel::Design::LayerDirectory*> survivingLayers;
146 383 : for (const auto& kvLayer : kvStep.second->GetLayersByName())
147 : {
148 367 : auto layerKey = Odb::Lib::Text::ToUtf8(kvLayer.first);
149 367 : if (survivingLayers.count(layerKey) != 0)
150 : {
151 3 : logwarn("UTF-8 sanitization collision in streamed layersByName (step \"" +
152 : kvStep.first + "\"): raw layer \"" + kvLayer.first +
153 : "\" sanitizes to a key already held; keeping the last entry (monolith last-wins)");
154 : }
155 367 : survivingLayers[layerKey] = kvLayer.second.get();
156 367 : }
157 :
158 367 : for (const auto& kvLayer : kvStep.second->GetLayersByName())
159 : {
160 352 : if (survivingLayers[Odb::Lib::Text::ToUtf8(kvLayer.first)] != kvLayer.second.get())
161 : {
162 127 : continue; // shadowed raw layer: the monolith's overwrite drops it
163 : }
164 :
165 351 : const auto& records = kvLayer.second->GetFeaturesFile().GetFeatureRecords();
166 351 : if (records.empty())
167 : {
168 126 : continue;
169 : }
170 :
171 225 : Odb::Grpc::DesignChunk chunk;
172 225 : auto* batch = chunk.mutable_features();
173 450 : batch->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
174 450 : batch->set_layer_name(Odb::Lib::Text::ToUtf8(kvLayer.first));
175 1127952 : for (const auto& pRecord : records)
176 : {
177 : // Identical to FeaturesFile::to_protobuf: no null
178 : // filtering, same per-record serialization.
179 1127728 : batch->add_records()->CopyFrom(*pRecord->to_protobuf());
180 1127728 : if (static_cast<std::size_t>(batch->records_size()) >= options.featureBatchSize)
181 : {
182 9600 : if (!emit(chunk))
183 : {
184 1 : return false;
185 : }
186 9599 : chunk.Clear();
187 9599 : batch = chunk.mutable_features();
188 19198 : batch->set_step_name(Odb::Lib::Text::ToUtf8(kvStep.first));
189 19198 : batch->set_layer_name(Odb::Lib::Text::ToUtf8(kvLayer.first));
190 : }
191 : }
192 224 : if (batch->records_size() > 0)
193 : {
194 211 : if (!emit(chunk))
195 : {
196 0 : return false;
197 : }
198 : }
199 225 : }
200 16 : }
201 16 : }
202 :
203 16 : if (includeNormalizedLists)
204 : {
205 : // ---- 4. Design-level repeated batches (same order and
206 : // per-element serialization as Design::to_protobuf) ----
207 : {
208 2 : Odb::Grpc::DesignChunk chunk;
209 2 : auto* batch = chunk.mutable_nets();
210 1290 : for (const auto& pNet : design.GetNets())
211 : {
212 1288 : batch->add_nets()->CopyFrom(*pNet->to_protobuf());
213 1288 : if (static_cast<std::size_t>(batch->nets_size()) >= options.entryBatchSize)
214 : {
215 0 : if (!emit(chunk)) return false;
216 0 : chunk.Clear();
217 0 : batch = chunk.mutable_nets();
218 : }
219 : }
220 2 : if (batch->nets_size() > 0)
221 : {
222 2 : if (!emit(chunk)) return false;
223 : }
224 2 : }
225 : {
226 2 : Odb::Grpc::DesignChunk chunk;
227 2 : auto* batch = chunk.mutable_packages();
228 144 : for (const auto& pPackage : design.GetPackages())
229 : {
230 142 : batch->add_packages()->CopyFrom(*pPackage->to_protobuf());
231 142 : if (static_cast<std::size_t>(batch->packages_size()) >= options.entryBatchSize)
232 : {
233 0 : if (!emit(chunk)) return false;
234 0 : chunk.Clear();
235 0 : batch = chunk.mutable_packages();
236 : }
237 : }
238 2 : if (batch->packages_size() > 0)
239 : {
240 2 : if (!emit(chunk)) return false;
241 : }
242 2 : }
243 : {
244 2 : Odb::Grpc::DesignChunk chunk;
245 2 : auto* batch = chunk.mutable_components();
246 1628 : for (const auto& pComponent : design.GetComponents())
247 : {
248 1626 : batch->add_components()->CopyFrom(*pComponent->to_protobuf());
249 1626 : if (static_cast<std::size_t>(batch->components_size()) >= options.entryBatchSize)
250 : {
251 0 : if (!emit(chunk)) return false;
252 0 : chunk.Clear();
253 0 : batch = chunk.mutable_components();
254 : }
255 : }
256 2 : if (batch->components_size() > 0)
257 : {
258 2 : if (!emit(chunk)) return false;
259 : }
260 2 : }
261 : {
262 2 : Odb::Grpc::DesignChunk chunk;
263 2 : auto* batch = chunk.mutable_parts();
264 288 : for (const auto& pPart : design.GetParts())
265 : {
266 286 : batch->add_parts()->CopyFrom(*pPart->to_protobuf());
267 286 : if (static_cast<std::size_t>(batch->parts_size()) >= options.entryBatchSize)
268 : {
269 0 : if (!emit(chunk)) return false;
270 0 : chunk.Clear();
271 0 : batch = chunk.mutable_parts();
272 : }
273 : }
274 2 : if (batch->parts_size() > 0)
275 : {
276 2 : if (!emit(chunk)) return false;
277 : }
278 2 : }
279 : }
280 :
281 : // ---- 5. Connectivity batches (sections in FillConnectivityMessage
282 : // order: nets, packages, netMembers, components; entries emitted by
283 : // the same Design::AddConnectivity* single-entry emitters) ----
284 : {
285 16 : const auto index = design.BuildConnectivityIndex();
286 :
287 : // FillConnectivityMessage runs unconditionally (D4), so the
288 : // monolith always serializes field 12 — a design with no
289 : // nets/packages/components as present-but-empty. Track whether
290 : // any ConnectivityBatch was emitted so a no-net design still
291 : // emits one (empty) batch and reassembly creates the
292 : // submessage; otherwise the reassembled message would lack
293 : // field 12 entirely and diverge from the monolithic bytes.
294 16 : bool connectivityEmitted = false;
295 52 : auto emitConnectivity = [&emit, &connectivityEmitted](const Odb::Grpc::DesignChunk& chunk)
296 : {
297 52 : connectivityEmitted = true;
298 52 : return emit(chunk);
299 16 : };
300 :
301 : // nets
302 : {
303 16 : Odb::Grpc::DesignChunk chunk;
304 16 : auto* entries = chunk.mutable_connectivity()->mutable_entries();
305 8388 : for (std::size_t i = 0; i < design.GetNets().size(); ++i)
306 : {
307 8372 : design.AddConnectivityNetEntry(i, *entries);
308 8372 : if (static_cast<std::size_t>(entries->nets_size()) >= options.entryBatchSize)
309 : {
310 0 : if (!emitConnectivity(chunk)) return false;
311 0 : chunk.Clear();
312 0 : entries = chunk.mutable_connectivity()->mutable_entries();
313 : }
314 : }
315 16 : if (entries->nets_size() > 0)
316 : {
317 13 : if (!emitConnectivity(chunk)) return false;
318 : }
319 16 : }
320 : // packages
321 : {
322 16 : Odb::Grpc::DesignChunk chunk;
323 16 : auto* entries = chunk.mutable_connectivity()->mutable_entries();
324 939 : for (std::size_t i = 0; i < design.GetPackages().size(); ++i)
325 : {
326 923 : design.AddConnectivityPackagePinEntry(i, *entries);
327 923 : if (static_cast<std::size_t>(entries->packages_size()) >= options.entryBatchSize)
328 : {
329 0 : if (!emitConnectivity(chunk)) return false;
330 0 : chunk.Clear();
331 0 : entries = chunk.mutable_connectivity()->mutable_entries();
332 : }
333 : }
334 16 : if (entries->packages_size() > 0)
335 : {
336 13 : if (!emitConnectivity(chunk)) return false;
337 : }
338 16 : }
339 : // netMembers
340 : {
341 16 : Odb::Grpc::DesignChunk chunk;
342 16 : auto* entries = chunk.mutable_connectivity()->mutable_entries();
343 8375 : for (auto membersIt = index.membersByNet.begin(); membersIt != index.membersByNet.end(); ++membersIt)
344 : {
345 8359 : design.AddConnectivityNetMemberEntry(index, membersIt, *entries);
346 8359 : if (static_cast<std::size_t>(entries->netmembers_size()) >= options.entryBatchSize)
347 : {
348 0 : if (!emitConnectivity(chunk)) return false;
349 0 : chunk.Clear();
350 0 : entries = chunk.mutable_connectivity()->mutable_entries();
351 : }
352 : }
353 16 : if (entries->netmembers_size() > 0)
354 : {
355 13 : if (!emitConnectivity(chunk)) return false;
356 : }
357 16 : }
358 : // components
359 : {
360 16 : Odb::Grpc::DesignChunk chunk;
361 16 : auto* entries = chunk.mutable_connectivity()->mutable_entries();
362 10585 : for (std::size_t i = 0; i < design.GetComponents().size(); ++i)
363 : {
364 10569 : design.AddConnectivityPlacedComponentEntry(index, i, *entries);
365 10569 : if (static_cast<std::size_t>(entries->components_size()) >= options.entryBatchSize)
366 : {
367 0 : if (!emitConnectivity(chunk)) return false;
368 0 : chunk.Clear();
369 0 : entries = chunk.mutable_connectivity()->mutable_entries();
370 : }
371 : }
372 16 : if (entries->components_size() > 0)
373 : {
374 13 : if (!emitConnectivity(chunk)) return false;
375 : }
376 16 : }
377 :
378 16 : if (!connectivityEmitted)
379 : {
380 : // Present-but-empty parity: ApplyDesignChunk's
381 : // mutable_connectivity() creates the submessage even for an
382 : // empty entries payload, matching the monolith's tag+len(0).
383 3 : Odb::Grpc::DesignChunk chunk;
384 3 : chunk.mutable_connectivity();
385 3 : if (!emit(chunk))
386 : {
387 0 : return false;
388 : }
389 3 : }
390 16 : }
391 :
392 16 : if (includeNormalizedLists)
393 : {
394 : // ---- 6. NormalizedMapBatch chunks. Same first-wins sanitized
395 : // key semantics as FillProtobufMapWithSanitizedKeys: entries
396 : // are emitted in std::map (sorted-key) order — the order the
397 : // monolithic build inserts them — and a raw key whose sanitized
398 : // form collides with an already-emitted key is skipped. The
399 : // collision set is PER MAP (refDes and net names legitimately
400 : // overlap across the four collections; the monolithic build
401 : // dedupes within each map field independently). ----
402 2 : if (!StreamNormalizedMap(design.GetNetsByName(), options.mapBatchSize,
403 : &Odb::Grpc::NormalizedMapBatch::mutable_nets_by_name, "netsByName", emit))
404 : {
405 0 : return false;
406 : }
407 2 : if (!StreamNormalizedMap(design.GetPackagesByName(), options.mapBatchSize,
408 : &Odb::Grpc::NormalizedMapBatch::mutable_packages_by_name, "packagesByName", emit))
409 : {
410 0 : return false;
411 : }
412 2 : if (!StreamNormalizedMap(design.GetComponentsByName(), options.mapBatchSize,
413 : &Odb::Grpc::NormalizedMapBatch::mutable_components_by_name, "componentsByName", emit))
414 : {
415 0 : return false;
416 : }
417 2 : if (!StreamNormalizedMap(design.GetPartsByName(), options.mapBatchSize,
418 : &Odb::Grpc::NormalizedMapBatch::mutable_parts_by_name, "partsByName", emit))
419 : {
420 0 : return false;
421 : }
422 : }
423 :
424 16 : return true;
425 17 : }
426 :
427 12053 : bool ApplyDesignChunk(Odb::Lib::Protobuf::ProductModel::Design& design,
428 : const Odb::Grpc::DesignChunk& chunk)
429 : {
430 12053 : switch (chunk.payload_case())
431 : {
432 21 : case Odb::Grpc::DesignChunk::kHeader:
433 21 : design.set_productmodel(chunk.header().product_model());
434 21 : design.set_name(chunk.header().name());
435 : // Presence parity with Design::to_protobuf: the monolith sets
436 : // filemodel only when the design has a file model, so the
437 : // header chunk carries it only then (Walk emits none for a
438 : // null/clipped file model) — never create an empty one.
439 21 : if (chunk.header().has_file_archive())
440 : {
441 20 : design.mutable_filemodel()->CopyFrom(chunk.header().file_archive());
442 : }
443 21 : return true;
444 :
445 21 : case Odb::Grpc::DesignChunk::kStep:
446 : {
447 21 : if (!design.has_filemodel())
448 : {
449 1 : return false;
450 : }
451 20 : const auto& step = chunk.step();
452 20 : (*design.mutable_filemodel()->mutable_stepsbyname())[step.step_name()].CopyFrom(step.step());
453 20 : return true;
454 : }
455 :
456 11927 : case Odb::Grpc::DesignChunk::kFeatures:
457 : {
458 11927 : if (!design.has_filemodel())
459 : {
460 1 : return false;
461 : }
462 11926 : const auto& batch = chunk.features();
463 11926 : auto* stepsByName = design.mutable_filemodel()->mutable_stepsbyname();
464 : // Linear scans, not find(): #599 diagnosed an address-layout
465 : // dependent protobuf-Map lookup bug (entry present under
466 : // iteration, absent under find() on the same object), never
467 : // root-caused. These maps hold a handful of entries, so
468 : // iteration is free — and the reference reassembly models
469 : // the recipe M6.4 clients will copy.
470 11926 : auto stepIt = stepsByName->end();
471 11926 : for (auto it = stepsByName->begin(); it != stepsByName->end(); ++it)
472 : {
473 11926 : if (it->first == batch.step_name())
474 : {
475 11926 : stepIt = it;
476 11926 : break;
477 : }
478 : }
479 11926 : if (stepIt == stepsByName->end())
480 : {
481 0 : return false;
482 : }
483 11926 : auto* layersByName = stepIt->second.mutable_layersbyname();
484 11926 : auto layerIt = layersByName->end();
485 163500 : for (auto it = layersByName->begin(); it != layersByName->end(); ++it)
486 : {
487 163499 : if (it->first == batch.layer_name())
488 : {
489 11925 : layerIt = it;
490 11925 : break;
491 : }
492 : }
493 11926 : if (layerIt == layersByName->end())
494 : {
495 1 : return false;
496 : }
497 11925 : auto* featureFile = layerIt->second.mutable_featurefile();
498 1349508 : for (const auto& record : batch.records())
499 : {
500 1337583 : featureFile->add_featurerecords()->CopyFrom(record);
501 : }
502 11925 : return true;
503 : }
504 :
505 2 : case Odb::Grpc::DesignChunk::kNets:
506 1290 : for (const auto& net : chunk.nets().nets())
507 : {
508 1288 : design.add_nets()->CopyFrom(net);
509 : }
510 2 : return true;
511 :
512 2 : case Odb::Grpc::DesignChunk::kPackages:
513 144 : for (const auto& packageEntry : chunk.packages().packages())
514 : {
515 142 : design.add_packages()->CopyFrom(packageEntry);
516 : }
517 2 : return true;
518 :
519 2 : case Odb::Grpc::DesignChunk::kComponents:
520 1628 : for (const auto& component : chunk.components().components())
521 : {
522 1626 : design.add_components()->CopyFrom(component);
523 : }
524 2 : return true;
525 :
526 2 : case Odb::Grpc::DesignChunk::kParts:
527 288 : for (const auto& part : chunk.parts().parts())
528 : {
529 286 : design.add_parts()->CopyFrom(part);
530 : }
531 2 : return true;
532 :
533 63 : case Odb::Grpc::DesignChunk::kConnectivity:
534 63 : design.mutable_connectivity()->MergeFrom(chunk.connectivity().entries());
535 63 : return true;
536 :
537 12 : case Odb::Grpc::DesignChunk::kNormalizedMaps:
538 : {
539 12 : const auto& maps = chunk.normalized_maps();
540 1300 : for (const auto& kv : maps.nets_by_name())
541 : {
542 1288 : (*design.mutable_netsbyname())[kv.first] = kv.second;
543 : }
544 154 : for (const auto& kv : maps.packages_by_name())
545 : {
546 142 : (*design.mutable_packagesbyname())[kv.first] = kv.second;
547 : }
548 1638 : for (const auto& kv : maps.components_by_name())
549 : {
550 1626 : (*design.mutable_componentsbyname())[kv.first] = kv.second;
551 : }
552 298 : for (const auto& kv : maps.parts_by_name())
553 : {
554 286 : (*design.mutable_partsbyname())[kv.first] = kv.second;
555 : }
556 12 : return true;
557 : }
558 :
559 1 : case Odb::Grpc::DesignChunk::PAYLOAD_NOT_SET:
560 : default:
561 1 : return false;
562 : }
563 : }
564 : }
565 : }
|