OdbDesign
OdbDesign ODB++ Parser - C++ library, gRPC/REST server, CLI, tests
 
Loading...
Searching...
No Matches
OdbDesignServiceImpl.cpp
Go to the documentation of this file.
6#include <utility>
7#include <odbdesign_export.h>
8#include <featuresfile.pb.h>
9#include <grpcpp/impl/status.h>
10#include <string>
11#include <sstream>
12#include <exception>
13#include <grpcpp/support/sync_stream.h>
14#include <grpcpp/support/status_code_enum.h>
15#include <memory>
16#include <algorithm>
17#include <cctype>
18#include <App/DesignCache.h>
19#include <design.pb.h>
20#include <standardfontsfile.pb.h>
21#include <vector>
22#include <Logger.h>
23
24#include <grpcpp/server_context.h>
26
27#if defined(__GLIBC__)
28#include <malloc.h>
29#endif
30
31namespace OdbDesignServer
32{
33 namespace Services
34 {
35
36 namespace
37 {
38 struct UnitsInfo
39 {
40 std::string units;
41 double unitsToMm{};
42 bool ok{ false };
43 std::string error;
44 };
45
46 std::string to_lower_copy(const std::string& s)
47 {
48 std::string out = s;
49 std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
50 return out;
51 }
52
53 // Client-facing exception detail for gRPC status messages.
54 // parse_error::what() is the bare constant "Parse error"; the
55 // diagnostics (data file/line, record, token, throw site) live in
56 // toString(), which otherwise reaches only the server log. Build
57 // them into the status, but reduce the data file to its basename:
58 // the extracted-archive path is server layout and must not reach
59 // clients of a server that may run with insecure credentials and
60 // no authentication (same policy as DesignCache::LoadFileArchive
61 // and the GetStandardFonts catch below).
62 std::string FormatExceptionForStatus(const std::exception& e)
63 {
64 const auto* pParseError = dynamic_cast<const Odb::Lib::FileModel::parse_error*>(&e);
65 if (pParseError == nullptr)
66 {
67 return e.what();
68 }
69
70 const auto& info = pParseError->getParseInfo();
71 std::ostringstream ss;
72 ss << e.what() << ": " << info.dataFile.filename().string();
73 if (info.dataLineNumber >= 0)
74 {
75 ss << ":" << info.dataLineNumber;
76 }
77 if (!info.dataLine.empty())
78 {
79 ss << " record [" << info.dataLine << "]";
80 }
81 if (!info.dataToken.empty())
82 {
83 ss << " token [" << info.dataToken << "]";
84 }
85 ss << " (thrown at " << pParseError->sourceLocation() << ")";
86 return ss.str();
87 }
88
89 UnitsInfo normalize_units(std::string rawUnits)
90 {
91 UnitsInfo info{};
92 if (rawUnits.empty())
93 {
94 // Return default "mm" units instead of error to allow client to handle gracefully
95 // This prevents FAILED_PRECONDITION errors for component layers (comp_+_top, comp_+_bot)
96 // that may lack units metadata in ODB++ files
97 info.units = "mm";
98 info.unitsToMm = 1.0;
99 info.ok = true;
100 return info;
101 }
102
103 auto u = to_lower_copy(rawUnits);
104 if (u == "inch" || u == "inches" || u == "in")
105 {
106 info.units = "inch";
107 info.unitsToMm = 25.4;
108 info.ok = true;
109 }
110 else if (u == "mm" || u == "millimeter" || u == "millimeters")
111 {
112 info.units = "mm";
113 info.unitsToMm = 1.0;
114 info.ok = true;
115 }
116 else if (u == "mil" || u == "mils")
117 {
118 info.units = "mil";
119 info.unitsToMm = 0.0254;
120 info.ok = true;
121 }
122 else if (u == "micron" || u == "microns" || u == "um" || u == "µm")
123 {
124 info.units = "micron";
125 info.unitsToMm = 0.001;
126 info.ok = true;
127 }
128 else
129 {
130 info.error = "unsupported units: " + rawUnits;
131 }
132
133 return info;
134 }
135
136 Odb::Lib::UnitType expected_symbol_unit_type_from_normalized_units(const std::string& normalizedUnits)
137 {
138 // Contract requires symbol numeric parameters to be in same unit system as feature coordinates.
139 auto u = to_lower_copy(normalizedUnits);
140 if (u == "mm" || u == "micron")
141 {
143 }
144 if (u == "inch" || u == "mil")
145 {
147 }
149 }
150
151 // M6: spool cache construction — a configured directory wins; the
152 // default is a private per-instance directory under the system temp
153 // dir (isolated from parallel server/test instances, removed on exit).
154 std::unique_ptr<StreamSpoolCache> MakeStreamSpoolCache(const Config::GrpcServiceConfig& config)
155 {
156 // stream_spool_max_bytes applies in BOTH branches: the
157 // private-directory cache previously hardcoded 4 GiB, so
158 // changing spool_max_bytes without setting spool_dir —
159 // including the documented unlimited sentinel — silently did
160 // nothing.
161 if (!config.stream_spool_dir.empty())
162 {
163 return std::make_unique<StreamSpoolCache>(
164 std::filesystem::path(config.stream_spool_dir), config.stream_spool_max_bytes);
165 }
166 return std::make_unique<StreamSpoolCache>(config.stream_spool_max_bytes);
167 }
168
169 // F9 (PR 609 review): a GetDesignStream burst leaves ~0.4-1.0 GiB
170 // of FREED-but-retained glibc arena pages mapped. Measured on
171 // 350-41017 (DesignStreamMemoryTests heap diagnostics): live
172 // in-use bytes are FLAT across a full 1.34 GB warm replay — the
173 // reader does not buffer — while free-but-held arena bytes grow
174 // ~0.36 GiB per replay, and malloc_trim(0) collapses RSS by
175 // ~0.87 GiB with zero live-byte change. glibc keeps those pages
176 // for reuse; on the 10 GiB pod the retention eats serve headroom
177 // for nothing. Return the pages when the burst ends (all exits,
178 // via RAII; POSIX/glibc-only, no-op elsewhere).
179 class StreamHeapTrimGuard
180 {
181 public:
182 ~StreamHeapTrimGuard()
183 {
184#if defined(__GLIBC__)
185 malloc_trim(0);
186#endif
187 }
188 };
189
190 }
191
192
193 OdbDesignServiceImpl::OdbDesignServiceImpl(std::shared_ptr<Odb::Lib::App::DesignCache> cache,
194 std::shared_ptr<Config::GrpcServiceConfig> config)
195 : m_designCache(std::move(cache)),
196 m_config(config ? config : Config::GrpcServiceConfig::CreateDefault()),
197 m_streamSpool(MakeStreamSpoolCache(*m_config))
198 {
199 m_streamOptions.featureBatchSize = static_cast<std::size_t>(
200 m_config->design_stream_feature_batch_size);
201 }
202
204 grpc::ServerContext *context,
205 const Odb::Grpc::GetDesignRequest *request,
206 Odb::Lib::Protobuf::ProductModel::Design *response)
207 {
208 try
209 {
210 const auto& designName = request->design_name();
211 // Normalized collections (nets/components/packages/parts + ByName maps)
212 // are omitted unless explicitly requested: they duplicate fileModel
213 // data clients actually read and dominate the payload.
214 const auto includeNormalizedLists = request->include_normalized_lists();
215 loginfo("[ConnTrace] GetDesign start: design_name=\"" + designName +
216 "\" include_normalized_lists=" + (includeNormalizedLists ? "true" : "false") + "\"");
217
218 // M1.4 fast path: when the serialized-response cache is warm,
219 // fill the response from cached wire bytes with a single
220 // parse — zero ProductModel-to-proto conversions. (The sync
221 // gRPC API serializes `response` itself after this handler
222 // returns, so raw bytes cannot be handed to the framework
223 // directly; parsing the cached buffer replaces the full tree
224 // build + deep copy and is still a strict win.)
225 std::string cachedBytes;
226 if (!includeNormalizedLists && m_designCache->TryGetDesignBytes(designName, cachedBytes))
227 {
228 // Budget enforced on the warm path too: the payload was
229 // serialized under the budget active at pre-serialization
230 // time; if the budget has since been lowered, parsing
231 // these bytes would allocate the exact protobuf tree the
232 // guard exists to prevent. Skip to the cold path, whose
233 // guard rejects over-budget designs.
234 const auto warmBudget = m_designCache->getMonolithicWireBudget();
235 if (warmBudget != 0 && cachedBytes.size() > warmBudget)
236 {
237 loginfo("[ConnTrace] GetDesign cached payload over current budget: design_name=\"" +
238 designName + "\" cachedBytes=" + std::to_string(cachedBytes.size()) +
239 " budget=" + std::to_string(warmBudget) + "; using cold path");
240 }
241 else if (response->ParseFromString(cachedBytes))
242 {
243 loginfo("[ConnTrace] GetDesign ok (cached bytes): design_name=\"" + designName +
244 "\" approx_bytes=" + std::to_string(response->ByteSizeLong()));
245 return grpc::Status::OK;
246 }
247 else
248 {
249 // Corrupt cached bytes: never serve an empty/partial
250 // message as OK. Log, clear the partially-merged message,
251 // and fall through to the cold path below (the poisoned
252 // payload is dropped by the next invalidation).
253 logerror("Cached design bytes failed to parse for \"" + designName +
254 "\"; falling back to full load");
255 response->Clear();
256 }
257 }
258
259 const auto design = m_designCache->GetDesign(designName);
260 if (design == nullptr)
261 {
262 return {grpc::StatusCode::NOT_FOUND, "Design not found: " + designName};
263 }
264
265 // M6 hard guard: a design whose estimated serialized size
266 // exceeds the monolithic serve budget must not have its
267 // protobuf tree built here — that build (plus the serialized
268 // copy) is the 10 GiB-pod OOM. Clean error instead of a pod
269 // kill; the chunked GetDesignStream RPC serves these designs.
270 // (Cached bytes served above are for designs pre-serialized
271 // under this same budget, so they cannot be over it.)
272 const auto monolithicBudget = m_designCache->getMonolithicWireBudget();
273 if (monolithicBudget != 0 &&
274 design->EstimateWireSizeBytes(includeNormalizedLists) > monolithicBudget)
275 {
276 return {grpc::StatusCode::RESOURCE_EXHAUSTED,
277 "Design \"" + designName + "\" exceeds the monolithic GetDesign serve budget (" +
278 std::to_string(monolithicBudget) + " estimated wire bytes); use the GetDesignStream RPC"};
279 }
280
281 // Cold path: serialize directly for this one response (the
282 // status-quo cost — exactly one tree build, on this thread).
283 // Cache population happens ONLY on the background
284 // pre-serialization worker scheduled at load completion —
285 // populating here as well meant two threads deep-building
286 // the full protobuf tree concurrently, which thrashed memory
287 // on large designs. The worker may land while we convert;
288 // either way the next request takes the cached-bytes path.
289 //
290 // include_normalized_lists responses are always served cold (the
291 // pre-serialization worker caches only the default pruned flavor),
292 // so opting in pays the full tree build per request.
293 *response = *(design->to_protobuf(includeNormalizedLists));
294
295 loginfo("[ConnTrace] GetDesign ok: design_name=\"" + request->design_name() +
296 "\" approx_bytes=" + std::to_string(response->ByteSizeLong()));
297
298 // Log component height data in gRPC response for traced components
299 if (response->has_filemodel())
300 {
301 const auto& fileModel = response->filemodel();
302 for (const auto& kvStep : fileModel.stepsbyname())
303 {
304 const auto& step = kvStep.second;
305 for (const auto& kvLayer : step.layersbyname())
306 {
307 const auto& layer = kvLayer.second;
308 if (layer.has_components())
309 {
310 const auto& componentsFile = layer.components();
311 std::vector<std::string> attributeNames;
312 for (const auto& attrName : componentsFile.attributenames())
313 {
314 attributeNames.push_back(attrName);
315 }
316
317 for (const auto& compRecord : componentsFile.componentrecords())
318 {
319 std::map<std::string, std::string> lookupTable;
320 for (const auto& kv : compRecord.attributelookuptable())
321 {
322 lookupTable[kv.first] = kv.second;
323 }
324
326 compRecord.compname(),
327 compRecord.pkgref(),
328 lookupTable,
329 attributeNames);
330 }
331 }
332 }
333 }
334 }
335
336 return grpc::Status::OK;
337 }
338 catch (const std::exception &e)
339 {
340 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
341 return {grpc::StatusCode::INTERNAL, error};
342 }
343 }
344
346 grpc::ServerContext* context,
347 const Odb::Grpc::GetDesignRequest* request,
348 grpc::ServerWriter<Odb::Grpc::DesignChunk>* writer)
349 {
350 try
351 {
352 // Return the burst's freed arena pages on every exit (see
353 // StreamHeapTrimGuard — F9 root-cause fix).
354 const StreamHeapTrimGuard heapTrimOnExit;
355
356 const auto& designName = request->design_name();
357 const auto includeNormalizedLists = request->include_normalized_lists();
358 loginfo("[ConnTrace] GetDesignStream start: design_name=\"" + designName +
359 "\" include_normalized_lists=" + (includeNormalizedLists ? "true" : "false"));
360
361 if (!m_config->enable_design_streaming)
362 {
363 return {grpc::StatusCode::UNIMPLEMENTED, "Design streaming is not enabled"};
364 }
365
366 // Read the response generation BEFORE fetching the design:
367 // an invalidation landing between the fetch and the
368 // generation read would otherwise hand the cold fill a
369 // stale Design published under the NEW generation — the
370 // commit-time re-check below compares against
371 // generationAtStart, so it cannot catch a bump that predates
372 // its read. Reading first makes any invalidation in the
373 // window abort the publish: the safe direction (the
374 // streaming client still gets its content; the next request
375 // refills).
376 const auto generationAtStart = m_designCache->GetResponseGeneration(designName);
377
378 // Resolve the design (parse + product-model build; the parse
379 // fits the pod budget by design — it is the monolithic
380 // serialization this RPC replaces that does not).
381 const auto design = m_designCache->GetDesign(designName);
382 if (design == nullptr)
383 {
384 return {grpc::StatusCode::NOT_FOUND, "Design not found: " + designName};
385 }
386
387 // Warm spool (default flavour only — the normalized flavour is
388 // never cached, mirroring the monolithic response cache).
389 // Identity is (design name, response-cache generation); a
390 // generation bump (re-upload/invalidation/eviction) drops it.
391 if (!includeNormalizedLists)
392 {
393 const auto generation = m_designCache->GetResponseGeneration(designName);
395 if (m_streamSpool->TryOpenReader(designName, generation, reader))
396 {
397 Odb::Grpc::DesignChunk chunk;
398 while (reader.Next(chunk))
399 {
400 if (context->IsCancelled())
401 {
402 loginfo("[ConnTrace] GetDesignStream client_disconnected (warm): design_name=\"" +
403 designName + "\" records=" + std::to_string(reader.RecordsRead()));
404 return grpc::Status::OK;
405 }
406 if (!writer->Write(chunk))
407 {
408 // Truncation without a cancelling client must
409 // not end in a clean OK trailer — chunk count
410 // is the only completion signal M6.4 clients
411 // have (see the cold path below).
412 logerror("[ConnTrace] GetDesignStream write failed (warm): design_name=\"" +
413 designName + "\" records=" + std::to_string(reader.RecordsRead()));
414 return {grpc::StatusCode::INTERNAL,
415 "Internal server error: GetDesignStream write failed (warm, truncated)"};
416 }
417 chunk.Clear();
418 }
419 if (reader.Failed() || reader.Truncated())
420 {
421 // Read error, OR a clean EOF short of the committed
422 // record count (truncation at a record boundary /
423 // emptied file — framing alone cannot distinguish
424 // that from a legitimate end-of-spool): either way
425 // the client would get a clean trailer after a
426 // partial chunk sequence. INTERNAL, and drop the
427 // damaged spool.
428 logerror("[ConnTrace] GetDesignStream spool read failed: design_name=\"" + designName +
429 "\" failed=" + (reader.Failed() ? "true" : "false") +
430 " recordsRead=" + std::to_string(reader.RecordsRead()) +
431 " recordsExpected=" + std::to_string(reader.ExpectedRecords()));
432 m_streamSpool->Drop(designName);
433 return {grpc::StatusCode::INTERNAL,
434 "Internal server error: cached stream spool read failure (truncated)"};
435 }
436 loginfo("[ConnTrace] GetDesignStream ok (warm spool): design_name=\"" + designName +
437 "\" records=" + std::to_string(reader.RecordsRead()));
438 return grpc::Status::OK;
439 }
440 }
441
442 // Cold fill: walk the product model chunk-by-chunk. Over-budget
443 // designs (the ones this RPC exists for) tee every chunk to the
444 // disk spool while writing it to the wire, so the NEXT request
445 // is served warm from page cache with no product-model walk.
446 const auto monolithicBudget = m_designCache->getMonolithicWireBudget();
447 const bool overBudget = monolithicBudget != 0 &&
448 design->EstimateWireSizeBytes(/*includeNormalizedLists=*/false) > monolithicBudget;
449
450 // Spool outcome for the done-log, tracked at each transition
451 // so the label reports what actually happened. The previous
452 // expression printed "spool=aborted" for normalized-flavour
453 // requests (no spool write is ever attempted for them) and
454 // "spool=committed" for fills that aborted on client
455 // cancellation or a generation bump.
456 enum class SpoolOutcome { NoneAttempted, OpenFailed, Aborted, Committed };
457 auto spoolOutcome = SpoolOutcome::NoneAttempted;
458
459 StreamSpoolCache::Writer spoolWriter;
460 bool spooling = false;
461 if (!includeNormalizedLists && overBudget)
462 {
463 spooling = m_streamSpool->BeginWrite(designName, generationAtStart, spoolWriter);
464 // An active fill publishes only on full success; Aborted
465 // is the default until Commit() says otherwise.
466 spoolOutcome = spooling ? SpoolOutcome::Aborted : SpoolOutcome::OpenFailed;
467 }
468
470 const auto sink = [&](const Odb::Grpc::DesignChunk& chunk) -> bool
471 {
472 if (context->IsCancelled())
473 {
474 return false;
475 }
476 if (!writer->Write(chunk))
477 {
478 return false;
479 }
480 if (spooling && !spoolWriter.Append(chunk))
481 {
482 // The wire response is unaffected by spool I/O trouble
483 // (disk full, etc.): drop the spool, keep streaming.
484 logerror("[ConnTrace] GetDesignStream spool append failed; continuing without spool");
485 spoolWriter.Abort();
486 spooling = false;
487 }
488 return true;
489 };
490
491 const bool completed = DesignStreamWalker::Walk(
492 *design, includeNormalizedLists, m_streamOptions, sink, &stats);
493
494 if (spooling)
495 {
496 // Publish only a complete, still-current fill: an aborted
497 // walk (client gone) or an invalidation that raced the fill
498 // (generation bumped) must not poison the warm path.
499 const bool generationUnchanged =
500 m_designCache->GetResponseGeneration(designName) == generationAtStart;
501 if (completed && generationUnchanged)
502 {
503 // Commit() reports whether the spool was actually
504 // published: a flush/close failure (ENOSPC surfacing
505 // at flush time) deletes the incomplete file instead.
506 spoolOutcome = spoolWriter.Commit()
507 ? SpoolOutcome::Committed
508 : SpoolOutcome::Aborted;
509 }
510 else
511 {
512 spoolWriter.Abort();
513 }
514 }
515
516 const char* spoolLabel =
517 spoolOutcome == SpoolOutcome::Committed ? " spool=committed" :
518 spoolOutcome == SpoolOutcome::Aborted ? " spool=aborted" :
519 spoolOutcome == SpoolOutcome::OpenFailed ? " spool=open-failed" : "";
520 loginfo("[ConnTrace] GetDesignStream done: design_name=\"" + designName +
521 "\" completed=" + (completed ? "true" : "false") +
522 " chunks=" + std::to_string(stats.chunks) +
523 " wire_bytes=" + std::to_string(stats.wireBytes) +
524 spoolLabel);
525
526 if (!completed && !context->IsCancelled())
527 {
528 // The sink aborted for a reason other than client
529 // cancellation (writer->Write failed — e.g. flow control
530 // or a broken connection without a propagated cancel).
531 // Returning OK here would hand the client a clean trailer
532 // after a truncated chunk sequence, and the chunk count is
533 // the only completion signal M6.4 clients have.
534 logerror("[ConnTrace] GetDesignStream write failed (cold): design_name=\"" + designName +
535 "\" chunks=" + std::to_string(stats.chunks));
536 return {grpc::StatusCode::INTERNAL,
537 "Internal server error: GetDesignStream write failed (cold, truncated)"};
538 }
539 return grpc::Status::OK;
540 }
541 catch (const std::exception& e)
542 {
543 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
544 return {grpc::StatusCode::INTERNAL, error};
545 }
546 }
547
549 grpc::ServerContext *context,
550 const Odb::Grpc::GetLayerFeaturesRequest *request,
551 grpc::ServerWriter<Odb::Lib::Protobuf::FeaturesFile::FeatureRecord> *writer)
552 {
553 try
554 {
555 loginfo("[ConnTrace] GetLayerFeaturesStream start: design=\"" + request->design_name() +
556 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
557 const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
558 if (fileArchive == nullptr)
559 {
560 return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
561 }
562
563 const auto pStep = fileArchive->GetStepDirectory(request->step_name());
564 if (pStep == nullptr)
565 {
566 return {grpc::StatusCode::NOT_FOUND, "Step not found: " + request->step_name()};
567 }
568
569 const auto pLayersByName = pStep->GetLayersByName();
570 const auto layerIt = pLayersByName.find(request->layer_name());
571 if (layerIt == pLayersByName.end())
572 {
573 return {grpc::StatusCode::NOT_FOUND, "Layer not found: " + request->layer_name()};
574 }
575
576 const auto &featuresFile = layerIt->second->GetFeaturesFile();
577 const auto &featureRecords = featuresFile.GetFeatureRecords();
578
579 // Reuse a single protobuf message instance to avoid per-iteration heap allocations.
580 Odb::Lib::Protobuf::FeaturesFile::FeatureRecord featureRecordMsg;
581
582 size_t sentCount = 0;
583 for (const auto &featureRecord : featureRecords)
584 {
585 if (!featureRecord)
586 continue;
587
588 // Populate reused message: obtain per-feature proto and swap into reusable instance.
589 auto pFeatureRecordMsg = featureRecord->to_protobuf();
590 if (pFeatureRecordMsg != nullptr)
591 {
592 featureRecordMsg.Swap(pFeatureRecordMsg.get()); // Efficient move of fields.
593 }
594 else
595 {
596 continue; // Skip if conversion failed or returned nullptr.
597 }
598
599 if (!writer->Write(featureRecordMsg))
600 {
601 loginfo("[ConnTrace] GetLayerFeaturesStream client_disconnected: design=\"" + request->design_name() +
602 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
603 "\" sent=" + std::to_string(sentCount));
604 break;
605 }
606
607 ++sentCount;
608
609 // Clear for reuse while retaining allocated capacity.
610 featureRecordMsg.Clear();
611 }
612
613 loginfo("[ConnTrace] GetLayerFeaturesStream done: design=\"" + request->design_name() +
614 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
615 "\" sent=" + std::to_string(sentCount));
616 return grpc::Status::OK;
617 }
618 catch (const std::exception &e)
619 {
620 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
621 return {grpc::StatusCode::INTERNAL, error};
622 }
623 }
624
626 grpc::ServerContext *context,
627 const Odb::Grpc::GetLayerFeaturesRequest *request,
628 grpc::ServerWriter<Odb::Grpc::FeatureRecordBatch> *writer)
629 {
630 try
631 {
632 loginfo("[ConnTrace] GetLayerFeaturesBatchStream start: design=\"" + request->design_name() +
633 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
634 // Feature flag check: return UNIMPLEMENTED if batch streaming is disabled
635 if (!m_config->enable_batch_streaming)
636 {
637 loginfo("[ConnTrace] GetLayerFeaturesBatchStream disabled: returning UNIMPLEMENTED");
638 return {grpc::StatusCode::UNIMPLEMENTED, "Batch streaming is not enabled"};
639 }
640
641 // Reuse validation logic from GetLayerFeaturesStream
642 const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
643 if (fileArchive == nullptr)
644 {
645 return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
646 }
647
648 const auto pStep = fileArchive->GetStepDirectory(request->step_name());
649 if (pStep == nullptr)
650 {
651 return {grpc::StatusCode::NOT_FOUND, "Step not found: " + request->step_name()};
652 }
653
654 const auto pLayersByName = pStep->GetLayersByName();
655 const auto layerIt = pLayersByName.find(request->layer_name());
656 if (layerIt == pLayersByName.end())
657 {
658 return {grpc::StatusCode::NOT_FOUND, "Layer not found: " + request->layer_name()};
659 }
660
661 const auto &featuresFile = layerIt->second->GetFeaturesFile();
662 const auto &featureRecords = featuresFile.GetFeatureRecords();
663
664 // Reuse a single protobuf message instance to avoid per-iteration heap allocations.
665 Odb::Lib::Protobuf::FeaturesFile::FeatureRecord featureRecordMsg;
666 Odb::Grpc::FeatureRecordBatch batch;
667
668 const int batchSize = m_config->batch_size;
669 int currentBatchCount = 0;
670 size_t batchesSent = 0;
671 size_t featuresSent = 0;
672
673 for (const auto &featureRecord : featureRecords)
674 {
675 // Check for cancellation
676 if (context->IsCancelled())
677 {
678 return {grpc::StatusCode::CANCELLED, "Request cancelled"};
679 }
680
681 if (!featureRecord)
682 continue;
683
684 // Populate reused message: obtain per-feature proto and swap into reusable instance.
685 auto pFeatureRecordMsg = featureRecord->to_protobuf();
686 if (pFeatureRecordMsg != nullptr)
687 {
688 featureRecordMsg.Swap(pFeatureRecordMsg.get()); // Efficient move of fields.
689 }
690 else
691 {
692 continue; // Skip if conversion failed or returned nullptr.
693 }
694
695 // Add feature to current batch
696 batch.add_features()->CopyFrom(featureRecordMsg);
697 currentBatchCount++;
698 featuresSent++;
699
700 // Clear for reuse while retaining allocated capacity.
701 featureRecordMsg.Clear();
702
703 // Send batch when it reaches the configured size
704 if (currentBatchCount >= batchSize)
705 {
706 // Check for cancellation before writing
707 if (context->IsCancelled())
708 {
709 return {grpc::StatusCode::CANCELLED, "Request cancelled"};
710 }
711
712 if (!writer->Write(batch))
713 {
714 loginfo("[ConnTrace] GetLayerFeaturesBatchStream client_disconnected: design=\"" + request->design_name() +
715 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
716 "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent));
717 return grpc::Status::OK;
718 }
719 batchesSent++;
720 batch.Clear();
721 currentBatchCount = 0;
722 }
723 }
724
725 // Send final batch if there are remaining features
726 if (currentBatchCount > 0)
727 {
728 // Check for cancellation before writing final batch
729 if (context->IsCancelled())
730 {
731 return {grpc::StatusCode::CANCELLED, "Request cancelled"};
732 }
733
734 if (!writer->Write(batch))
735 {
736 loginfo("[ConnTrace] GetLayerFeaturesBatchStream client_disconnected(final): design=\"" + request->design_name() +
737 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
738 "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent));
739 return grpc::Status::OK;
740 }
741 batchesSent++;
742 }
743
744 // Stream completion is indicated by normal gRPC end-of-stream semantics
745 loginfo("[ConnTrace] GetLayerFeaturesBatchStream done: design=\"" + request->design_name() +
746 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
747 "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent) +
748 " batch_size=" + std::to_string(batchSize));
749 return grpc::Status::OK;
750 }
751 catch (const std::exception &e)
752 {
753 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
754 return {grpc::StatusCode::INTERNAL, error};
755 }
756 }
757
759 grpc::ServerContext* context,
760 const Odb::Grpc::GetLayerSymbolsRequest* request,
761 Odb::Grpc::GetLayerSymbolsResponse* response)
762 {
763 try
764 {
765 loginfo("[ConnTrace] GetLayerSymbols start: design=\"" + request->design_name() +
766 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
767 const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
768 if (fileArchive == nullptr)
769 {
770 return { grpc::StatusCode::NOT_FOUND, "Design not found" };
771 }
772
773 const auto pStep = fileArchive->GetStepDirectory(request->step_name());
774 if (pStep == nullptr)
775 {
776 return { grpc::StatusCode::NOT_FOUND, "Step not found" };
777 }
778
779 const auto& layersByName = pStep->GetLayersByName();
780 const auto layerIt = layersByName.find(request->layer_name());
781 if (layerIt == layersByName.end())
782 {
783 return { grpc::StatusCode::NOT_FOUND, "Layer not found" };
784 }
785
786 const auto& featuresFile = layerIt->second->GetFeaturesFile();
787 auto unitsInfo = normalize_units(featuresFile.GetUnits());
788 if (!unitsInfo.ok)
789 {
790 return { grpc::StatusCode::FAILED_PRECONDITION, unitsInfo.error };
791 }
792
793 response->set_sym_num_base(0);
794 response->set_units(unitsInfo.units);
795 response->set_units_to_mm(unitsInfo.unitsToMm);
796
797 loginfo("[ConnTrace] GetLayerSymbols units: layer=\"" + request->layer_name() + "\" units=\"" +
798 unitsInfo.units + "\" units_to_mm=" + std::to_string(unitsInfo.unitsToMm));
799
800 const auto symbols = Odb::Lib::FileModel::Design::collect_symbols(featuresFile);
801 const auto& featureRecords = featuresFile.GetFeatureRecords();
802
803 // Strict contract enforcement (S2): symbol unit types must match the layer units.
804 const auto expectedSymbolUnitType = expected_symbol_unit_type_from_normalized_units(unitsInfo.units);
805 int metricCount = 0;
806 int imperialCount = 0;
807 int noneCount = 0;
808 for (const auto& sym : symbols)
809 {
810 if (!sym) continue;
811 switch (sym->GetUnitType())
812 {
813 case Odb::Lib::UnitType::Metric: metricCount++; break;
814 case Odb::Lib::UnitType::Imperial: imperialCount++; break;
815 case Odb::Lib::UnitType::None: noneCount++; break;
816 }
817 }
818
819 const bool hasMixedSymbolUnitTypes = metricCount > 0 && imperialCount > 0;
820 // Spec: if the symbol record omits its unit, it inherits the enclosing features file unit system.
821 // So UnitType::None is treated as compatible with the layer units.
822 const bool hasMismatchVsLayerUnits =
823 (expectedSymbolUnitType == Odb::Lib::UnitType::Metric && imperialCount > 0) ||
824 (expectedSymbolUnitType == Odb::Lib::UnitType::Imperial && metricCount > 0);
825
826 if (hasMixedSymbolUnitTypes || hasMismatchVsLayerUnits)
827 {
828 std::string msg =
829 "symbol UnitType mismatch vs layer units (strict): layer_units=" + unitsInfo.units +
830 ", symbol_counts(metric=" + std::to_string(metricCount) +
831 ", imperial=" + std::to_string(imperialCount) +
832 ", none=" + std::to_string(noneCount) + ")";
833 return { grpc::StatusCode::FAILED_PRECONDITION, msg };
834 }
835
836 // Sanity check: limit symbol indices to avoid excessive memory allocation or logic errors due to invalid/corrupted data.
837 // 10,000,000 is chosen as a very high upper bound, far exceeding any realistic symbol count in production designs.
838 // If a valid symbol index exceeds this limit, it will be ignored, which may cause rendering or data issues.
839 // Adjust this value with caution and ensure all code using symbol indices is reviewed if changed.
840 constexpr int kMaxReasonableIndex = 10'000'000;
841 int maxSymRef = -1;
842 // Single pass to validate pad contracts and capture the max referenced symbol id without extra traversals.
843 for (const auto& fr : featureRecords)
844 {
845 if (!fr) continue;
846
847 // Strict contract enforcement (S1): Pads must reference symbols via FeatureRecord.sym_num.
849 fr->apt_def_symbol_num >= 0 && fr->sym_num != fr->apt_def_symbol_num)
850 {
851 return { grpc::StatusCode::FAILED_PRECONDITION, "pad sym_num does not match apt_def_symbol_num (contract violation)" };
852 }
853
854 if (fr->sym_num >= 0 && fr->sym_num < kMaxReasonableIndex)
855 {
856 maxSymRef = std::max(maxSymRef, fr->sym_num);
857 }
858 if (fr->apt_def_symbol_num >= 0 && fr->apt_def_symbol_num < kMaxReasonableIndex)
859 {
860 maxSymRef = std::max(maxSymRef, fr->apt_def_symbol_num);
861 }
862 }
863
864 int resultSize = maxSymRef + 1;
865 if (resultSize < 0) resultSize = 0;
866
867 std::vector<std::shared_ptr<Odb::Lib::FileModel::Design::SymbolName>> ordered(static_cast<size_t>(resultSize), nullptr);
868
869 // Ordering strategy:
870 // 1) Place symbols that declare explicit indices; keep the first occurrence on duplicates.
871 // 2) Fill remaining gaps with symbols lacking indices in deterministic input order.
872 // This preserves stable ordering for callers that rely on sym_num alignment.
873 int maxIndex = -1;
874 for (const auto& sym : symbols)
875 {
876 if (!sym) continue;
877 const auto idx = sym->GetIndex();
878 maxIndex = std::max(maxIndex, idx);
879 if (idx < 0) continue;
880 const auto idxSize = static_cast<size_t>(idx);
881 if (idx >= static_cast<int>(ordered.size()))
882 {
883 ordered.resize(idxSize + 1, nullptr);
884 }
885 if (!ordered[idxSize])
886 {
887 ordered[idxSize] = sym;
888 }
889 }
890
891 // Then, fill remaining slots with symbols lacking indices in deterministic order.
892 // Precompute available slots for O(n) filling instead of O(n²) linear search.
893 std::vector<size_t> availableSlots;
894 for (size_t i = 0; i < ordered.size(); ++i)
895 {
896 if (!ordered[i])
897 {
898 availableSlots.push_back(i);
899 }
900 }
901 size_t slotIdx = 0;
902 for (const auto& sym : symbols)
903 {
904 if (!sym) continue;
905 if (sym->GetIndex() >= 0) continue;
906
907 if (slotIdx < availableSlots.size())
908 {
909 ordered[availableSlots[slotIdx]] = sym;
910 ++slotIdx;
911 }
912 else
913 {
914 ordered.push_back(sym);
915 }
916 }
917
918 // Pad to cover all referenced sym_num values.
919 const int requiredSize = std::max(maxSymRef, maxIndex) + 1;
920 if (requiredSize > static_cast<int>(ordered.size()))
921 {
922 ordered.resize(static_cast<size_t>(requiredSize), nullptr);
923 }
924
925 Odb::Lib::Protobuf::SymbolName placeholder;
926 placeholder.set_name("");
927 placeholder.set_unittype(Odb::Lib::Protobuf::UnitType::None);
928
929 for (const auto& sym : ordered)
930 {
931 if (sym)
932 {
933 auto pb = sym->to_protobuf();
934 if (pb && pb->unittype() == Odb::Lib::Protobuf::UnitType::None)
935 {
936 pb->set_unittype(static_cast<Odb::Lib::Protobuf::UnitType>(expectedSymbolUnitType));
937 }
938 response->add_symbol_names()->CopyFrom(*pb);
939 }
940 else
941 {
942 response->add_symbol_names()->CopyFrom(placeholder);
943 }
944 }
945
946 loginfo("[ConnTrace] GetLayerSymbols done: design=\"" + request->design_name() +
947 "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
948 "\" symbol_names=" + std::to_string(response->symbol_names_size()));
949
950 return grpc::Status::OK;
951 }
952 catch (const std::exception& e)
953 {
954 std::string error = "Internal server error: " + std::string(e.what());
955 return { grpc::StatusCode::INTERNAL, error };
956 }
957 }
958
960 grpc::ServerContext *context,
961 const Odb::Grpc::GetStandardFontsRequest *request,
962 Odb::Lib::Protobuf::StandardFontsFile *response)
963 {
964 try
965 {
966 loginfo("[ConnTrace] GetStandardFonts start: design_name=\"" + request->design_name() + "\"");
967
968 const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
969 if (fileArchive == nullptr)
970 {
971 return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
972 }
973
974 auto fontsFilePb = fileArchive->GetStandardFontsFile().to_protobuf();
975 if (fontsFilePb == nullptr)
976 {
977 return {grpc::StatusCode::INTERNAL, "Failed to serialize standard fonts file"};
978 }
979 response->CopyFrom(*fontsFilePb);
980
981 loginfo("[ConnTrace] GetStandardFonts done: design_name=\"" + request->design_name() +
982 "\" characters=" + std::to_string(response->m_characterblocks_size()));
983
984 return grpc::Status::OK;
985 }
986 catch (const std::exception &e)
987 {
988 // Log the full details server-side only: e.what() can embed
989 // filesystem paths, which must not reach clients of a server
990 // that runs with insecure credentials and no authentication.
991 logerror(std::string("[ConnTrace] GetStandardFonts failed: ") + e.what());
992 return {grpc::StatusCode::INTERNAL, "Internal server error"};
993 }
994 }
995
997 grpc::ServerContext *context,
998 const Odb::Grpc::RequestLoadDesignRequest *request,
999 Odb::Grpc::RequestLoadDesignResponse *response)
1000 {
1001 try
1002 {
1003 loginfo("[ConnTrace] RequestLoadDesign: design_name=\"" + request->design_name() + "\"");
1004 response->set_design_name(request->design_name());
1005 response->set_status(ComputeRequestLoadStatus(*m_designCache, request->design_name()));
1006 return grpc::Status::OK;
1007 }
1008 catch (const std::exception &e)
1009 {
1010 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
1011 return {grpc::StatusCode::INTERNAL, error};
1012 }
1013 }
1014
1016 grpc::ServerContext *context,
1017 const Odb::Grpc::HealthCheckRequest *request,
1018 Odb::Grpc::HealthCheckResponse *response)
1019 {
1020 try
1021 {
1022 // Optionally, check the service name from the request
1023 if (!request->service().empty() && request->service() != "OdbDesignService")
1024 {
1025 response->set_status(Odb::Grpc::HealthCheckResponse::NOT_SERVING);
1026 return grpc::Status::OK;
1027 }
1028
1029 response->set_status(Odb::Grpc::HealthCheckResponse::SERVING);
1030 return grpc::Status::OK;
1031 }
1032 catch (const std::exception &e)
1033 {
1034 std::string error = "Internal server error: " + FormatExceptionForStatus(e);
1035 return {grpc::StatusCode::INTERNAL, error};
1036 }
1037 }
1038
1039 } // namespace Services
1040} // namespace OdbDesignServer
#define logerror(s)
Definition Logger.h:17
#define loginfo(s)
Definition Logger.h:14
std::string error
std::string units
double unitsToMm
static bool Walk(const Odb::Lib::ProductModel::Design &design, bool includeNormalizedLists, const DesignStreamOptions &options, const ChunkSink &sink, Stats *stats=nullptr)
OdbDesignServiceImpl(std::shared_ptr< Odb::Lib::App::DesignCache > cache, std::shared_ptr< Config::GrpcServiceConfig > config=nullptr)
grpc::Status GetDesignStream(grpc::ServerContext *context, const Odb::Grpc::GetDesignRequest *request, grpc::ServerWriter< Odb::Grpc::DesignChunk > *writer) override
grpc::Status GetStandardFonts(grpc::ServerContext *context, const Odb::Grpc::GetStandardFontsRequest *request, Odb::Lib::Protobuf::StandardFontsFile *response) override
grpc::Status GetLayerFeaturesStream(grpc::ServerContext *context, const Odb::Grpc::GetLayerFeaturesRequest *request, grpc::ServerWriter< Odb::Lib::Protobuf::FeaturesFile::FeatureRecord > *writer) override
grpc::Status RequestLoadDesign(grpc::ServerContext *context, const Odb::Grpc::RequestLoadDesignRequest *request, Odb::Grpc::RequestLoadDesignResponse *response) override
grpc::Status GetLayerSymbols(grpc::ServerContext *context, const Odb::Grpc::GetLayerSymbolsRequest *request, Odb::Grpc::GetLayerSymbolsResponse *response) override
grpc::Status GetDesign(grpc::ServerContext *context, const Odb::Grpc::GetDesignRequest *request, Odb::Lib::Protobuf::ProductModel::Design *response) override
grpc::Status GetLayerFeaturesBatchStream(grpc::ServerContext *context, const Odb::Grpc::GetLayerFeaturesRequest *request, grpc::ServerWriter< Odb::Grpc::FeatureRecordBatch > *writer) override
grpc::Status HealthCheck(grpc::ServerContext *context, const Odb::Grpc::HealthCheckRequest *request, Odb::Grpc::HealthCheckResponse *response) override
bool Next(google::protobuf::Message &record)
bool Append(const google::protobuf::Message &record)
void logGrpcResponse(const std::string &compName, unsigned int pkgRef, const std::map< std::string, std::string > &lookupTable, const std::vector< std::string > &attributeNames) const
char const * what() const noexcept
Odb::Grpc::LoadStatus ComputeRequestLoadStatus(Odb::Lib::App::DesignCache &designCache, const std::string &designName)
SymbolName::Vector collect_symbols(const FeaturesFile &featuresFile)
UnitType
Definition enums.h:25