Line data Source code
1 : #include "OdbDesignServiceImpl.h"
2 : #include "DesignStreamWalker.h"
3 : #include "FileModel/Design/FeaturesFile.h"
4 : #include "FileModel/Design/FileArchive.h"
5 : #include "FileModel/parse_error.h"
6 : #include <utility>
7 : #include <odbdesign_export.h>
8 : #include <featuresfile.pb.h>
9 : #include <grpcpp/impl/status.h>
10 : #include <string>
11 : #include <sstream>
12 : #include <exception>
13 : #include <grpcpp/support/sync_stream.h>
14 : #include <grpcpp/support/status_code_enum.h>
15 : #include <memory>
16 : #include <algorithm>
17 : #include <cctype>
18 : #include <App/DesignCache.h>
19 : #include <design.pb.h>
20 : #include <standardfontsfile.pb.h>
21 : #include <vector>
22 : #include <Logger.h>
23 :
24 : #include <grpcpp/server_context.h>
25 : #include <FileModel/Design/ComponentHeightTracer.h>
26 :
27 : #if defined(__GLIBC__)
28 : #include <malloc.h>
29 : #endif
30 :
31 : namespace OdbDesignServer
32 : {
33 : namespace Services
34 : {
35 :
36 : namespace
37 : {
38 : struct UnitsInfo
39 : {
40 : std::string units;
41 : double unitsToMm{};
42 : bool ok{ false };
43 : std::string error;
44 : };
45 :
46 6 : std::string to_lower_copy(const std::string& s)
47 : {
48 6 : std::string out = s;
49 26 : std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
50 6 : return out;
51 : }
52 :
53 : // Client-facing exception detail for gRPC status messages.
54 : // parse_error::what() is the bare constant "Parse error"; the
55 : // diagnostics (data file/line, record, token, throw site) live in
56 : // toString(), which otherwise reaches only the server log. Build
57 : // them into the status, but reduce the data file to its basename:
58 : // the extracted-archive path is server layout and must not reach
59 : // clients of a server that may run with insecure credentials and
60 : // no authentication (same policy as DesignCache::LoadFileArchive
61 : // and the GetStandardFonts catch below).
62 1 : std::string FormatExceptionForStatus(const std::exception& e)
63 : {
64 1 : const auto* pParseError = dynamic_cast<const Odb::Lib::FileModel::parse_error*>(&e);
65 1 : if (pParseError == nullptr)
66 : {
67 0 : return e.what();
68 : }
69 :
70 1 : const auto& info = pParseError->getParseInfo();
71 1 : std::ostringstream ss;
72 1 : ss << e.what() << ": " << info.dataFile.filename().string();
73 1 : if (info.dataLineNumber >= 0)
74 : {
75 1 : ss << ":" << info.dataLineNumber;
76 : }
77 1 : if (!info.dataLine.empty())
78 : {
79 1 : ss << " record [" << info.dataLine << "]";
80 : }
81 1 : if (!info.dataToken.empty())
82 : {
83 1 : ss << " token [" << info.dataToken << "]";
84 : }
85 1 : ss << " (thrown at " << pParseError->sourceLocation() << ")";
86 1 : return ss.str();
87 1 : }
88 :
89 4 : UnitsInfo normalize_units(std::string rawUnits)
90 : {
91 4 : UnitsInfo info{};
92 4 : if (rawUnits.empty())
93 : {
94 : // Return default "mm" units instead of error to allow client to handle gracefully
95 : // This prevents FAILED_PRECONDITION errors for component layers (comp_+_top, comp_+_bot)
96 : // that may lack units metadata in ODB++ files
97 2 : info.units = "mm";
98 2 : info.unitsToMm = 1.0;
99 2 : info.ok = true;
100 2 : return info;
101 : }
102 :
103 2 : auto u = to_lower_copy(rawUnits);
104 2 : if (u == "inch" || u == "inches" || u == "in")
105 : {
106 2 : info.units = "inch";
107 2 : info.unitsToMm = 25.4;
108 2 : info.ok = true;
109 : }
110 0 : else if (u == "mm" || u == "millimeter" || u == "millimeters")
111 : {
112 0 : info.units = "mm";
113 0 : info.unitsToMm = 1.0;
114 0 : info.ok = true;
115 : }
116 0 : else if (u == "mil" || u == "mils")
117 : {
118 0 : info.units = "mil";
119 0 : info.unitsToMm = 0.0254;
120 0 : info.ok = true;
121 : }
122 0 : else if (u == "micron" || u == "microns" || u == "um" || u == "µm")
123 : {
124 0 : info.units = "micron";
125 0 : info.unitsToMm = 0.001;
126 0 : info.ok = true;
127 : }
128 : else
129 : {
130 0 : info.error = "unsupported units: " + rawUnits;
131 : }
132 :
133 2 : return info;
134 2 : }
135 :
136 4 : Odb::Lib::UnitType expected_symbol_unit_type_from_normalized_units(const std::string& normalizedUnits)
137 : {
138 : // Contract requires symbol numeric parameters to be in same unit system as feature coordinates.
139 4 : auto u = to_lower_copy(normalizedUnits);
140 4 : if (u == "mm" || u == "micron")
141 : {
142 2 : return Odb::Lib::UnitType::Metric;
143 : }
144 2 : if (u == "inch" || u == "mil")
145 : {
146 2 : return Odb::Lib::UnitType::Imperial;
147 : }
148 0 : return Odb::Lib::UnitType::None;
149 4 : }
150 :
151 : // M6: spool cache construction — a configured directory wins; the
152 : // default is a private per-instance directory under the system temp
153 : // dir (isolated from parallel server/test instances, removed on exit).
154 58 : std::unique_ptr<StreamSpoolCache> MakeStreamSpoolCache(const Config::GrpcServiceConfig& config)
155 : {
156 : // stream_spool_max_bytes applies in BOTH branches: the
157 : // private-directory cache previously hardcoded 4 GiB, so
158 : // changing spool_max_bytes without setting spool_dir —
159 : // including the documented unlimited sentinel — silently did
160 : // nothing.
161 58 : if (!config.stream_spool_dir.empty())
162 : {
163 : return std::make_unique<StreamSpoolCache>(
164 2 : std::filesystem::path(config.stream_spool_dir), config.stream_spool_max_bytes);
165 : }
166 56 : return std::make_unique<StreamSpoolCache>(config.stream_spool_max_bytes);
167 : }
168 :
169 : // F9 (PR 609 review): a GetDesignStream burst leaves ~0.4-1.0 GiB
170 : // of FREED-but-retained glibc arena pages mapped. Measured on
171 : // 350-41017 (DesignStreamMemoryTests heap diagnostics): live
172 : // in-use bytes are FLAT across a full 1.34 GB warm replay — the
173 : // reader does not buffer — while free-but-held arena bytes grow
174 : // ~0.36 GiB per replay, and malloc_trim(0) collapses RSS by
175 : // ~0.87 GiB with zero live-byte change. glibc keeps those pages
176 : // for reuse; on the 10 GiB pod the retention eats serve headroom
177 : // for nothing. Return the pages when the burst ends (all exits,
178 : // via RAII; POSIX/glibc-only, no-op elsewhere).
179 : class StreamHeapTrimGuard
180 : {
181 : public:
182 19 : ~StreamHeapTrimGuard()
183 : {
184 : #if defined(__GLIBC__)
185 19 : malloc_trim(0);
186 : #endif
187 19 : }
188 : };
189 :
190 : }
191 :
192 :
193 58 : OdbDesignServiceImpl::OdbDesignServiceImpl(std::shared_ptr<Odb::Lib::App::DesignCache> cache,
194 58 : std::shared_ptr<Config::GrpcServiceConfig> config)
195 58 : : m_designCache(std::move(cache)),
196 58 : m_config(config ? config : Config::GrpcServiceConfig::CreateDefault()),
197 116 : m_streamSpool(MakeStreamSpoolCache(*m_config))
198 : {
199 58 : m_streamOptions.featureBatchSize = static_cast<std::size_t>(
200 58 : m_config->design_stream_feature_batch_size);
201 58 : }
202 :
203 15 : grpc::Status OdbDesignServiceImpl::GetDesign(
204 : grpc::ServerContext *context,
205 : const Odb::Grpc::GetDesignRequest *request,
206 : Odb::Lib::Protobuf::ProductModel::Design *response)
207 : {
208 : try
209 : {
210 15 : const auto& designName = request->design_name();
211 : // Normalized collections (nets/components/packages/parts + ByName maps)
212 : // are omitted unless explicitly requested: they duplicate fileModel
213 : // data clients actually read and dominate the payload.
214 15 : const auto includeNormalizedLists = request->include_normalized_lists();
215 30 : loginfo("[ConnTrace] GetDesign start: design_name=\"" + designName +
216 : "\" include_normalized_lists=" + (includeNormalizedLists ? "true" : "false") + "\"");
217 :
218 : // M1.4 fast path: when the serialized-response cache is warm,
219 : // fill the response from cached wire bytes with a single
220 : // parse — zero ProductModel-to-proto conversions. (The sync
221 : // gRPC API serializes `response` itself after this handler
222 : // returns, so raw bytes cannot be handed to the framework
223 : // directly; parsing the cached buffer replaces the full tree
224 : // build + deep copy and is still a strict win.)
225 15 : std::string cachedBytes;
226 15 : if (!includeNormalizedLists && m_designCache->TryGetDesignBytes(designName, cachedBytes))
227 : {
228 : // Budget enforced on the warm path too: the payload was
229 : // serialized under the budget active at pre-serialization
230 : // time; if the budget has since been lowered, parsing
231 : // these bytes would allocate the exact protobuf tree the
232 : // guard exists to prevent. Skip to the cold path, whose
233 : // guard rejects over-budget designs.
234 1 : const auto warmBudget = m_designCache->getMonolithicWireBudget();
235 1 : if (warmBudget != 0 && cachedBytes.size() > warmBudget)
236 : {
237 3 : loginfo("[ConnTrace] GetDesign cached payload over current budget: design_name=\"" +
238 : designName + "\" cachedBytes=" + std::to_string(cachedBytes.size()) +
239 : " budget=" + std::to_string(warmBudget) + "; using cold path");
240 : }
241 0 : else if (response->ParseFromString(cachedBytes))
242 : {
243 0 : loginfo("[ConnTrace] GetDesign ok (cached bytes): design_name=\"" + designName +
244 : "\" approx_bytes=" + std::to_string(response->ByteSizeLong()));
245 0 : return grpc::Status::OK;
246 : }
247 : else
248 : {
249 : // Corrupt cached bytes: never serve an empty/partial
250 : // message as OK. Log, clear the partially-merged message,
251 : // and fall through to the cold path below (the poisoned
252 : // payload is dropped by the next invalidation).
253 0 : logerror("Cached design bytes failed to parse for \"" + designName +
254 : "\"; falling back to full load");
255 0 : response->Clear();
256 : }
257 : }
258 :
259 15 : const auto design = m_designCache->GetDesign(designName);
260 14 : if (design == nullptr)
261 : {
262 0 : return {grpc::StatusCode::NOT_FOUND, "Design not found: " + designName};
263 : }
264 :
265 : // M6 hard guard: a design whose estimated serialized size
266 : // exceeds the monolithic serve budget must not have its
267 : // protobuf tree built here — that build (plus the serialized
268 : // copy) is the 10 GiB-pod OOM. Clean error instead of a pod
269 : // kill; the chunked GetDesignStream RPC serves these designs.
270 : // (Cached bytes served above are for designs pre-serialized
271 : // under this same budget, so they cannot be over it.)
272 14 : const auto monolithicBudget = m_designCache->getMonolithicWireBudget();
273 27 : if (monolithicBudget != 0 &&
274 13 : design->EstimateWireSizeBytes(includeNormalizedLists) > monolithicBudget)
275 : {
276 : return {grpc::StatusCode::RESOURCE_EXHAUSTED,
277 4 : "Design \"" + designName + "\" exceeds the monolithic GetDesign serve budget (" +
278 6 : std::to_string(monolithicBudget) + " estimated wire bytes); use the GetDesignStream RPC"};
279 : }
280 :
281 : // Cold path: serialize directly for this one response (the
282 : // status-quo cost — exactly one tree build, on this thread).
283 : // Cache population happens ONLY on the background
284 : // pre-serialization worker scheduled at load completion —
285 : // populating here as well meant two threads deep-building
286 : // the full protobuf tree concurrently, which thrashed memory
287 : // on large designs. The worker may land while we convert;
288 : // either way the next request takes the cached-bytes path.
289 : //
290 : // include_normalized_lists responses are always served cold (the
291 : // pre-serialization worker caches only the default pruned flavor),
292 : // so opting in pays the full tree build per request.
293 12 : *response = *(design->to_protobuf(includeNormalizedLists));
294 :
295 24 : loginfo("[ConnTrace] GetDesign ok: design_name=\"" + request->design_name() +
296 : "\" approx_bytes=" + std::to_string(response->ByteSizeLong()));
297 :
298 : // Log component height data in gRPC response for traced components
299 12 : if (response->has_filemodel())
300 : {
301 12 : const auto& fileModel = response->filemodel();
302 24 : for (const auto& kvStep : fileModel.stepsbyname())
303 : {
304 12 : const auto& step = kvStep.second;
305 324 : for (const auto& kvLayer : step.layersbyname())
306 : {
307 312 : const auto& layer = kvLayer.second;
308 312 : if (layer.has_components())
309 : {
310 312 : const auto& componentsFile = layer.components();
311 312 : std::vector<std::string> attributeNames;
312 360 : for (const auto& attrName : componentsFile.attributenames())
313 : {
314 48 : attributeNames.push_back(attrName);
315 : }
316 :
317 10068 : for (const auto& compRecord : componentsFile.componentrecords())
318 : {
319 9756 : std::map<std::string, std::string> lookupTable;
320 29004 : for (const auto& kv : compRecord.attributelookuptable())
321 : {
322 19248 : lookupTable[kv.first] = kv.second;
323 : }
324 :
325 9756 : Odb::Lib::FileModel::Design::ComponentHeightTracer::instance().logGrpcResponse(
326 : compRecord.compname(),
327 : compRecord.pkgref(),
328 : lookupTable,
329 : attributeNames);
330 9756 : }
331 312 : }
332 : }
333 : }
334 : }
335 :
336 12 : return grpc::Status::OK;
337 15 : }
338 1 : catch (const std::exception &e)
339 : {
340 1 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
341 1 : return {grpc::StatusCode::INTERNAL, error};
342 1 : }
343 : }
344 :
345 19 : grpc::Status OdbDesignServiceImpl::GetDesignStream(
346 : grpc::ServerContext* context,
347 : const Odb::Grpc::GetDesignRequest* request,
348 : grpc::ServerWriter<Odb::Grpc::DesignChunk>* writer)
349 : {
350 : try
351 : {
352 : // Return the burst's freed arena pages on every exit (see
353 : // StreamHeapTrimGuard — F9 root-cause fix).
354 : const StreamHeapTrimGuard heapTrimOnExit;
355 :
356 19 : const auto& designName = request->design_name();
357 19 : const auto includeNormalizedLists = request->include_normalized_lists();
358 38 : loginfo("[ConnTrace] GetDesignStream start: design_name=\"" + designName +
359 : "\" include_normalized_lists=" + (includeNormalizedLists ? "true" : "false"));
360 :
361 19 : if (!m_config->enable_design_streaming)
362 : {
363 2 : return {grpc::StatusCode::UNIMPLEMENTED, "Design streaming is not enabled"};
364 : }
365 :
366 : // Read the response generation BEFORE fetching the design:
367 : // an invalidation landing between the fetch and the
368 : // generation read would otherwise hand the cold fill a
369 : // stale Design published under the NEW generation — the
370 : // commit-time re-check below compares against
371 : // generationAtStart, so it cannot catch a bump that predates
372 : // its read. Reading first makes any invalidation in the
373 : // window abort the publish: the safe direction (the
374 : // streaming client still gets its content; the next request
375 : // refills).
376 18 : const auto generationAtStart = m_designCache->GetResponseGeneration(designName);
377 :
378 : // Resolve the design (parse + product-model build; the parse
379 : // fits the pod budget by design — it is the monolithic
380 : // serialization this RPC replaces that does not).
381 18 : const auto design = m_designCache->GetDesign(designName);
382 18 : if (design == nullptr)
383 : {
384 1 : return {grpc::StatusCode::NOT_FOUND, "Design not found: " + designName};
385 : }
386 :
387 : // Warm spool (default flavour only — the normalized flavour is
388 : // never cached, mirroring the monolithic response cache).
389 : // Identity is (design name, response-cache generation); a
390 : // generation bump (re-upload/invalidation/eviction) drops it.
391 17 : if (!includeNormalizedLists)
392 : {
393 16 : const auto generation = m_designCache->GetResponseGeneration(designName);
394 16 : StreamSpoolCache::Reader reader;
395 16 : if (m_streamSpool->TryOpenReader(designName, generation, reader))
396 : {
397 5 : Odb::Grpc::DesignChunk chunk;
398 2137 : while (reader.Next(chunk))
399 : {
400 2132 : if (context->IsCancelled())
401 : {
402 0 : loginfo("[ConnTrace] GetDesignStream client_disconnected (warm): design_name=\"" +
403 : designName + "\" records=" + std::to_string(reader.RecordsRead()));
404 0 : return grpc::Status::OK;
405 : }
406 2132 : if (!writer->Write(chunk))
407 : {
408 : // Truncation without a cancelling client must
409 : // not end in a clean OK trailer — chunk count
410 : // is the only completion signal M6.4 clients
411 : // have (see the cold path below).
412 0 : logerror("[ConnTrace] GetDesignStream write failed (warm): design_name=\"" +
413 : designName + "\" records=" + std::to_string(reader.RecordsRead()));
414 : return {grpc::StatusCode::INTERNAL,
415 0 : "Internal server error: GetDesignStream write failed (warm, truncated)"};
416 : }
417 2132 : chunk.Clear();
418 : }
419 5 : if (reader.Failed() || reader.Truncated())
420 : {
421 : // Read error, OR a clean EOF short of the committed
422 : // record count (truncation at a record boundary /
423 : // emptied file — framing alone cannot distinguish
424 : // that from a legitimate end-of-spool): either way
425 : // the client would get a clean trailer after a
426 : // partial chunk sequence. INTERNAL, and drop the
427 : // damaged spool.
428 6 : logerror("[ConnTrace] GetDesignStream spool read failed: design_name=\"" + designName +
429 : "\" failed=" + (reader.Failed() ? "true" : "false") +
430 : " recordsRead=" + std::to_string(reader.RecordsRead()) +
431 : " recordsExpected=" + std::to_string(reader.ExpectedRecords()));
432 3 : m_streamSpool->Drop(designName);
433 : return {grpc::StatusCode::INTERNAL,
434 6 : "Internal server error: cached stream spool read failure (truncated)"};
435 : }
436 4 : loginfo("[ConnTrace] GetDesignStream ok (warm spool): design_name=\"" + designName +
437 : "\" records=" + std::to_string(reader.RecordsRead()));
438 2 : return grpc::Status::OK;
439 5 : }
440 16 : }
441 :
442 : // Cold fill: walk the product model chunk-by-chunk. Over-budget
443 : // designs (the ones this RPC exists for) tee every chunk to the
444 : // disk spool while writing it to the wire, so the NEXT request
445 : // is served warm from page cache with no product-model walk.
446 12 : const auto monolithicBudget = m_designCache->getMonolithicWireBudget();
447 24 : const bool overBudget = monolithicBudget != 0 &&
448 12 : design->EstimateWireSizeBytes(/*includeNormalizedLists=*/false) > monolithicBudget;
449 :
450 : // Spool outcome for the done-log, tracked at each transition
451 : // so the label reports what actually happened. The previous
452 : // expression printed "spool=aborted" for normalized-flavour
453 : // requests (no spool write is ever attempted for them) and
454 : // "spool=committed" for fills that aborted on client
455 : // cancellation or a generation bump.
456 : enum class SpoolOutcome { NoneAttempted, OpenFailed, Aborted, Committed };
457 12 : auto spoolOutcome = SpoolOutcome::NoneAttempted;
458 :
459 12 : StreamSpoolCache::Writer spoolWriter;
460 12 : bool spooling = false;
461 12 : if (!includeNormalizedLists && overBudget)
462 : {
463 10 : spooling = m_streamSpool->BeginWrite(designName, generationAtStart, spoolWriter);
464 : // An active fill publishes only on full success; Aborted
465 : // is the default until Commit() says otherwise.
466 10 : spoolOutcome = spooling ? SpoolOutcome::Aborted : SpoolOutcome::OpenFailed;
467 : }
468 :
469 12 : DesignStreamWalker::Stats stats;
470 9758 : const auto sink = [&](const Odb::Grpc::DesignChunk& chunk) -> bool
471 : {
472 9758 : if (context->IsCancelled())
473 : {
474 1 : return false;
475 : }
476 9757 : if (!writer->Write(chunk))
477 : {
478 0 : return false;
479 : }
480 9757 : if (spooling && !spoolWriter.Append(chunk))
481 : {
482 : // The wire response is unaffected by spool I/O trouble
483 : // (disk full, etc.): drop the spool, keep streaming.
484 0 : logerror("[ConnTrace] GetDesignStream spool append failed; continuing without spool");
485 0 : spoolWriter.Abort();
486 0 : spooling = false;
487 : }
488 9757 : return true;
489 12 : };
490 :
491 24 : const bool completed = DesignStreamWalker::Walk(
492 12 : *design, includeNormalizedLists, m_streamOptions, sink, &stats);
493 :
494 12 : if (spooling)
495 : {
496 : // Publish only a complete, still-current fill: an aborted
497 : // walk (client gone) or an invalidation that raced the fill
498 : // (generation bumped) must not poison the warm path.
499 : const bool generationUnchanged =
500 9 : m_designCache->GetResponseGeneration(designName) == generationAtStart;
501 9 : if (completed && generationUnchanged)
502 : {
503 : // Commit() reports whether the spool was actually
504 : // published: a flush/close failure (ENOSPC surfacing
505 : // at flush time) deletes the incomplete file instead.
506 16 : spoolOutcome = spoolWriter.Commit()
507 8 : ? SpoolOutcome::Committed
508 : : SpoolOutcome::Aborted;
509 : }
510 : else
511 : {
512 1 : spoolWriter.Abort();
513 : }
514 : }
515 :
516 12 : const char* spoolLabel =
517 16 : spoolOutcome == SpoolOutcome::Committed ? " spool=committed" :
518 7 : spoolOutcome == SpoolOutcome::Aborted ? " spool=aborted" :
519 3 : spoolOutcome == SpoolOutcome::OpenFailed ? " spool=open-failed" : "";
520 24 : loginfo("[ConnTrace] GetDesignStream done: design_name=\"" + designName +
521 : "\" completed=" + (completed ? "true" : "false") +
522 : " chunks=" + std::to_string(stats.chunks) +
523 : " wire_bytes=" + std::to_string(stats.wireBytes) +
524 : spoolLabel);
525 :
526 12 : if (!completed && !context->IsCancelled())
527 : {
528 : // The sink aborted for a reason other than client
529 : // cancellation (writer->Write failed — e.g. flow control
530 : // or a broken connection without a propagated cancel).
531 : // Returning OK here would hand the client a clean trailer
532 : // after a truncated chunk sequence, and the chunk count is
533 : // the only completion signal M6.4 clients have.
534 0 : logerror("[ConnTrace] GetDesignStream write failed (cold): design_name=\"" + designName +
535 : "\" chunks=" + std::to_string(stats.chunks));
536 : return {grpc::StatusCode::INTERNAL,
537 0 : "Internal server error: GetDesignStream write failed (cold, truncated)"};
538 : }
539 12 : return grpc::Status::OK;
540 19 : }
541 0 : catch (const std::exception& e)
542 : {
543 0 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
544 0 : return {grpc::StatusCode::INTERNAL, error};
545 0 : }
546 : }
547 :
548 0 : grpc::Status OdbDesignServiceImpl::GetLayerFeaturesStream(
549 : grpc::ServerContext *context,
550 : const Odb::Grpc::GetLayerFeaturesRequest *request,
551 : grpc::ServerWriter<Odb::Lib::Protobuf::FeaturesFile::FeatureRecord> *writer)
552 : {
553 : try
554 : {
555 0 : loginfo("[ConnTrace] GetLayerFeaturesStream start: design=\"" + request->design_name() +
556 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
557 0 : const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
558 0 : if (fileArchive == nullptr)
559 : {
560 0 : return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
561 : }
562 :
563 0 : const auto pStep = fileArchive->GetStepDirectory(request->step_name());
564 0 : if (pStep == nullptr)
565 : {
566 0 : return {grpc::StatusCode::NOT_FOUND, "Step not found: " + request->step_name()};
567 : }
568 :
569 0 : const auto pLayersByName = pStep->GetLayersByName();
570 0 : const auto layerIt = pLayersByName.find(request->layer_name());
571 0 : if (layerIt == pLayersByName.end())
572 : {
573 0 : return {grpc::StatusCode::NOT_FOUND, "Layer not found: " + request->layer_name()};
574 : }
575 :
576 0 : const auto &featuresFile = layerIt->second->GetFeaturesFile();
577 0 : const auto &featureRecords = featuresFile.GetFeatureRecords();
578 :
579 : // Reuse a single protobuf message instance to avoid per-iteration heap allocations.
580 0 : Odb::Lib::Protobuf::FeaturesFile::FeatureRecord featureRecordMsg;
581 :
582 0 : size_t sentCount = 0;
583 0 : for (const auto &featureRecord : featureRecords)
584 : {
585 0 : if (!featureRecord)
586 0 : continue;
587 :
588 : // Populate reused message: obtain per-feature proto and swap into reusable instance.
589 0 : auto pFeatureRecordMsg = featureRecord->to_protobuf();
590 0 : if (pFeatureRecordMsg != nullptr)
591 : {
592 0 : featureRecordMsg.Swap(pFeatureRecordMsg.get()); // Efficient move of fields.
593 : }
594 : else
595 : {
596 0 : continue; // Skip if conversion failed or returned nullptr.
597 : }
598 :
599 0 : if (!writer->Write(featureRecordMsg))
600 : {
601 0 : loginfo("[ConnTrace] GetLayerFeaturesStream client_disconnected: design=\"" + request->design_name() +
602 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
603 : "\" sent=" + std::to_string(sentCount));
604 0 : break;
605 : }
606 :
607 0 : ++sentCount;
608 :
609 : // Clear for reuse while retaining allocated capacity.
610 0 : featureRecordMsg.Clear();
611 0 : }
612 :
613 0 : loginfo("[ConnTrace] GetLayerFeaturesStream done: design=\"" + request->design_name() +
614 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
615 : "\" sent=" + std::to_string(sentCount));
616 0 : return grpc::Status::OK;
617 0 : }
618 0 : catch (const std::exception &e)
619 : {
620 0 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
621 0 : return {grpc::StatusCode::INTERNAL, error};
622 0 : }
623 : }
624 :
625 0 : grpc::Status OdbDesignServiceImpl::GetLayerFeaturesBatchStream(
626 : grpc::ServerContext *context,
627 : const Odb::Grpc::GetLayerFeaturesRequest *request,
628 : grpc::ServerWriter<Odb::Grpc::FeatureRecordBatch> *writer)
629 : {
630 : try
631 : {
632 0 : loginfo("[ConnTrace] GetLayerFeaturesBatchStream start: design=\"" + request->design_name() +
633 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
634 : // Feature flag check: return UNIMPLEMENTED if batch streaming is disabled
635 0 : if (!m_config->enable_batch_streaming)
636 : {
637 0 : loginfo("[ConnTrace] GetLayerFeaturesBatchStream disabled: returning UNIMPLEMENTED");
638 0 : return {grpc::StatusCode::UNIMPLEMENTED, "Batch streaming is not enabled"};
639 : }
640 :
641 : // Reuse validation logic from GetLayerFeaturesStream
642 0 : const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
643 0 : if (fileArchive == nullptr)
644 : {
645 0 : return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
646 : }
647 :
648 0 : const auto pStep = fileArchive->GetStepDirectory(request->step_name());
649 0 : if (pStep == nullptr)
650 : {
651 0 : return {grpc::StatusCode::NOT_FOUND, "Step not found: " + request->step_name()};
652 : }
653 :
654 0 : const auto pLayersByName = pStep->GetLayersByName();
655 0 : const auto layerIt = pLayersByName.find(request->layer_name());
656 0 : if (layerIt == pLayersByName.end())
657 : {
658 0 : return {grpc::StatusCode::NOT_FOUND, "Layer not found: " + request->layer_name()};
659 : }
660 :
661 0 : const auto &featuresFile = layerIt->second->GetFeaturesFile();
662 0 : const auto &featureRecords = featuresFile.GetFeatureRecords();
663 :
664 : // Reuse a single protobuf message instance to avoid per-iteration heap allocations.
665 0 : Odb::Lib::Protobuf::FeaturesFile::FeatureRecord featureRecordMsg;
666 0 : Odb::Grpc::FeatureRecordBatch batch;
667 :
668 0 : const int batchSize = m_config->batch_size;
669 0 : int currentBatchCount = 0;
670 0 : size_t batchesSent = 0;
671 0 : size_t featuresSent = 0;
672 :
673 0 : for (const auto &featureRecord : featureRecords)
674 : {
675 : // Check for cancellation
676 0 : if (context->IsCancelled())
677 : {
678 0 : return {grpc::StatusCode::CANCELLED, "Request cancelled"};
679 : }
680 :
681 0 : if (!featureRecord)
682 0 : continue;
683 :
684 : // Populate reused message: obtain per-feature proto and swap into reusable instance.
685 0 : auto pFeatureRecordMsg = featureRecord->to_protobuf();
686 0 : if (pFeatureRecordMsg != nullptr)
687 : {
688 0 : featureRecordMsg.Swap(pFeatureRecordMsg.get()); // Efficient move of fields.
689 : }
690 : else
691 : {
692 0 : continue; // Skip if conversion failed or returned nullptr.
693 : }
694 :
695 : // Add feature to current batch
696 0 : batch.add_features()->CopyFrom(featureRecordMsg);
697 0 : currentBatchCount++;
698 0 : featuresSent++;
699 :
700 : // Clear for reuse while retaining allocated capacity.
701 0 : featureRecordMsg.Clear();
702 :
703 : // Send batch when it reaches the configured size
704 0 : if (currentBatchCount >= batchSize)
705 : {
706 : // Check for cancellation before writing
707 0 : if (context->IsCancelled())
708 : {
709 0 : return {grpc::StatusCode::CANCELLED, "Request cancelled"};
710 : }
711 :
712 0 : if (!writer->Write(batch))
713 : {
714 0 : loginfo("[ConnTrace] GetLayerFeaturesBatchStream client_disconnected: design=\"" + request->design_name() +
715 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
716 : "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent));
717 0 : return grpc::Status::OK;
718 : }
719 0 : batchesSent++;
720 0 : batch.Clear();
721 0 : currentBatchCount = 0;
722 : }
723 0 : }
724 :
725 : // Send final batch if there are remaining features
726 0 : if (currentBatchCount > 0)
727 : {
728 : // Check for cancellation before writing final batch
729 0 : if (context->IsCancelled())
730 : {
731 0 : return {grpc::StatusCode::CANCELLED, "Request cancelled"};
732 : }
733 :
734 0 : if (!writer->Write(batch))
735 : {
736 0 : loginfo("[ConnTrace] GetLayerFeaturesBatchStream client_disconnected(final): design=\"" + request->design_name() +
737 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
738 : "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent));
739 0 : return grpc::Status::OK;
740 : }
741 0 : batchesSent++;
742 : }
743 :
744 : // Stream completion is indicated by normal gRPC end-of-stream semantics
745 0 : loginfo("[ConnTrace] GetLayerFeaturesBatchStream done: design=\"" + request->design_name() +
746 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
747 : "\" batches=" + std::to_string(batchesSent) + " features=" + std::to_string(featuresSent) +
748 : " batch_size=" + std::to_string(batchSize));
749 0 : return grpc::Status::OK;
750 0 : }
751 0 : catch (const std::exception &e)
752 : {
753 0 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
754 0 : return {grpc::StatusCode::INTERNAL, error};
755 0 : }
756 : }
757 :
758 7 : grpc::Status OdbDesignServiceImpl::GetLayerSymbols(
759 : grpc::ServerContext* context,
760 : const Odb::Grpc::GetLayerSymbolsRequest* request,
761 : Odb::Grpc::GetLayerSymbolsResponse* response)
762 : {
763 : try
764 : {
765 14 : loginfo("[ConnTrace] GetLayerSymbols start: design=\"" + request->design_name() +
766 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() + "\"");
767 7 : const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
768 7 : if (fileArchive == nullptr)
769 : {
770 2 : return { grpc::StatusCode::NOT_FOUND, "Design not found" };
771 : }
772 :
773 6 : const auto pStep = fileArchive->GetStepDirectory(request->step_name());
774 6 : if (pStep == nullptr)
775 : {
776 2 : return { grpc::StatusCode::NOT_FOUND, "Step not found" };
777 : }
778 :
779 5 : const auto& layersByName = pStep->GetLayersByName();
780 5 : const auto layerIt = layersByName.find(request->layer_name());
781 5 : if (layerIt == layersByName.end())
782 : {
783 2 : return { grpc::StatusCode::NOT_FOUND, "Layer not found" };
784 : }
785 :
786 4 : const auto& featuresFile = layerIt->second->GetFeaturesFile();
787 4 : auto unitsInfo = normalize_units(featuresFile.GetUnits());
788 4 : if (!unitsInfo.ok)
789 : {
790 0 : return { grpc::StatusCode::FAILED_PRECONDITION, unitsInfo.error };
791 : }
792 :
793 4 : response->set_sym_num_base(0);
794 : response->set_units(unitsInfo.units);
795 4 : response->set_units_to_mm(unitsInfo.unitsToMm);
796 :
797 8 : loginfo("[ConnTrace] GetLayerSymbols units: layer=\"" + request->layer_name() + "\" units=\"" +
798 : unitsInfo.units + "\" units_to_mm=" + std::to_string(unitsInfo.unitsToMm));
799 :
800 4 : const auto symbols = Odb::Lib::FileModel::Design::collect_symbols(featuresFile);
801 4 : const auto& featureRecords = featuresFile.GetFeatureRecords();
802 :
803 : // Strict contract enforcement (S2): symbol unit types must match the layer units.
804 4 : const auto expectedSymbolUnitType = expected_symbol_unit_type_from_normalized_units(unitsInfo.units);
805 4 : int metricCount = 0;
806 4 : int imperialCount = 0;
807 4 : int noneCount = 0;
808 230 : for (const auto& sym : symbols)
809 : {
810 226 : if (!sym) continue;
811 226 : switch (sym->GetUnitType())
812 : {
813 0 : case Odb::Lib::UnitType::Metric: metricCount++; break;
814 226 : case Odb::Lib::UnitType::Imperial: imperialCount++; break;
815 0 : case Odb::Lib::UnitType::None: noneCount++; break;
816 : }
817 : }
818 :
819 4 : const bool hasMixedSymbolUnitTypes = metricCount > 0 && imperialCount > 0;
820 : // Spec: if the symbol record omits its unit, it inherits the enclosing features file unit system.
821 : // So UnitType::None is treated as compatible with the layer units.
822 4 : const bool hasMismatchVsLayerUnits =
823 6 : (expectedSymbolUnitType == Odb::Lib::UnitType::Metric && imperialCount > 0) ||
824 2 : (expectedSymbolUnitType == Odb::Lib::UnitType::Imperial && metricCount > 0);
825 :
826 4 : if (hasMixedSymbolUnitTypes || hasMismatchVsLayerUnits)
827 : {
828 : std::string msg =
829 0 : "symbol UnitType mismatch vs layer units (strict): layer_units=" + unitsInfo.units +
830 0 : ", symbol_counts(metric=" + std::to_string(metricCount) +
831 0 : ", imperial=" + std::to_string(imperialCount) +
832 0 : ", none=" + std::to_string(noneCount) + ")";
833 0 : return { grpc::StatusCode::FAILED_PRECONDITION, msg };
834 0 : }
835 :
836 : // Sanity check: limit symbol indices to avoid excessive memory allocation or logic errors due to invalid/corrupted data.
837 : // 10,000,000 is chosen as a very high upper bound, far exceeding any realistic symbol count in production designs.
838 : // If a valid symbol index exceeds this limit, it will be ignored, which may cause rendering or data issues.
839 : // Adjust this value with caution and ensure all code using symbol indices is reviewed if changed.
840 4 : constexpr int kMaxReasonableIndex = 10'000'000;
841 4 : int maxSymRef = -1;
842 : // Single pass to validate pad contracts and capture the max referenced symbol id without extra traversals.
843 32018 : for (const auto& fr : featureRecords)
844 : {
845 32014 : if (!fr) continue;
846 :
847 : // Strict contract enforcement (S1): Pads must reference symbols via FeatureRecord.sym_num.
848 32014 : if (fr->type == Odb::Lib::FileModel::Design::FeaturesFile::FeatureRecord::Type::Pad &&
849 32014 : fr->apt_def_symbol_num >= 0 && fr->sym_num != fr->apt_def_symbol_num)
850 : {
851 0 : return { grpc::StatusCode::FAILED_PRECONDITION, "pad sym_num does not match apt_def_symbol_num (contract violation)" };
852 : }
853 :
854 32014 : if (fr->sym_num >= 0 && fr->sym_num < kMaxReasonableIndex)
855 : {
856 28804 : maxSymRef = std::max(maxSymRef, fr->sym_num);
857 : }
858 32014 : if (fr->apt_def_symbol_num >= 0 && fr->apt_def_symbol_num < kMaxReasonableIndex)
859 : {
860 12848 : maxSymRef = std::max(maxSymRef, fr->apt_def_symbol_num);
861 : }
862 : }
863 :
864 4 : int resultSize = maxSymRef + 1;
865 4 : if (resultSize < 0) resultSize = 0;
866 :
867 8 : std::vector<std::shared_ptr<Odb::Lib::FileModel::Design::SymbolName>> ordered(static_cast<size_t>(resultSize), nullptr);
868 :
869 : // Ordering strategy:
870 : // 1) Place symbols that declare explicit indices; keep the first occurrence on duplicates.
871 : // 2) Fill remaining gaps with symbols lacking indices in deterministic input order.
872 : // This preserves stable ordering for callers that rely on sym_num alignment.
873 4 : int maxIndex = -1;
874 230 : for (const auto& sym : symbols)
875 : {
876 226 : if (!sym) continue;
877 226 : const auto idx = sym->GetIndex();
878 226 : maxIndex = std::max(maxIndex, idx);
879 226 : if (idx < 0) continue;
880 226 : const auto idxSize = static_cast<size_t>(idx);
881 226 : if (idx >= static_cast<int>(ordered.size()))
882 : {
883 0 : ordered.resize(idxSize + 1, nullptr);
884 : }
885 226 : if (!ordered[idxSize])
886 : {
887 226 : ordered[idxSize] = sym;
888 : }
889 : }
890 :
891 : // Then, fill remaining slots with symbols lacking indices in deterministic order.
892 : // Precompute available slots for O(n) filling instead of O(n²) linear search.
893 8 : std::vector<size_t> availableSlots;
894 230 : for (size_t i = 0; i < ordered.size(); ++i)
895 : {
896 226 : if (!ordered[i])
897 : {
898 0 : availableSlots.push_back(i);
899 : }
900 : }
901 4 : size_t slotIdx = 0;
902 230 : for (const auto& sym : symbols)
903 : {
904 226 : if (!sym) continue;
905 226 : if (sym->GetIndex() >= 0) continue;
906 :
907 0 : if (slotIdx < availableSlots.size())
908 : {
909 0 : ordered[availableSlots[slotIdx]] = sym;
910 0 : ++slotIdx;
911 : }
912 : else
913 : {
914 0 : ordered.push_back(sym);
915 : }
916 : }
917 :
918 : // Pad to cover all referenced sym_num values.
919 4 : const int requiredSize = std::max(maxSymRef, maxIndex) + 1;
920 4 : if (requiredSize > static_cast<int>(ordered.size()))
921 : {
922 0 : ordered.resize(static_cast<size_t>(requiredSize), nullptr);
923 : }
924 :
925 8 : Odb::Lib::Protobuf::SymbolName placeholder;
926 : placeholder.set_name("");
927 4 : placeholder.set_unittype(Odb::Lib::Protobuf::UnitType::None);
928 :
929 230 : for (const auto& sym : ordered)
930 : {
931 226 : if (sym)
932 : {
933 226 : auto pb = sym->to_protobuf();
934 226 : if (pb && pb->unittype() == Odb::Lib::Protobuf::UnitType::None)
935 : {
936 0 : pb->set_unittype(static_cast<Odb::Lib::Protobuf::UnitType>(expectedSymbolUnitType));
937 : }
938 226 : response->add_symbol_names()->CopyFrom(*pb);
939 226 : }
940 : else
941 : {
942 0 : response->add_symbol_names()->CopyFrom(placeholder);
943 : }
944 : }
945 :
946 8 : loginfo("[ConnTrace] GetLayerSymbols done: design=\"" + request->design_name() +
947 : "\" step=\"" + request->step_name() + "\" layer=\"" + request->layer_name() +
948 : "\" symbol_names=" + std::to_string(response->symbol_names_size()));
949 :
950 4 : return grpc::Status::OK;
951 7 : }
952 0 : catch (const std::exception& e)
953 : {
954 0 : std::string error = "Internal server error: " + std::string(e.what());
955 0 : return { grpc::StatusCode::INTERNAL, error };
956 0 : }
957 : }
958 :
959 5 : grpc::Status OdbDesignServiceImpl::GetStandardFonts(
960 : grpc::ServerContext *context,
961 : const Odb::Grpc::GetStandardFontsRequest *request,
962 : Odb::Lib::Protobuf::StandardFontsFile *response)
963 : {
964 : try
965 : {
966 10 : loginfo("[ConnTrace] GetStandardFonts start: design_name=\"" + request->design_name() + "\"");
967 :
968 5 : const auto fileArchive = m_designCache->GetFileArchive(request->design_name());
969 4 : if (fileArchive == nullptr)
970 : {
971 1 : return {grpc::StatusCode::NOT_FOUND, "Design not found: " + request->design_name()};
972 : }
973 :
974 3 : auto fontsFilePb = fileArchive->GetStandardFontsFile().to_protobuf();
975 3 : if (fontsFilePb == nullptr)
976 : {
977 0 : return {grpc::StatusCode::INTERNAL, "Failed to serialize standard fonts file"};
978 : }
979 3 : response->CopyFrom(*fontsFilePb);
980 :
981 6 : loginfo("[ConnTrace] GetStandardFonts done: design_name=\"" + request->design_name() +
982 : "\" characters=" + std::to_string(response->m_characterblocks_size()));
983 :
984 3 : return grpc::Status::OK;
985 4 : }
986 1 : catch (const std::exception &e)
987 : {
988 : // Log the full details server-side only: e.what() can embed
989 : // filesystem paths, which must not reach clients of a server
990 : // that runs with insecure credentials and no authentication.
991 5 : logerror(std::string("[ConnTrace] GetStandardFonts failed: ") + e.what());
992 1 : return {grpc::StatusCode::INTERNAL, "Internal server error"};
993 1 : }
994 : }
995 :
996 5 : grpc::Status OdbDesignServiceImpl::RequestLoadDesign(
997 : grpc::ServerContext *context,
998 : const Odb::Grpc::RequestLoadDesignRequest *request,
999 : Odb::Grpc::RequestLoadDesignResponse *response)
1000 : {
1001 : try
1002 : {
1003 10 : loginfo("[ConnTrace] RequestLoadDesign: design_name=\"" + request->design_name() + "\"");
1004 5 : response->set_design_name(request->design_name());
1005 5 : response->set_status(ComputeRequestLoadStatus(*m_designCache, request->design_name()));
1006 5 : return grpc::Status::OK;
1007 : }
1008 0 : catch (const std::exception &e)
1009 : {
1010 0 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
1011 0 : return {grpc::StatusCode::INTERNAL, error};
1012 0 : }
1013 : }
1014 :
1015 0 : grpc::Status OdbDesignServiceImpl::HealthCheck(
1016 : grpc::ServerContext *context,
1017 : const Odb::Grpc::HealthCheckRequest *request,
1018 : Odb::Grpc::HealthCheckResponse *response)
1019 : {
1020 : try
1021 : {
1022 : // Optionally, check the service name from the request
1023 0 : if (!request->service().empty() && request->service() != "OdbDesignService")
1024 : {
1025 0 : response->set_status(Odb::Grpc::HealthCheckResponse::NOT_SERVING);
1026 0 : return grpc::Status::OK;
1027 : }
1028 :
1029 0 : response->set_status(Odb::Grpc::HealthCheckResponse::SERVING);
1030 0 : return grpc::Status::OK;
1031 : }
1032 0 : catch (const std::exception &e)
1033 : {
1034 0 : std::string error = "Internal server error: " + FormatExceptionForStatus(e);
1035 0 : return {grpc::StatusCode::INTERNAL, error};
1036 0 : }
1037 : }
1038 :
1039 : } // namespace Services
1040 : } // namespace OdbDesignServer
|