Line data Source code
1 : #include "StreamSpoolCache.h"
2 : #include <Logger.h>
3 :
4 : #include <atomic>
5 : #include <chrono>
6 : #include <system_error>
7 :
8 : #if defined(_WIN32)
9 : #include <process.h>
10 : #else
11 : #include <unistd.h>
12 : #endif
13 :
14 : namespace OdbDesignServer
15 : {
16 : namespace Services
17 : {
18 : namespace
19 : {
20 : // 64-bit FNV-1a: collision-free-enough namespacing of spool file
21 : // names (not a security boundary — the design name may contain
22 : // filesystem-hostile characters).
23 12 : std::uint64_t fnv1a64(const std::string& s)
24 : {
25 12 : std::uint64_t hash = 14695981039346656037ull;
26 147 : for (const char c : s)
27 : {
28 135 : hash ^= static_cast<std::uint64_t>(static_cast<unsigned char>(c));
29 135 : hash *= 1099511628211ull;
30 : }
31 12 : return hash;
32 : }
33 :
34 : // Filesystem-safe projection of the design name (kept short; the
35 : // FNV suffix disambiguates).
36 12 : std::string sanitize_name(const std::string& designName)
37 : {
38 12 : std::string out;
39 12 : out.reserve(designName.size() < 48 ? designName.size() : 48);
40 147 : for (const char c : designName)
41 : {
42 135 : const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
43 270 : (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.';
44 135 : out.push_back(ok ? c : '_');
45 135 : if (out.size() >= 48) break;
46 : }
47 12 : return out;
48 0 : }
49 :
50 68 : unsigned long current_pid()
51 : {
52 : #if defined(_WIN32)
53 : return static_cast<unsigned long>(_getpid());
54 : #else
55 68 : return static_cast<unsigned long>(::getpid());
56 : #endif
57 : }
58 :
59 8607 : void append_varint(std::string& out, std::uint64_t value)
60 : {
61 22580 : while (value >= 0x80)
62 : {
63 13973 : out.push_back(static_cast<char>((value & 0x7f) | 0x80));
64 13973 : value >>= 7;
65 : }
66 8607 : out.push_back(static_cast<char>(value));
67 8607 : }
68 :
69 : enum class VarintResult
70 : {
71 : Ok,
72 : CleanEof, // no bytes consumed: legitimate end of the record stream
73 : Error // truncated varint or malformed (too long) encoding
74 : };
75 :
76 2638 : VarintResult read_varint(std::istream& in, std::uint64_t& valueOut)
77 : {
78 2638 : std::uint64_t value = 0;
79 2638 : int shift = 0;
80 6828 : for (int i = 0; i < 10; ++i)
81 : {
82 6828 : const int byte = in.get();
83 6828 : if (byte == std::char_traits<char>::eof())
84 : {
85 5 : return (i == 0) ? VarintResult::CleanEof : VarintResult::Error;
86 : }
87 6823 : value |= static_cast<std::uint64_t>(byte & 0x7f) << shift;
88 6823 : if ((byte & 0x80) == 0)
89 : {
90 2633 : valueOut = value;
91 2633 : return VarintResult::Ok;
92 : }
93 4190 : shift += 7;
94 : }
95 0 : return VarintResult::Error;
96 : }
97 :
98 : // Sanity cap for one record: chunk payloads are bounded by batch
99 : // sizes (single-digit MBs); refuse absurd lengths from a corrupt
100 : // file instead of allocating them.
101 : constexpr std::uint64_t kMaxRecordBytes = 512ull * 1024ull * 1024ull;
102 :
103 : // Process-global fill counter: the spool path embeds pid + fillId
104 : // so concurrent fills never share a file. pid uniquifies across
105 : // processes; the GLOBAL counter (not a per-instance member)
106 : // uniquifies same-process cache instances sharing an explicit
107 : // spool directory — two instances would otherwise both start at
108 : // fill 1 and truncate-open each other's path.
109 : std::atomic<std::uint64_t> g_fillCounter{ 0 };
110 : }
111 :
112 : // ---- Reader ----
113 :
114 2638 : bool StreamSpoolCache::Reader::Next(google::protobuf::Message& record)
115 : {
116 2638 : if (m_failed || !m_stream.is_open())
117 : {
118 0 : return false;
119 : }
120 :
121 2638 : std::uint64_t length = 0;
122 2638 : const auto varint = read_varint(m_stream, length);
123 2638 : if (varint == VarintResult::CleanEof)
124 : {
125 5 : return false;
126 : }
127 2633 : if (varint == VarintResult::Error || length == 0 || length > kMaxRecordBytes)
128 : {
129 0 : m_failed = true;
130 0 : return false;
131 : }
132 :
133 2633 : std::string bytes(static_cast<std::size_t>(length), '\0');
134 2633 : m_stream.read(bytes.data(), bytes.size());
135 2633 : if (static_cast<std::uint64_t>(m_stream.gcount()) != length)
136 : {
137 1 : m_failed = true;
138 1 : return false;
139 : }
140 2632 : if (!record.ParseFromString(bytes))
141 : {
142 0 : m_failed = true;
143 0 : return false;
144 : }
145 2632 : ++m_recordsRead;
146 2632 : return true;
147 2633 : }
148 :
149 : // ---- Writer ----
150 :
151 15 : StreamSpoolCache::Writer::~Writer()
152 : {
153 15 : if (m_openForWrite && !m_committed)
154 : {
155 0 : Close(/*commit=*/false);
156 : }
157 15 : }
158 :
159 0 : StreamSpoolCache::Writer::Writer(Writer&& other) noexcept
160 0 : : m_stream(std::move(other.m_stream)),
161 0 : m_path(std::move(other.m_path)),
162 0 : m_pCache(other.m_pCache),
163 0 : m_designName(std::move(other.m_designName)),
164 0 : m_generation(other.m_generation),
165 0 : m_bytesWritten(other.m_bytesWritten),
166 0 : m_recordsWritten(other.m_recordsWritten),
167 0 : m_openForWrite(other.m_openForWrite),
168 0 : m_committed(other.m_committed),
169 0 : m_publishSucceeded(other.m_publishSucceeded)
170 : {
171 0 : other.m_pCache = nullptr;
172 0 : other.m_openForWrite = false;
173 0 : other.m_committed = false;
174 0 : other.m_publishSucceeded = false;
175 0 : }
176 :
177 0 : StreamSpoolCache::Writer& StreamSpoolCache::Writer::operator=(Writer&& other) noexcept
178 : {
179 0 : if (this != &other)
180 : {
181 0 : if (m_openForWrite && !m_committed)
182 : {
183 0 : Close(/*commit=*/false);
184 : }
185 0 : m_stream = std::move(other.m_stream);
186 0 : m_path = std::move(other.m_path);
187 0 : m_pCache = other.m_pCache;
188 0 : m_designName = std::move(other.m_designName);
189 0 : m_generation = other.m_generation;
190 0 : m_bytesWritten = other.m_bytesWritten;
191 0 : m_recordsWritten = other.m_recordsWritten;
192 0 : m_openForWrite = other.m_openForWrite;
193 0 : m_committed = other.m_committed;
194 0 : m_publishSucceeded = other.m_publishSucceeded;
195 0 : other.m_pCache = nullptr;
196 0 : other.m_openForWrite = false;
197 0 : other.m_committed = false;
198 0 : other.m_publishSucceeded = false;
199 : }
200 0 : return *this;
201 : }
202 :
203 8607 : bool StreamSpoolCache::Writer::Append(const google::protobuf::Message& record)
204 : {
205 8607 : if (!m_openForWrite || !m_stream.is_open())
206 : {
207 0 : return false;
208 : }
209 :
210 8607 : std::string bytes;
211 8607 : if (!record.SerializeToString(&bytes))
212 : {
213 0 : return false;
214 : }
215 :
216 8607 : std::string framed;
217 8607 : framed.reserve(bytes.size() + 10);
218 8607 : append_varint(framed, bytes.size());
219 8607 : framed.append(bytes);
220 :
221 8607 : m_stream.write(framed.data(), framed.size());
222 8607 : if (!m_stream.good())
223 : {
224 0 : return false;
225 : }
226 8607 : m_bytesWritten += framed.size();
227 8607 : ++m_recordsWritten;
228 8607 : return true;
229 8607 : }
230 :
231 11 : bool StreamSpoolCache::Writer::Commit()
232 : {
233 11 : if (!m_openForWrite || m_committed)
234 : {
235 0 : return false;
236 : }
237 11 : Close(/*commit=*/true);
238 11 : return m_publishSucceeded;
239 : }
240 :
241 1 : void StreamSpoolCache::Writer::Abort()
242 : {
243 1 : if (!m_openForWrite || m_committed)
244 : {
245 0 : return;
246 : }
247 1 : Close(/*commit=*/false);
248 : }
249 :
250 12 : void StreamSpoolCache::Writer::Close(bool commit)
251 : {
252 12 : m_openForWrite = false;
253 12 : bool streamOk = true;
254 12 : if (m_stream.is_open())
255 : {
256 : // ofstream buffers; an I/O error such as ENOSPC typically
257 : // surfaces only at flush/close time. A failed flush means the
258 : // final bytes never reached the disk — publishing the file
259 : // would poison the warm path with a silently truncated spool.
260 12 : m_stream.flush();
261 12 : streamOk = m_stream.good();
262 12 : m_stream.close();
263 : }
264 :
265 12 : if (commit)
266 : {
267 11 : m_committed = true;
268 11 : m_publishSucceeded = streamOk && m_pCache != nullptr;
269 11 : if (m_publishSucceeded)
270 : {
271 33 : m_publishSucceeded = m_pCache->PublishEntry(m_designName,
272 11 : Entry{ m_path, m_generation, m_bytesWritten, m_recordsWritten, 0 });
273 : }
274 : else
275 : {
276 0 : std::error_code ec;
277 0 : std::filesystem::remove(m_path, ec); // best-effort
278 0 : logerror("StreamSpoolCache: spool flush/close failed for \"" + m_designName +
279 : "\"; incomplete spool deleted, not published (next request refills cold)");
280 : }
281 : }
282 : else
283 : {
284 1 : std::error_code ec;
285 1 : std::filesystem::remove(m_path, ec); // best-effort temp cleanup
286 : }
287 12 : }
288 :
289 : // ---- Cache ----
290 :
291 56 : StreamSpoolCache::StreamSpoolCache(std::uint64_t maxTotalBytes)
292 : {
293 : namespace fs = std::filesystem;
294 :
295 56 : std::error_code ec;
296 56 : const auto base = fs::temp_directory_path(ec);
297 56 : if (ec)
298 : {
299 0 : logerror("StreamSpoolCache: cannot resolve temp directory; spooling disabled");
300 0 : return;
301 : }
302 :
303 : // Unique per-instance directory (pid + steady ticks);
304 : // create_directory fails on collision, so retry with a fresh suffix.
305 56 : const auto pid = current_pid();
306 56 : for (int attempt = 0; attempt < 8; ++attempt)
307 : {
308 56 : const auto tick = static_cast<std::uint64_t>(
309 56 : std::chrono::steady_clock::now().time_since_epoch().count()) + attempt;
310 56 : const auto candidate = base / ("odbdesign-spool-" + std::to_string(pid) + "-" + std::to_string(tick));
311 56 : std::error_code createEc;
312 56 : if (fs::create_directory(candidate, createEc) && !createEc)
313 : {
314 56 : InitializeDirectory(candidate, /*ownsDirectory=*/true, maxTotalBytes);
315 56 : return;
316 : }
317 56 : }
318 0 : logerror("StreamSpoolCache: failed to create spool directory; spooling disabled");
319 56 : }
320 :
321 4 : StreamSpoolCache::StreamSpoolCache(const std::filesystem::path& directory, std::uint64_t maxTotalBytes)
322 : {
323 4 : InitializeDirectory(directory, /*ownsDirectory=*/false, maxTotalBytes);
324 4 : }
325 :
326 64 : StreamSpoolCache::~StreamSpoolCache()
327 : {
328 60 : if (m_ownsDirectory)
329 : {
330 56 : std::error_code ec;
331 56 : std::filesystem::remove_all(m_directory, ec); // best-effort
332 56 : return;
333 : }
334 : // Explicit (caller-owned) directory: still remove the spool files
335 : // THIS instance published — the constructor's documented contract.
336 : // Without it a clean shutdown leaves every entry behind, and a
337 : // restarted service neither indexes nor accounts for them, so
338 : // restarts accumulate multi-GB spools outside the configured cap.
339 : // Unrelated files are untouched; POSIX unlink-while-open keeps
340 : // any concurrent warm reader's fd valid through the removal.
341 4 : std::lock_guard<std::mutex> lock(m_mutex);
342 6 : for (const auto& kv : m_entriesByName)
343 : {
344 2 : std::error_code ec;
345 2 : std::filesystem::remove(kv.second.path, ec); // best-effort
346 : }
347 116 : }
348 :
349 60 : void StreamSpoolCache::InitializeDirectory(const std::filesystem::path& directory, bool ownsDirectory, std::uint64_t maxTotalBytes)
350 : {
351 : namespace fs = std::filesystem;
352 60 : std::error_code ec;
353 60 : fs::create_directories(directory, ec);
354 60 : if (ec || !fs::is_directory(directory))
355 : {
356 2 : logerror("StreamSpoolCache: spool directory unusable (" + directory.string() + "); spooling disabled");
357 1 : return;
358 : }
359 59 : m_directory = directory;
360 59 : m_ownsDirectory = ownsDirectory;
361 59 : m_maxTotalBytes = maxTotalBytes;
362 59 : m_usable = true;
363 : }
364 :
365 12 : std::filesystem::path StreamSpoolCache::MakeSpoolPath(const std::string& designName, std::uint64_t generation, std::uint64_t fillId) const
366 : {
367 12 : auto name = sanitize_name(designName);
368 12 : if (name.empty())
369 : {
370 0 : name = "design";
371 : }
372 : // pid uniquifies across processes sharing an explicit spool
373 : // directory; fillId uniquifies concurrent fills within one
374 : // process. Without the per-fill suffix two cold fills of the
375 : // same (design, generation) would open one path with
376 : // std::ios::trunc and interleave their length-delimited writes.
377 24 : return m_directory / (name + "-" + std::to_string(fnv1a64(designName)) + "-" +
378 48 : std::to_string(generation) + "-" +
379 48 : std::to_string(current_pid()) + "-" + std::to_string(fillId) + ".spool");
380 12 : }
381 :
382 17 : bool StreamSpoolCache::TryOpenReader(const std::string& designName, std::uint64_t generation, Reader& readerOut)
383 : {
384 17 : if (!m_usable)
385 : {
386 1 : return false;
387 : }
388 :
389 16 : std::filesystem::path path;
390 16 : std::uint64_t expectedRecords = 0;
391 : {
392 16 : std::lock_guard<std::mutex> lock(m_mutex);
393 16 : auto findIt = m_entriesByName.find(designName);
394 16 : if (findIt == m_entriesByName.end())
395 : {
396 8 : return false;
397 : }
398 8 : if (findIt->second.generation != generation)
399 : {
400 : // Stale spool (the design was invalidated/evicted/re-uploaded
401 : // since it was written): drop it and refill cold.
402 2 : std::error_code ec;
403 2 : std::filesystem::remove(findIt->second.path, ec);
404 2 : m_totalBytes -= findIt->second.bytes;
405 2 : m_entriesByName.erase(findIt);
406 2 : return false;
407 : }
408 6 : path = findIt->second.path;
409 6 : expectedRecords = findIt->second.records;
410 6 : findIt->second.lastUsed = ++m_useCounter;
411 16 : }
412 :
413 6 : std::error_code ec;
414 6 : if (!std::filesystem::is_regular_file(path, ec))
415 : {
416 : // Path-guarded: a concurrent PublishEntry (a different
417 : // generation, or a same-generation last-committer-wins fill)
418 : // may have swapped in a replacement and unlinked the old path
419 : // between the lock-guarded lookup above and this probe; only
420 : // the entry still referring to the file we observed may be
421 : // dropped.
422 0 : DropStaleEntry(designName, path);
423 0 : return false;
424 : }
425 :
426 6 : readerOut.m_stream.open(path, std::ios::binary);
427 6 : if (!readerOut.m_stream.is_open())
428 : {
429 0 : return false;
430 : }
431 6 : readerOut.m_recordsRead = 0;
432 : // Committed record count: a clean EOF short of it means the file
433 : // was truncated AT a record boundary (or emptied) — framing alone
434 : // cannot distinguish that from a legitimate end-of-spool, so the
435 : // reader carries the count and Truncated() reports the mismatch.
436 6 : readerOut.m_expectedRecords = expectedRecords;
437 6 : readerOut.m_failed = false;
438 :
439 6 : std::lock_guard<std::mutex> lock(m_mutex);
440 6 : ++m_warmOpens;
441 6 : return true;
442 16 : }
443 :
444 13 : bool StreamSpoolCache::BeginWrite(const std::string& designName, std::uint64_t generation, Writer& writerOut)
445 : {
446 13 : if (!m_usable)
447 : {
448 1 : return false;
449 : }
450 :
451 : // Each fill writes its OWN file (the path embeds a per-fill
452 : // suffix — see MakeSpoolPath): concurrent cold fills of the same
453 : // (design, generation) must never share a file, because the
454 : // ios::trunc open would interleave their length-delimited writes
455 : // and publish a corrupt spool. Publishing is a bookkeeping
456 : // insert, not a rename: the last committer wins, and
457 : // PublishEntry removes the loser's (distinct) file. A warm
458 : // reader already streaming the loser's file is safe on POSIX
459 : // (unlink-while-open keeps its fd valid); on Windows the remove
460 : // can fail while the handle is open — best-effort, the orphan is
461 : // reclaimed when an owned spool directory is torn down.
462 12 : std::uint64_t fillId = 0;
463 : {
464 : // Process-global counter (not a per-instance member): two
465 : // cache instances in one process sharing an explicit spool
466 : // directory would otherwise both start at fill 1 and produce
467 : // the same pathname, trunc-opening each other's spool. The
468 : // pid component in the path already uniquifies across
469 : // processes.
470 12 : fillId = ++g_fillCounter;
471 : }
472 12 : const auto path = MakeSpoolPath(designName, generation, fillId);
473 :
474 12 : writerOut.m_stream.open(path, std::ios::binary | std::ios::trunc);
475 12 : if (!writerOut.m_stream.is_open())
476 : {
477 0 : return false;
478 : }
479 12 : writerOut.m_pCache = this;
480 12 : writerOut.m_path = path;
481 12 : writerOut.m_designName = designName;
482 12 : writerOut.m_generation = generation;
483 12 : writerOut.m_bytesWritten = 0;
484 12 : writerOut.m_recordsWritten = 0;
485 12 : writerOut.m_committed = false;
486 12 : writerOut.m_openForWrite = true;
487 12 : return true;
488 12 : }
489 :
490 11 : bool StreamSpoolCache::PublishEntry(const std::string& designName, const Entry& entry)
491 : {
492 11 : std::lock_guard<std::mutex> lock(m_mutex);
493 11 : Entry published = entry;
494 11 : published.lastUsed = ++m_useCounter;
495 :
496 : // A single entry larger than the whole budget would evict every
497 : // other design and still leave the cache over cap indefinitely
498 : // (the just-published design is eviction-protected). Reject it
499 : // instead: the completed stream simply stays unspooled and every
500 : // request for this design refills cold — raise spool_max_bytes
501 : // if that design should be warm-servable.
502 11 : if (m_maxTotalBytes != 0 && published.bytes > m_maxTotalBytes)
503 : {
504 1 : std::error_code ec;
505 1 : std::filesystem::remove(published.path, ec); // best-effort
506 2 : logwarn("StreamSpoolCache: spool for \"" + designName + "\" (" +
507 : std::to_string(published.bytes) + " bytes) exceeds spool_max_bytes (" +
508 : std::to_string(m_maxTotalBytes) + "); not published (design serves cold)");
509 1 : return false;
510 : }
511 :
512 10 : auto findIt = m_entriesByName.find(designName);
513 10 : if (findIt != m_entriesByName.end())
514 : {
515 1 : std::error_code ec;
516 1 : std::filesystem::remove(findIt->second.path, ec); // best-effort
517 1 : m_totalBytes -= findIt->second.bytes;
518 1 : findIt->second = published;
519 : }
520 : else
521 : {
522 9 : m_entriesByName.emplace(designName, published);
523 : }
524 10 : m_totalBytes += published.bytes;
525 :
526 10 : EvictOverBudgetLocked(designName);
527 10 : return true;
528 11 : }
529 :
530 3 : void StreamSpoolCache::Drop(const std::string& designName)
531 : {
532 3 : std::lock_guard<std::mutex> lock(m_mutex);
533 3 : auto findIt = m_entriesByName.find(designName);
534 3 : if (findIt == m_entriesByName.end())
535 : {
536 0 : return;
537 : }
538 3 : std::error_code ec;
539 3 : std::filesystem::remove(findIt->second.path, ec); // best-effort
540 3 : m_totalBytes -= findIt->second.bytes;
541 3 : m_entriesByName.erase(findIt);
542 3 : }
543 :
544 0 : void StreamSpoolCache::DropStaleEntry(const std::string& designName, const std::filesystem::path& expectedPath)
545 : {
546 0 : std::lock_guard<std::mutex> lock(m_mutex);
547 0 : auto findIt = m_entriesByName.find(designName);
548 0 : if (findIt == m_entriesByName.end() || findIt->second.path != expectedPath)
549 : {
550 0 : return; // replaced or removed since it was observed - leave the new entry alone
551 : }
552 0 : std::error_code ec;
553 0 : std::filesystem::remove(findIt->second.path, ec); // best-effort
554 0 : m_totalBytes -= findIt->second.bytes;
555 0 : m_entriesByName.erase(findIt);
556 0 : }
557 :
558 10 : void StreamSpoolCache::EvictOverBudgetLocked(const std::string& protectedName)
559 : {
560 10 : if (m_maxTotalBytes == 0)
561 : {
562 0 : return;
563 : }
564 10 : while (m_totalBytes > m_maxTotalBytes)
565 : {
566 : // Least-recently-used victim, never the just-published design.
567 0 : auto victimIt = m_entriesByName.end();
568 0 : for (auto it = m_entriesByName.begin(); it != m_entriesByName.end(); ++it)
569 : {
570 0 : if (it->first == protectedName) continue;
571 0 : if (victimIt == m_entriesByName.end() || it->second.lastUsed < victimIt->second.lastUsed)
572 : {
573 0 : victimIt = it;
574 : }
575 : }
576 0 : if (victimIt == m_entriesByName.end())
577 : {
578 0 : break; // only the protected entry remains; tolerate over-budget
579 : }
580 0 : std::error_code ec;
581 0 : std::filesystem::remove(victimIt->second.path, ec); // best-effort
582 0 : m_totalBytes -= victimIt->second.bytes;
583 0 : m_entriesByName.erase(victimIt);
584 : }
585 : }
586 :
587 6 : std::uint64_t StreamSpoolCache::WarmOpens() const
588 : {
589 6 : std::lock_guard<std::mutex> lock(m_mutex);
590 6 : return m_warmOpens;
591 6 : }
592 :
593 1 : std::uint64_t StreamSpoolCache::TotalSpoolBytes() const
594 : {
595 1 : std::lock_guard<std::mutex> lock(m_mutex);
596 1 : return m_totalBytes;
597 1 : }
598 :
599 13 : std::size_t StreamSpoolCache::EntryCount() const
600 : {
601 13 : std::lock_guard<std::mutex> lock(m_mutex);
602 26 : return m_entriesByName.size();
603 13 : }
604 : }
605 : }
|